diff --git a/.github/scripts/nightly.sh b/.github/scripts/nightly.sh new file mode 100644 index 0000000000..ec5ba9af7b --- /dev/null +++ b/.github/scripts/nightly.sh @@ -0,0 +1,106 @@ +#!/bin/bash + +########################################################## +# Build and test the GTSAM Python wrapper. +########################################################## + +set -x -e + +# install TBB with _debug.so files +function install_tbb() +{ + echo install_tbb + if [ "$(uname)" == "Linux" ]; then + sudo apt-get -y install libtbb-dev + + elif [ "$(uname)" == "Darwin" ]; then + brew install tbb + fi +} + +if [ -z ${PYTHON_VERSION+x} ]; then + echo "Please provide the Python version to build against!" + exit 127 +fi + +export PYTHON="python${PYTHON_VERSION}" + +function install_dependencies() +{ + if [[ $(uname) == "Darwin" ]]; then + brew install wget + else + # Install a system package required by our library + sudo apt-get install -y wget libicu-dev python3-pip python3-setuptools + fi + + export PATH=$PATH:$($PYTHON -c "import site; print(site.USER_BASE)")/bin + + if [ "${GTSAM_WITH_TBB:-OFF}" == "ON" ]; then + install_tbb + fi +} + +# Chirality exception is thrown by the camera projection when the solver +# randomly places the camera such that the landmarks are behind it. +# This breaks the dependent python custom factor with numeric derivatives. + +# Build needs to be static to be included in the wheel at the end. + +function build() +{ + export CMAKE_GENERATOR=Ninja + BUILD_PYBIND="ON" + cmake $GITHUB_WORKSPACE \ + -B build \ + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ + -DGTSAM_BUILD_TESTS=OFF \ + -DGTSAM_FORCE_SHARED_LIB=OFF \ + -DGTSAM_BUILD_UNSTABLE=${GTSAM_BUILD_UNSTABLE:-ON} \ + -DGTSAM_THROW_CHEIRALITY_EXCEPTION=OFF \ + -DGTSAM_USE_QUATERNIONS=OFF \ + -DGTSAM_WITH_TBB=${GTSAM_WITH_TBB:-OFF} \ + -DGTSAM_BUILD_EXAMPLES_ALWAYS=OFF \ + -DGTSAM_BUILD_WITH_MARCH_NATIVE=OFF \ + -DGTSAM_BUILD_PYTHON=${BUILD_PYBIND} \ + -DBUILD_SHARED_LIBS=OFF \ + -DBUILD_STATIC_METIS=ON \ + -DGTSAM_UNSTABLE_BUILD_PYTHON=${GTSAM_BUILD_UNSTABLE:-ON} \ + -DGTSAM_PYTHON_VERSION=$PYTHON_VERSION \ + -DPYTHON_EXECUTABLE:FILEPATH=$(which $PYTHON) \ + -DGTSAM_ALLOW_DEPRECATED_SINCE_V43=OFF \ + -DCMAKE_INSTALL_PREFIX=$GITHUB_WORKSPACE/gtsam_install + + + # Set to 2 cores so that Actions does not error out during resource provisioning. + cmake --build build -j2 + + cmake --build build --target python-install +} + +function test() +{ + cd $GITHUB_WORKSPACE/python/gtsam/tests + $PYTHON -m unittest discover -v + cd $GITHUB_WORKSPACE + + cd $GITHUB_WORKSPACE/python/gtsam_unstable/tests + $PYTHON -m unittest discover -v + cd $GITHUB_WORKSPACE + + # cmake --build build --target python-test + # cmake --build build --target python-test-unstable +} + +# select between build or test +case $1 in + -d) + install_dependencies + ;; + -b) + build + ;; + -t) + test + ;; +esac diff --git a/.github/workflows/arm-nightly.yml b/.github/workflows/arm-nightly.yml new file mode 100644 index 0000000000..348db80447 --- /dev/null +++ b/.github/workflows/arm-nightly.yml @@ -0,0 +1,111 @@ +# Nightly build for Raspberry Pi +# Uploads to test.pypi.org. + +name: ARM Nightly + +on: [workflow_dispatch] + +env: + IMG: raspios_lite_arm64:latest + BIND: true + CPU: cortex-a76 + +jobs: + build: + name: Build + runs-on: ubuntu-24.04 + # strategy: + # fail-fast: false + # matrix: + # python_version: ['3.9', '3.10', '3.11', '3.12'] + + + env: + # quote the number because yaml thinks 3.10 and 3.1 are the same, like it's a number, man. + # https://yaml.org/spec/1.2.2/ the yaml spec is 50 pages + PYTHON_VERSION: '3.11' + # PYTHON_VERSION: ${{matrix.python_version}} + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: env + run: | + echo "=== SET VARS ===" + echo "CC=gcc" >> $GITHUB_ENV + echo "CXX=g++" >> $GITHUB_ENV + echo "NIGHTLY=1" >> $GITHUB_ENV + + - name: Run on ARM + uses: pguyot/arm-runner-action@v2 + with: + base_image: $IMG + bind_mount_repository: $BIND + copy_repository_path: /home/runner/work/gtsam/gtsam + cpu: $CPU + debug: true + image_additional_mb: 20000 + commands: | + echo "=== CPU INFO ===" + cat /proc/cpuinfo + echo "=== UPDATE ===" + sudo apt-get -y update + echo "=== INSTALL 1 ===" + sudo apt-get -y install python3 cmake build-essential pkg-config python3-numpy + echo "=== INSTALL 2 ===" + sudo apt-get -y install libboost-all-dev ninja-build + echo "=== INSTALL 3 ===" + sudo apt-get -y install g++ patchelf + echo "=== DEPEND ===" + bash .github/scripts/nightly.sh -d + echo "=== REQ ===" + python3 -m pip install --break-system-packages -r python/dev_requirements.txt + echo "=== PWD ===" + pwd + echo "=== ls ===" + ls -R + echo "=== BUILD ===" + bash .github/scripts/nightly.sh -b + echo "=== TEST ===" + bash .github/scripts/nightly.sh -t + echo "=== INSTALL AUDITWHEEL ===" + python3 -m pip install --break-system-packages --upgrade pip auditwheel + echo "=== BUILD WHEEL ===" + python3 -m pip wheel build/python -w build/python/dist + echo "=== CHECK LDD VERSION ===" + ldd --version + echo "=== AUDITWHEEL ===" + python3 -m auditwheel repair --plat manylinux_2_35_aarch64 -w build/python/repaired/ build/python/dist/truher_gtsam_nightly* + + + - name: Upload Artifacts + uses: actions/upload-artifact@v4 + with: + # TODO: define the name as ENV, use it in CMakeLists.txt + name: artifact311 + # name: artifact${{matrix.python_version}} + path: build/python/repaired/truher_gtsam_nightly* + + + upload_all: + # Waits for all the builds to finish + # Downloads all their wheels + # Uploads them all at once. + name: Upload All + needs: build + runs-on: ubuntu-latest + permissions: + id-token: write + steps: + - name: Download Artifacts + uses: actions/download-artifact@v4 + with: + path: dist/ + merge-multiple: true + + - name: Publish + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: dist/ + repository-url: https://test.pypi.org/legacy/ \ No newline at end of file diff --git a/.github/workflows/build-linux.yml b/.github/workflows/build-linux.yml index da398ad235..6d1c4ce0c5 100644 --- a/.github/workflows/build-linux.yml +++ b/.github/workflows/build-linux.yml @@ -1,6 +1,6 @@ name: Linux CI -on: [pull_request] +on: [pull_request, workflow_dispatch] # Every time you make a push to your PR, it cancel immediately the previous checks, # and start a new one. The other runner will be available more quickly to your PR. diff --git a/.github/workflows/build-macos.yml b/.github/workflows/build-macos.yml index e519164ec3..233cb9b4a7 100644 --- a/.github/workflows/build-macos.yml +++ b/.github/workflows/build-macos.yml @@ -1,6 +1,6 @@ name: macOS CI -on: [pull_request] +on: [pull_request, workflow_dispatch] # Every time you make a push to your PR, it cancel immediately the previous checks, # and start a new one. The other runner will be available more quickly to your PR. diff --git a/.github/workflows/build-python.yml b/.github/workflows/build-python.yml index 24a7f6c90e..f9ced83256 100644 --- a/.github/workflows/build-python.yml +++ b/.github/workflows/build-python.yml @@ -1,6 +1,6 @@ name: Python CI -on: [pull_request] +on: [pull_request, workflow_dispatch] # Every time you make a push to your PR, it cancel immediately the previous checks, # and start a new one. The other runner will be available more quickly to your PR. diff --git a/.github/workflows/build-special.yml b/.github/workflows/build-special.yml index 3a7dd974dc..abd0af7fc4 100644 --- a/.github/workflows/build-special.yml +++ b/.github/workflows/build-special.yml @@ -1,6 +1,6 @@ name: Special Cases CI -on: [pull_request] +on: [pull_request, workflow_dispatch] # Every time you make a push to your PR, it cancel immediately the previous checks, # and start a new one. The other runner will be available more quickly to your PR. diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index 20b4a846f9..89cbe61edd 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -1,6 +1,6 @@ name: Windows CI -on: [pull_request] +on: [pull_request, workflow_dispatch] # Every time you make a push to your PR, it cancel immediately the previous checks, # and start a new one. The other runner will be available more quickly to your PR. diff --git a/.github/workflows/linux-nightly.yml b/.github/workflows/linux-nightly.yml new file mode 100644 index 0000000000..24bb78dfd4 --- /dev/null +++ b/.github/workflows/linux-nightly.yml @@ -0,0 +1,104 @@ +# Nightly build for Linux. +# Uploads to test.pypi.org. + +name: Linux Nightly + +on: [workflow_dispatch] + +jobs: + build: + name: Build + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python_version: ['3.9', '3.10', '3.11', '3.12'] + + + env: + # PYTHON_VERSION: '3.10' + PYTHON_VERSION: ${{matrix.python_version}} + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + #python-version: '3.10' + python-version: ${{matrix.python_version}} + + - name: Install (Linux) + run: | + sudo apt-get -y update + sudo apt-get -y install cmake build-essential pkg-config python3-numpy libboost-all-dev ninja-build + + sudo apt-get install -y g++ g++-multilib + echo "CC=gcc" >> $GITHUB_ENV + echo "CXX=g++" >> $GITHUB_ENV + + - name: Set Swap Space (Linux) + uses: pierotofy/set-swap-space@master + with: + swap-size-gb: 6 + + - name: Install System Dependencies + run: bash .github/scripts/nightly.sh -d + + - name: Install Python Dependencies + shell: bash + run: python3 -m pip install -r python/dev_requirements.txt + + - name: Set Nightly Flag + run: echo "NIGHTLY=1" >> $GITHUB_ENV + + - name: Build + # Builds the cmake "python-install" target which is a local install. + shell: bash + run: bash .github/scripts/nightly.sh -b + + - name: Test + # Uses the local install for python3 -m unittest. + shell: bash + run: bash .github/scripts/python.sh -t + + - name: Install Build Tools + run: python3 -m pip install --upgrade pip auditwheel + + - name: Build for Publishing + run: python3 -m pip wheel build/python -w build/python/dist + + - name: Repair Wheels + # TODO: use the actual glibc version here + run: python3 -m auditwheel repair --plat manylinux_2_35_x86_64 -w build/python/repaired/ build/python/dist/truher_gtsam_nightly* + + - name: Upload Artifacts + uses: actions/upload-artifact@v4 + with: + # TODO: define the name as ENV, use it in CMakeLists.txt + name: artifact${{matrix.python_version}} + path: build/python/repaired/truher_gtsam_nightly* + + + upload_all: + # Waits for all the builds to finish + # Downloads all their wheels + # Uploads them all at once. + name: Upload All + needs: build + runs-on: ubuntu-latest + permissions: + id-token: write + steps: + - name: Download Artifacts + uses: actions/download-artifact@v4 + with: + path: dist/ + merge-multiple: true + + - name: Publish + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: dist/ + repository-url: https://test.pypi.org/legacy/ \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index e3b462eec5..eea7029d81 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -25,10 +25,19 @@ set (GTSAM_VERSION_PATCH 0) set (GTSAM_PRERELEASE_VERSION "a0") math (EXPR GTSAM_VERSION_NUMERIC "10000 * ${GTSAM_VERSION_MAJOR} + 100 * ${GTSAM_VERSION_MINOR} + ${GTSAM_VERSION_PATCH}") -if ("${GTSAM_PRERELEASE_VERSION}" STREQUAL "") +# Use the current timestamp as the version string, if the "NIGHTLY" flag is set. +if (DEFINED ENV{NIGHTLY}) + string(TIMESTAMP NOW "%Y.%m.%d.%H.%M") + message(NOTICE "NIGHTLY BUILD ${NOW}") + set (GTSAM_VERSION_STRING "${NOW}") + # TODO: use the correct pypi nightly name + set (SETUP_NAME "truher-gtsam-nightly") +elseif ("${GTSAM_PRERELEASE_VERSION}" STREQUAL "") set (GTSAM_VERSION_STRING "${GTSAM_VERSION_MAJOR}.${GTSAM_VERSION_MINOR}.${GTSAM_VERSION_PATCH}") + set (SETUP_NAME "gtsam") else() set (GTSAM_VERSION_STRING "${GTSAM_VERSION_MAJOR}.${GTSAM_VERSION_MINOR}${GTSAM_PRERELEASE_VERSION}") + set (SETUP_NAME "gtsam") endif() project(GTSAM @@ -70,6 +79,7 @@ include(GtsamPrinting) ############### Decide on BOOST ###################################### # Enable or disable serialization with GTSAM_ENABLE_BOOST_SERIALIZATION +# (Serialization tests fail when this is off.) option(GTSAM_ENABLE_BOOST_SERIALIZATION "Enable Boost serialization" ON) if(GTSAM_ENABLE_BOOST_SERIALIZATION) add_definitions(-DGTSAM_ENABLE_BOOST_SERIALIZATION) diff --git a/gtsam/3rdparty/cephes/CMakeLists.txt b/gtsam/3rdparty/cephes/CMakeLists.txt index 190b73db9b..aafc45aa9c 100644 --- a/gtsam/3rdparty/cephes/CMakeLists.txt +++ b/gtsam/3rdparty/cephes/CMakeLists.txt @@ -89,7 +89,13 @@ set(CEPHES_SOURCES cephes/zetac.c) # Add library source files -add_library(cephes-gtsam SHARED ${CEPHES_SOURCES}) +if (DEFINED ENV{NIGHTLY}) + # Needs to be static for the nightly build to work. + add_definitions(-fPIC) + add_library(cephes-gtsam STATIC ${CEPHES_SOURCES}) +else() + add_library(cephes-gtsam SHARED ${CEPHES_SOURCES}) +endif() # Add include directory (aka headers) target_include_directories( diff --git a/gtsam/nonlinear/nonlinear.i b/gtsam/nonlinear/nonlinear.i index 09c234630e..5d15c11634 100644 --- a/gtsam/nonlinear/nonlinear.i +++ b/gtsam/nonlinear/nonlinear.i @@ -715,6 +715,9 @@ virtual class BatchFixedLagSmoother : gtsam::FixedLagSmoother { void print(string s = "BatchFixedLagSmoother:\n") const; gtsam::LevenbergMarquardtParams params() const; + + gtsam::NonlinearFactorGraph getFactors() const; + template diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index f0fc3f796c..f5ddf95712 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -7,8 +7,16 @@ endif() # Generate setup.py. file(READ "${PROJECT_SOURCE_DIR}/README.md" README_CONTENTS) -configure_file(${PROJECT_PYTHON_SOURCE_DIR}/setup.py.in - ${GTSAM_PYTHON_BUILD_DIRECTORY}/setup.py) + +if (DEFINED ENV{NIGHTLY}) + # TODO: move all the builds to pyproject.toml + configure_file(${PROJECT_PYTHON_SOURCE_DIR}/pyproject.toml.in + ${GTSAM_PYTHON_BUILD_DIRECTORY}/pyproject.toml) +else() + configure_file(${PROJECT_PYTHON_SOURCE_DIR}/setup.py.in + ${GTSAM_PYTHON_BUILD_DIRECTORY}/setup.py) +endif() + # Supply MANIFEST.in for older versions of Python file(COPY ${PROJECT_PYTHON_SOURCE_DIR}/MANIFEST.in @@ -300,11 +308,20 @@ endif() # Note below we make sure to install with --user iff not in a virtualenv set(GTSAM_PYTHON_INSTALL_TARGET python-install) -add_custom_target(${GTSAM_PYTHON_INSTALL_TARGET} +if (DEFINED ENV{NIGHTLY}) + # nightly requires breaking? TODO: figure this out + add_custom_target(${GTSAM_PYTHON_INSTALL_TARGET} + COMMAND ${PYTHON_EXECUTABLE} -c "import sys, subprocess; cmd = [sys.executable, '-m', 'pip', 'install', '--break-system-packages']; has_venv = hasattr(sys, 'real_prefix') or (hasattr(sys, 'base_prefix') and sys.base_prefix != sys.prefix); cmd.append('--user' if not has_venv else ''); cmd.append('.'); subprocess.check_call([c for c in cmd if c])" + DEPENDS ${GTSAM_PYTHON_DEPENDENCIES} ${GTSAM_PYTHON_INSTALL_EXTRA} + WORKING_DIRECTORY ${GTSAM_PYTHON_BUILD_DIRECTORY} + VERBATIM) +else() + add_custom_target(${GTSAM_PYTHON_INSTALL_TARGET} COMMAND ${PYTHON_EXECUTABLE} -c "import sys, subprocess; cmd = [sys.executable, '-m', 'pip', 'install']; has_venv = hasattr(sys, 'real_prefix') or (hasattr(sys, 'base_prefix') and sys.base_prefix != sys.prefix); cmd.append('--user' if not has_venv else ''); cmd.append('.'); subprocess.check_call([c for c in cmd if c])" DEPENDS ${GTSAM_PYTHON_DEPENDENCIES} ${GTSAM_PYTHON_INSTALL_EXTRA} WORKING_DIRECTORY ${GTSAM_PYTHON_BUILD_DIRECTORY} VERBATIM) +endif() # Custom make command to run all GTSAM Python tests add_custom_target( diff --git a/python/pyproject.toml.in b/python/pyproject.toml.in new file mode 100644 index 0000000000..c9081bf3ed --- /dev/null +++ b/python/pyproject.toml.in @@ -0,0 +1,55 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[project] +name = "${SETUP_NAME}" +version = "${GTSAM_VERSION_STRING}" +description = "Georgia Tech Smoothing And Mapping library" +readme = "README.md" +license = {text = "Simplified BSD license"} +authors = [ + { name = "Frank Dellaert et. al.", email = "frank.dellaert@gtsam.org" }, +] +keywords = [ + "localization", + "mapping", + "optimization", + "robotics", + "sam", + "slam", +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: Education", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: BSD License", + "Operating System :: MacOS", + "Operating System :: Microsoft :: Windows", + "Operating System :: POSIX", + "Programming Language :: Python :: 3", +] +dependencies = [ + "numpy>=1.11.0,<2", +] + +[project.urls] +Homepage = "https://gtsam.org/" + +[tool.setuptools] +ext-modules = [ + {name = "gtsam", sources = []}, + {name = "gtsam_unstable", sources = []} +] + +[tool.setuptools.packages.find] +where = ["."] +exclude = ['build', 'build.*', 'CMakeFiles', 'CMakeFiles.*', + 'gtsam.notebooks', '*.preamble', '*.specializations', 'dist'] + +[tool.setuptools.package-data] +"*" = ["./*.so", + "./*.dll", + "./*.pyd", + "*.pyi", "**/*.pyi"] \ No newline at end of file diff --git a/python/setup.py.in b/python/setup.py.in index b9d7392c7e..8cd9c8fec2 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -21,7 +21,8 @@ package_data = { readme_contents = open("${GTSAM_SOURCE_DIR}/README.md").read() setup( - name='gtsam', + # This is set in the top-level CMakeLists.txt. + name='${SETUP_NAME}', description='Georgia Tech Smoothing And Mapping library', url='https://gtsam.org/', version='${GTSAM_VERSION_STRING}', # https://www.python.org/dev/peps/pep-0440/