Skip to content

neurodamus build and test #1381

neurodamus build and test

neurodamus build and test #1381

name: neurodamus build and test
on:
schedule:
# every evening at 20:00 UTC
- cron: '0 20 * * *'
pull_request:
branches:
- main
push:
branches:
- main
workflow_dispatch:
inputs:
NEURON_BRANCH:
description: 'NEURON branch to use'
required: false
LIBSONATA_REPORT_BRANCH:
description: 'libsonatareport branch to use'
required: false
LIBSONATA_BRANCH:
description: 'libsonata branch to use'
required: false
NEURODAMUS_MODELS_BRANCH:
description: 'neurodamus-models branch to use'
required: false
env:
NEURON_COMMIT_ID: 'c48d7d5'
RDMAV_FORK_SAFE: '1'
jobs:
simulation:
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ubuntu-22.04]
python-version: ['3.9.20', '3.10.15', '3.11.10', '3.12.6']
steps:
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- name: Checkout neurodamus repository
uses: actions/checkout@v4
with:
ref: ${{ github.event.pull_request.head.sha }}
fetch-depth: 0
- name: Cache pip
uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{ matrix.os }}-pip-${{ matrix.python-version }}-v1
restore-keys: |
${{ matrix.os }}-pip-
- name: Get latest tags
run: |
echo "NEURON_BRANCH=${{ inputs.NEURON_BRANCH || github.event.inputs.NEURON_BRANCH || 'master' }}" >> $GITHUB_ENV
LIBSONATA_LATEST=$(curl -s https://api.github.com/repos/BlueBrain/libsonata/releases/latest | grep -oP '"tag_name": "\K(.*)(?=")')
echo "LIBSONATA_BRANCH=${{ inputs.LIBSONATA_BRANCH || github.event.inputs.LIBSONATA_BRANCH || '$LIBSONATA_LATEST' }}" >> $GITHUB_ENV
LIBSONATA_REPORT_LATEST=$(curl -s https://api.github.com/repos/BlueBrain/libsonatareport/releases/latest | grep -oP '"tag_name": "\K(.*)(?=")')
echo "LIBSONATA_REPORT_BRANCH=${{ inputs.LIBSONATA_REPORT_BRANCH || github.event.inputs.LIBSONATA_REPORT_BRANCH || '$LIBSONATA_REPORT_LATEST' }}" >> $GITHUB_ENV
NEURODAMUS_MODELS_LATEST=$(curl -s https://api.github.com/repos/BlueBrain/neurodamus-models/releases/latest | grep -oP '"tag_name": "\K(.*)(?=")')
echo "NEURODAMUS_MODELS_BRANCH=${{ inputs.NEURODAMUS_MODELS_BRANCH || github.event.inputs.NEURODAMUS_MODELS_BRANCH || '$NEURODAMUS_MODELS_LATEST' }}" >> $GITHUB_ENV
- name: Get HEAD commit message and look for branches
run: |
COMMIT_MESSAGE=$(git log -1 --pretty=%B)
echo $COMMIT_MESSAGE
NEURON_BRANCH=$(echo $COMMIT_MESSAGE | grep -Po 'NEURON_BRANCH=\K[0-9a-zA-Z/_.\-]*' || true)
if [[ ! -z $NEURON_BRANCH ]]; then echo "NEURON_BRANCH=$NEURON_BRANCH" >> $GITHUB_ENV; fi
LIBSONATA_REPORT_BRANCH=$(echo $COMMIT_MESSAGE | grep -Po 'LIBSONATA_REPORT_BRANCH=\K[0-9a-zA-Z/_.\-]*' || true)
if [[ ! -z $LIBSONATA_REPORT_BRANCH ]]; then echo "LIBSONATA_REPORT_BRANCH=$LIBSONATA_REPORT_BRANCH" >> $GITHUB_ENV; fi
LIBSONATA_BRANCH=$(echo $COMMIT_MESSAGE | grep -Po 'LIBSONATA_BRANCH=\K[0-9a-zA-Z/_.\-]*' || true)
if [[ ! -z $LIBSONATA_BRANCH ]]; then echo "LIBSONATA_BRANCH=$LIBSONATA_BRANCH" >> $GITHUB_ENV; fi
- name: Install system dependencies
run: |
sudo apt-get update
sudo apt-get install -y mpich libmpich-dev libhdf5-mpich-dev hdf5-tools flex libfl-dev bison ninja-build
- name: Cache python virtual env
id: cache-venv
uses: actions/cache@v3
env:
cache-name: cache-venv
with:
path: venv
key: ${{ matrix.os }}-libsonata-${{ env.LIBSONATA_BRANCH }}-py${{ matrix.python-version }}
- name: Upgrade pip and install base Python packages
if: steps.cache-venv.outputs.cache-hit != 'true'
run: |
python -m venv venv
. ./venv/bin/activate
python -m pip install --upgrade pip setuptools
pip install cython numpy wheel pkgconfig
- name: Add virtual environment to PATH
run: echo "${{ github.workspace }}/venv/bin" >> $GITHUB_PATH
- name: Install libsonata
if: steps.cache-venv.outputs.cache-hit != 'true'
run: |
CC=mpicc CXX=mpic++ pip install git+https://github.com/BlueBrain/libsonata@${{ env.LIBSONATA_BRANCH }}
- name: Cache libsonatareport
id: cache-libsonatareport
uses: actions/cache@v3
env:
cache-name: cache-libsonatareport
with:
path: libsonatareport
key: ${{ matrix.os }}-libsonatareport-${{ env.LIBSONATA_REPORT_BRANCH }}-py${{ matrix.python-version }}
- name: Install libsonatareport
if: steps.cache-libsonatareport.outputs.cache-hit != 'true'
run: |
git clone --branch="${{ env.LIBSONATA_REPORT_BRANCH}}" https://github.com/BlueBrain/libsonatareport.git --recursive --depth=1
cd libsonatareport
mkdir build && cd build
cmake -DCMAKE_INSTALL_PREFIX=$(pwd)/install -DCMAKE_BUILD_TYPE=Release -DSONATA_REPORT_ENABLE_SUBMODULES=ON -DSONATA_REPORT_ENABLE_MPI=ON ..
cmake --build . --parallel
cmake --build . --target install
- name: Cache NEURON
id: cache-neuron
uses: actions/cache@v3
env:
cache-name: cache-neuron
with:
path: nrn
key: ${{ matrix.os }}-neuron-${{ env.NEURON_BRANCH }}-${{ env.NEURON_COMMIT_ID }}-py${{ matrix.python-version }}
- name: Install NEURON
if: steps.cache-neuron.outputs.cache-hit != 'true'
run: |
export SONATAREPORT_DIR=$(pwd)/libsonatareport/build/install
if [[ ${{ env.NEURON_BRANCH }} == 'master' ]]; then
git clone --branch=${{ env.NEURON_BRANCH }} https://github.com/neuronsimulator/nrn.git
cd nrn
git checkout ${{ env.NEURON_COMMIT_ID }}
cd ..
else
git clone --branch=${{ env.NEURON_BRANCH }} https://github.com/neuronsimulator/nrn.git --depth=1
fi
python -m pip install --upgrade pip -r nrn/nrn_requirements.txt
cmake -B nrn/build -S nrn -G Ninja \
-DPYTHON_EXECUTABLE=$(which python) \
-DCMAKE_INSTALL_PREFIX=$(pwd)/nrn/build/install \
-DNRN_ENABLE_MPI=ON \
-DNRN_ENABLE_INTERVIEWS=OFF \
-DNRN_ENABLE_CORENEURON=ON \
-DCMAKE_C_COMPILER=gcc \
-DCMAKE_CXX_COMPILER=g++ \
-DCORENRN_ENABLE_REPORTING=ON \
-DCMAKE_PREFIX_PATH=$SONATAREPORT_DIR
cmake --build nrn/build --parallel
cmake --build nrn/build --target install
- name: Install mpi4py
if: steps.cache-venv.outputs.cache-hit != 'true'
run: |
pip install mpi4py
- name: Build h5py with the local hdf5
run: |
CC="mpicc" HDF5_MPI="ON" HDF5_INCLUDEDIR=/usr/include/hdf5/mpich HDF5_LIBDIR=/usr/lib/x86_64-linux-gnu/hdf5/mpich \
pip install --no-cache-dir --no-binary=h5py h5py --no-build-isolation
- name: Install neurodamus
run: |
pip install .[full]
- name: Build neocortex models
run: |
export SONATAREPORT_DIR=$(pwd)/libsonatareport/build/install
export PATH=$(pwd)/nrn/build/install/bin:$PATH
# Clone neurodamus-models repository
git clone --branch=${{ env.NEURODAMUS_MODELS_BRANCH }} https://github.com/BlueBrain/neurodamus-models.git
# Build neocortex model
DATADIR=$(python -c "import neurodamus; from pathlib import Path; print(Path(neurodamus.__file__).parent / 'data')")
cmake -B neurodamus-models/build -S neurodamus-models/ \
-DCMAKE_INSTALL_PREFIX=$PWD/neurodamus-models/install \
-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON \
-DCMAKE_PREFIX_PATH=$SONATAREPORT_DIR \
-DNEURODAMUS_CORE_DIR=${DATADIR} \
-DNEURODAMUS_MECHANISMS=neocortex \
-DNEURODAMUS_NCX_V5=ON
cmake --build neurodamus-models/build
cmake --install neurodamus-models/build
echo "NEURODAMUS_NEOCORTEX_ROOT=$(pwd)/neurodamus-models/install" >> $GITHUB_ENV;
# Build neocortex-multiscale model
echo "build neurodamus-neocortex-multiscale model"
cmake -B neurodamus-models/build_multiscale -S neurodamus-models/ \
-DCMAKE_INSTALL_PREFIX=$PWD/neurodamus-models/install_multiscale \
-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON \
-DCMAKE_PREFIX_PATH=$SONATAREPORT_DIR \
-DNEURODAMUS_CORE_DIR=${DATADIR} \
-DNEURODAMUS_MECHANISMS=neocortex \
-DNEURODAMUS_NCX_METABOLISM=ON \
-DNEURODAMUS_NCX_NGV=ON \
-DNEURODAMUS_ENABLE_CORENEURON=OFF
cmake --build neurodamus-models/build_multiscale
cmake --install neurodamus-models/build_multiscale
echo "NEURODAMUS_NEOCORTEX_MULTISCALE_ROOT=$(pwd)/neurodamus-models/install_multiscale" >> $GITHUB_ENV;
- name: load neocortex model
run: |
export PYTHONPATH=$(pwd)/nrn/build/install/lib/python:$PYTHONPATH
export HOC_LIBRARY_PATH=$NEURODAMUS_NEOCORTEX_ROOT/share/neurodamus_neocortex/hoc
export NEURODAMUS_PYTHON=$(pwd)/neurodamus/data
export CORENEURONLIB=$NEURODAMUS_NEOCORTEX_ROOT/lib/libcorenrnmech.so
export NRNMECH_LIB_PATH=$NEURODAMUS_NEOCORTEX_ROOT/lib/libnrnmech.so
export PATH=$NEURODAMUS_NEOCORTEX_ROOT/bin:$PATH
echo "PYTHONPATH=$PYTHONPATH" >> $GITHUB_ENV;
echo "HOC_LIBRARY_PATH=$HOC_LIBRARY_PATH" >> $GITHUB_ENV;
echo "CORENEURONLIB=$CORENEURONLIB" >> $GITHUB_ENV;
echo "NRNMECH_LIB_PATH=$NRNMECH_LIB_PATH" >> $GITHUB_ENV;
echo "NEURODAMUS_PYTHON=$NEURODAMUS_PYTHON" >> $GITHUB_ENV;
which special
# launch simulation with NEURON
cd tests/simulations/usecase3/
mpirun -np 2 special -mpi -python $NEURODAMUS_PYTHON/init.py --configFile=simulation_sonata.json
ls reporting/*.h5
# launch simulation with CORENEURON
mpirun -np 2 special -mpi -python $NEURODAMUS_PYTHON/init.py --configFile=simulation_sonata_coreneuron.json
ls reporting_coreneuron/*.h5
- name: Install test dependencies
run: |
pip install -r tests/requirements.txt
- name: test-unit
run: |
pytest tests/unit
- name: test-integration
run: |
pytest -x --forked tests/integration
- name: test-integration-e2e
run: |
pytest -s -x --forked --durations=5 --durations-min=15 tests/integration-e2e
- name: test-scientific
run: |
pytest -s -x --forked --durations=5 --durations-min=15 tests/scientific
- name: test-scientific-ngv
run: |
export HOC_LIBRARY_PATH=$NEURODAMUS_NEOCORTEX_MULTISCALE_ROOT/share/neurodamus_neocortex/hoc
export NRNMECH_LIB_PATH=$NEURODAMUS_NEOCORTEX_MULTISCALE_ROOT/lib/libnrnmech.so
unset CORENEURONLIB
pytest -s -x --forked --durations=5 --durations-min=15 tests/scientific-ngv
- name: live debug session, comment out
if: failure()
uses: mxschmitt/action-tmate@v3
with:
timeout-minutes: 60 # or any other value you need