Expand all our workflows to run on Python version 3.12 Issue #579 (#591) #631
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: cornflow-client tests | |
| on: | |
| pull_request: | |
| types: [ opened, edited, synchronize, reopened ] | |
| paths: | |
| - 'libs/client/**' | |
| - '.github/workflows/test_cornflow_client.yml' | |
| - '!libs/client/README.rst' | |
| - '!libs/client/setup.py' | |
| push: | |
| branches: | |
| - master | |
| - develop | |
| paths: | |
| - 'libs/client/**' | |
| - '.github/workflows/test_cornflow_client.yml' | |
| - '!libs/client/README.rst' | |
| - '!libs/client/setup.py' | |
| jobs: | |
| unit: | |
| runs-on: ${{ matrix.os }} | |
| name: Run unit test on cornflow-client | |
| defaults: | |
| run: | |
| working-directory: ./libs/client | |
| strategy: | |
| max-parallel: 21 | |
| matrix: | |
| python-version: [ 3.9, '3.10', '3.11', '3.12' ] | |
| os: [ubuntu-latest, macOS-latest, windows-latest] | |
| steps: | |
| - uses: actions/checkout@v1 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements-dev.txt | |
| pip install . | |
| - name: Test | |
| run: | | |
| coverage run --source=./cornflow_client/ --rcfile=./.coveragerc -m unittest discover -s cornflow_client/tests/unit | |
| coverage report -m | |
| integration: | |
| name: Run all test suites on cornflow-client | |
| needs: unit | |
| runs-on: ${{ matrix.os }} | |
| defaults: | |
| run: | |
| working-directory: ./libs/client | |
| strategy: | |
| max-parallel: 21 | |
| matrix: | |
| python-version: [3.9, '3.10', '3.11', '3.12'] | |
| os: [ ubuntu-latest ] | |
| steps: | |
| - uses: actions/checkout@v1 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v1 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Copy DAG files | |
| run: | | |
| cd .. | |
| cd .. | |
| cp -r cornflow-dags/DAG/* cornflow-server/airflow_config/dags/ | |
| cp cornflow-dags/requirements.txt cornflow-server/airflow_config/ | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements-dev.txt | |
| pip install . | |
| - name: Install airflow | |
| run: | | |
| cd .. | |
| cd .. | |
| cd cornflow-server | |
| python -m venv afvenv | |
| source afvenv/bin/activate | |
| AIRFLOW_VERSION=2.9.1 | |
| PYTHON_VERSION="$(python3 --version | cut -d " " -f 2 | cut -d "." -f 1-2)" | |
| CLIENT_BRANCH="${{ github.head_ref || github.ref_name }}" | |
| CONSTRAINT_URL="https://raw.githubusercontent.com/apache/airflow/constraints-${AIRFLOW_VERSION}/constraints-${PYTHON_VERSION}.txt" | |
| python -m pip install "apache-airflow==${AIRFLOW_VERSION}" --constraint "${CONSTRAINT_URL}" | |
| python -m pip install -U orloge pulp | |
| python -m pip install -U "git+https://github.com/baobabsoluciones/cornflow@${CLIENT_BRANCH}#subdirectory=libs/client" | |
| python -m pip install -U -r airflow_config/requirements.txt | |
| airflow db init | |
| airflow users create -u admin -f admin -l admin -r Admin -p admin -e [email protected] | |
| airflow webserver -p 8080 & | |
| airflow scheduler & | |
| deactivate | |
| sleep 5s | |
| env: | |
| AIRFLOW__SCHEDULER__CATCHUP_BY_DEFAULT: 0 | |
| AIRFLOW_HOME: "${GITHUB_WORKSPACE}/cornflow-server/airflow_config" | |
| AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 0 | |
| AIRFLOW__API__AUTH_BACKEND: airflow.api.auth.backend.basic_auth | |
| AIRFLOW__WEBSERVER__SECRET_KEY: e9adafa751fd35adfc1fdd3285019be15eea0758f76e38e1e37a1154fb36 | |
| AIRFLOW__CORE__LOAD_EXAMPLES: 0 | |
| AIRFLOW_CONN_CF_URI: http://airflow:Airflow_test_password1@localhost:5050 | |
| - name: Run cornflow | |
| run: | | |
| cd .. | |
| cd .. | |
| cd cornflow-server | |
| python -m venv cfvenv | |
| source cfvenv/bin/activate | |
| python -m pip install -U -r requirements.txt | |
| CLIENT_BRANCH="${{ github.head_ref || github.ref_name }}" | |
| python -m pip install -U "git+https://github.com/baobabsoluciones/cornflow@${CLIENT_BRANCH}#subdirectory=libs/client" | |
| flask db upgrade -d cornflow/migrations/ | |
| flask access_init | |
| flask register_deployed_dags -r http://127.0.0.1:8080 -u admin -p admin | |
| flask create_admin_user -u admin -e [email protected] -p Adminpassword1! | |
| flask create_service_user -u airflow -e [email protected] -p Airflow_test_password1 | |
| flask create_base_user -u user -e [email protected] -p UserPassword1! | |
| flask register_dag_permissions -o 1 | |
| flask run -p 5050 & | |
| deactivate | |
| sleep 5s | |
| env: | |
| FLASK_ENV: testing | |
| FLASK_APP: cornflow.app | |
| AIRFLOW_URL: http://127.0.0.1:8080 | |
| AIRFLOW_USER: admin | |
| AIRFLOW_PWD: admin | |
| SECRET_KEY: THISNEEDSTOBECHANGED | |
| OPEN_DEPLOYMENT: 1 | |
| AUTH_TYPE: 1 | |
| LOG_LEVEL: 10 | |
| CORNFLOW_SERVICE_USER: airflow | |
| - name: Run unit tests | |
| run: | | |
| coverage run --source=./cornflow_client/ --rcfile=./.coveragerc -m unittest discover -s cornflow_client/tests/unit | |
| coverage report -m | |
| - name: Run integration tests | |
| run: | | |
| coverage run -a --source=./cornflow_client/ --rcfile=./.coveragerc -m unittest discover -s cornflow_client/tests/integration | |
| coverage report -m | |
| coverage xml | |
| - name: Upload coverage to codecov | |
| uses: codecov/codecov-action@v3 | |
| with: | |
| flags: client-tests | |
| token: ${{secrets.CODECOV_TOKEN}} |