Merge pull request #431 from baobabsoluciones/development

Development

.github/workflows/cornflow-publish-to-pypi.yml

@@ -41,7 +41,7 @@ jobs:
       uses: jungwinter/split@v2
       id: split
       with:
-        msg : ${{ github.ref_name}}
+        msg : ${{ github.ref_name }}
         separator: "w"
     - name: Notify slack channel
       uses: slackapi/[email protected]
@@ -52,8 +52,9 @@ jobs:
         SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_USER_OAUTH_ACCESS_TOKEN }}
     - name: Tag published version for docker hub trigger
       run: |
+        sleep 30s
         git config --local user.email "[email protected]"
         git config --local user.name "cornflow"
-        git tag -a v${{ steps.split.outputs_1}} -m "Published cornflow version v${{ steps.split.outputs._1 }}"
-        git push origin v${{ steps.split.outputs_1}}
+        git tag -a v${{ steps.split.outputs._1 }} -m "Published cornflow version v${{ steps.split.outputs._1 }}"
+        git push origin v${{ steps.split.outputs._1 }}
         git push --delete origin ${{ github.ref_name }}

.github/workflows/test_cornflow_client.yml

@@ -137,6 +137,7 @@ jobs:
           OPEN_DEPLOYMENT: 1
           AUTH_TYPE: 1
           LOG_LEVEL: 10
+          CORNFLOW_SERVICE_USER: airflow
       - name: Run unit tests
         run: |
           coverage run --source=./cornflow_client/ -m unittest discover -s cornflow_client/tests/unit

cornflow-dags/DAG/rostering/__init__.py

@@ -33,6 +33,8 @@ class Rostering(ApplicationCore):
         "schedule_interval": None,
     }
 
+    extra_args = {"max_active_runs": 2}
+
     @property
     def test_cases(self) -> List[Union[Dict, Tuple[Dict, Dict]]]:
         # Base case with a split opening time on one day

cornflow-server/Dockerfile

@@ -9,7 +9,7 @@ ENV DEBIAN_FRONTEND noninteractive
 ENV TERM linux
 
 # CORNFLOW vars
-ARG CORNFLOW_VERSION=1.0.1
+ARG CORNFLOW_VERSION=1.0.2
 
 # install linux pkg
 RUN apt update -y && apt-get install -y --no-install-recommends \

cornflow-server/changelog.rst

@@ -1,8 +1,21 @@
+version 1.0.2
+---------------
+
+- released: 2023-03-17
+- description: fixes error on startup on google cloud because the monkey patch from gevent was not getting applied properly on urllib3 ssl dependency.
+- changelog:
+    - applied monkey patch from gevent before app startup.
+    - change on service command to not start up the gunicorn process inside the app context.
+    - change on health endpoint so by default is unhealthy.
+    - adjusted health endpoint unit and integration tests.
+    - fixed version of cornflow-client to 1.0.11
+
+
 version 1.0.1
 ---------------
 
 - released: 2023-03-16
-- comments: fixed requirements versions in order to better handle the dockerfile construction on dockerhub.
+- description: fixed requirements versions in order to better handle the dockerfile construction on dockerhub.
 - changelog:
     - fixed version of cornflow-core to 0.1.9
     - fixed version of cornflow-client to 1.0.10
@@ -11,4 +24,4 @@ version 1.0.0
 --------------
 
 - released: 2023-03-15
-- comments: initial release of cornflow package
+- description: initial release of cornflow package

cornflow-server/cornflow/app.py

@@ -1,6 +1,10 @@
 """
 Main file with the creation of the app logic
 """
+from gevent import monkey
+
+monkey.patch_all()
+
 # Full imports
 import os
 import click

cornflow-server/cornflow/cli/service.py

@@ -154,10 +154,11 @@ def init_cornflow_service():
             register_dag_permissions_command(open_deployment, verbose=True)
             update_schemas_command(airflow_url, airflow_user, airflow_pwd, verbose=True)
 
-            # execute gunicorn application
-            os.system(
-                "/usr/local/bin/gunicorn -c python:cornflow.gunicorn \"cornflow:create_app('$FLASK_ENV')\""
-            )
+        # execute gunicorn application
+        os.system(
+            "/usr/local/bin/gunicorn -c python:cornflow.gunicorn \"cornflow:create_app('$FLASK_ENV')\""
+        )
+
     elif external_application == 1:
         click.echo(f"Starting cornflow + {os.getenv('EXTERNAL_APP_MODULE')}")
         os.chdir("/usr/src/app")

cornflow-server/cornflow/endpoints/health.py

@@ -1,6 +1,8 @@
 """
-
+Endpoint to check the health of the services.
+It performs a health check to airflow and a health check to cornflow database
 """
+import os
 
 # Import from libraries
 from cornflow_client.airflow.api import Airflow
@@ -10,21 +12,34 @@
 # Import from internal modules
 from cornflow.schemas.health import HealthResponse
 from cornflow.shared.const import STATUS_HEALTHY, STATUS_UNHEALTHY
-from cornflow_core.shared import db
 from cornflow_core.resources import BaseMetaResource
+from cornflow.models import UserModel
 
 
 class HealthEndpoint(BaseMetaResource):
     @doc(description="Health check", tags=["Health"])
     @marshal_with(HealthResponse)
     def get(self):
+        """
+        The get function is a simple health check endpoint that returns the status of both cornflow and airflow.
+
+        :return: A dictionary with the keys 'cornflow_status' and 'airflow_status' and the state of each service
+        :rtype: dict
+        :doc-author: baobab soluciones
+        """
         af_client = Airflow.from_config(current_app.config)
-        airflow_status = STATUS_HEALTHY
-        cornflow_status = STATUS_HEALTHY
-        if not af_client.is_alive():
-            airflow_status = STATUS_UNHEALTHY
-        try:
-            db.engine.execute("SELECT 1")
-        except Exception:
-            cornflow_status = STATUS_UNHEALTHY
+        airflow_status = STATUS_UNHEALTHY
+        cornflow_status = STATUS_UNHEALTHY
+        if af_client.is_alive():
+            airflow_status = STATUS_HEALTHY
+
+        if (
+            UserModel.get_one_user_by_username(os.getenv("CORNFLOW_SERVICE_USER"))
+            is not None
+        ):
+            cornflow_status = STATUS_HEALTHY
+
+        current_app.logger.info(
+            f"Health check: cornflow {cornflow_status}, airflow {airflow_status}"
+        )
         return {"cornflow_status": cornflow_status, "airflow_status": airflow_status}

cornflow-server/cornflow/tests/custom_liveServer.py

@@ -1,18 +1,20 @@
 # Full imports
+import os
+
 import cornflow_client as cf
+from cornflow_core.shared import db
 
 # External libraries
 from flask import current_app
 from flask_testing import LiveServerTestCase
 
 # Internal modules
 from cornflow.app import create_app
-from cornflow.models import UserRoleModel
 from cornflow.commands.access import access_init_command
 from cornflow.commands.dag import register_deployed_dags_command_test
 from cornflow.commands.permissions import register_dag_permissions_command
+from cornflow.models import UserRoleModel
 from cornflow.shared.const import ADMIN_ROLE, SERVICE_ROLE
-from cornflow_core.shared import db
 from cornflow.tests.const import PREFIX
 
 
@@ -52,6 +54,13 @@ def setUp(self, create_all=True):
         register_dag_permissions_command(
             open_deployment=current_app.config["OPEN_DEPLOYMENT"], verbose=False
         )
+        os.environ["CORNFLOW_SERVICE_USER"] = "service_user"
+
+        self.create_service_user(
+            dict(
+                username="service_user", pwd="Airflow_test_password1", email="[email protected]"
+            )
+        )
 
     def tearDown(self):
         db.session.remove()

cornflow-server/cornflow/tests/unit/test_health.py

@@ -1,24 +1,30 @@
-from flask_testing import TestCase
+import os
 
-from cornflow.app import create_app
 from cornflow_core.shared import db
-from cornflow.tests.const import HEALTH_URL
+
+from cornflow.app import create_app
+from cornflow.commands import access_init_command
 from cornflow.shared.const import STATUS_HEALTHY
+from cornflow.tests.const import HEALTH_URL
+from cornflow.tests.custom_test_case import CustomTestCase
 
 
-class TestHealth(TestCase):
+class TestHealth(CustomTestCase):
     def create_app(self):
         app = create_app("testing")
         return app
 
     def setUp(self):
         db.create_all()
+        access_init_command(verbose=False)
 
     def tearDown(self):
         db.session.remove()
         db.drop_all()
 
     def test_health(self):
+        self.create_service_user()
+        os.environ["CORNFLOW_SERVICE_USER"] = "testuser4"
         response = self.client.get(HEALTH_URL)
         self.assertEqual(200, response.status_code)
         cf_status = response.json["cornflow_status"]

cornflow-server/requirements.txt

@@ -2,7 +2,7 @@ alembic==1.9.2
 apispec<=6.2.0
 click<=8.1.3
 cornflow-core<=0.1.9
-cornflow-client<=1.0.10
+cornflow-client<=1.0.11
 cryptography<=39.0.2
 Flask==2.1.3
 flask-apispec<=0.11.4

cornflow-server/setup.py

@@ -9,7 +9,7 @@
 
 setuptools.setup(
     name="cornflow",
-    version="1.0.1",
+    version="1.0.2",
     author="baobab soluciones",
     author_email="[email protected]",
     description="Cornflow is an open source multi-solver optimization server with a REST API built using flask.",

libs/client/changelog.rst

@@ -0,0 +1,7 @@
+version 1.0.11
+----------------
+
+- released: 2023-03-17
+- description: change the way airflow api behaves doing the is_alive check.
+- changelog:
+    - change the way airflow api behaves doing the is_alive check.

libs/client/cornflow_client/airflow/api.py

@@ -35,12 +35,14 @@ def is_alive(self):
             return False
         try:
             data = response.json()
+            database = data["metadatabase"]["status"] == "healthy"
+            scheduler = data["scheduler"]["status"] == "healthy"
         except json.JSONDecodeError:
             return False
-        return (
-            data["metadatabase"]["status"] == "healthy"
-            and data["scheduler"]["status"] == "healthy"
-        )
+        except KeyError:
+            return False
+
+        return database and scheduler
 
     def request_headers_auth(self, status=200, **kwargs):
         def_headers = {"Content-type": "application/json", "Accept": "application/json"}

libs/client/setup.py

@@ -12,7 +12,7 @@
 
 setuptools.setup(
     name="cornflow-client",
-    version="1.0.10",
+    version="1.0.11",
     author="baobab soluciones",
     author_email="[email protected]",
     description="Client to connect to a cornflow server",
@@ -24,7 +24,7 @@
         "Programming Language :: Python :: 3",
         "License :: OSI Approved :: Apache Software License",
         "Operating System :: OS Independent",
-        "Development Status :: 3 - Alpha",
+        "Development Status :: 5 - Production/Stable",
     ],
     python_requires=">=3.7",
     include_package_data=True,