Merge pull request #421 from baobabsoluciones/development

Feature/package (#371)

.github/workflows/cornflow-publish-to-pypi.yml

@@ -0,0 +1,60 @@
+name: Publish cornflow Python 🐍 distributions 📦 to PyPI
+
+on:
+  push:
+    tags:
+      - "cornflow*"
+
+jobs:
+  build-n-publish:
+    name: Build and publish cornflow Python 🐍 distributions 📦 to PyPI
+    defaults:
+      run:
+        working-directory: ./cornflow-server
+    runs-on: ubuntu-18.04
+    steps:
+    - uses: actions/checkout@master
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.8
+    - name: Install wheel
+      run: >-
+        python -m
+        pip install
+        wheel
+        --user
+    - name: Build a binary wheel and a source tarball
+      run: python setup.py sdist bdist_wheel
+    - name: Publish distribution 📦 to Test PyPI
+      uses: pypa/gh-action-pypi-publish@master
+      with:
+        password: ${{ secrets.CORNFLOW_TEST_PYPI_TOKEN }}
+        repository_url: https://test.pypi.org/legacy/
+        packages_dir: cornflow-server/dist/
+    - name: Publish distribution 📦 to PyPI
+      uses: pypa/gh-action-pypi-publish@master
+      with:
+        password: ${{ secrets.CORNFLOW_PYPI_TOKEN }}
+        packages_dir: cornflow-server/dist/
+    - name: Get version number
+      uses: jungwinter/split@v2
+      id: split
+      with:
+        msg : ${{ github.ref_name}}
+        separator: "w"
+    - name: Notify slack channel
+      uses: slackapi/[email protected]
+      with:
+        slack-message: "A new version of cornflow (v${{ steps.split.outputs._1 }}) has been deployed"
+        channel-id: ${{ secrets.SLACK_CHANNEL }}
+      env:
+        SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_USER_OAUTH_ACCESS_TOKEN }}
+    - name: Tag published version for docker hub trigger
+      run: |
+        git tag -a v${{ steps.split.outputs_1}} -m "Published cornflow version v${{ steps.split.outputs._1 }}"
+        git push origin v${{ steps.split.outputs_1}}
+    - name: Delete tag to launch release
+      run: git push --delete origin ${{ github.ref_name }}
+
+

cornflow-server/Dockerfile

@@ -8,10 +8,14 @@ LABEL maintainer="sistemas@baobabsoluciones"
 ENV DEBIAN_FRONTEND noninteractive
 ENV TERM linux
 
+# CORNFLOW vars
+ARG CORNFLOW_VERSION=1.0.0
+
 # install linux pkg
 RUN apt update -y && apt-get install -y --no-install-recommends \
 		gcc \
 		git \
+        ssh \
 		python3-dev \
 		libffi-dev \
 		libpq-dev \
@@ -26,19 +30,16 @@ ENV PYTHONUNBUFFERED 1
 
 # install dependencies
 RUN pip install --upgrade pip
-COPY ./requirements.txt /usr/src/app/requirements.txt
-RUN pip install -r requirements.txt --force-reinstall
-
-# copy project
-COPY cornflow /usr/src/app/cornflow
-COPY docs /usr/src/app/docs
-COPY migrations /usr/src/app/migrations
-COPY examples /usr/src/app/examples
-COPY *.py /usr/src/app/
+RUN pip install "cornflow==${CORNFLOW_VERSION}"
+
+
+# create folder for logs
 RUN mkdir -p /usr/src/app/log
 
+# create folder for custom ssh keys
+RUN mkdir /usr/src/app/.ssh
+
 EXPOSE 5000
 
 # execute python init script
-ENTRYPOINT [ "python" ]
-CMD ["/usr/src/app/init_cornflow_service.py"]
+CMD ["cornflow", "service", "init"]

cornflow-server/MANIFEST.in

@@ -0,0 +1,6 @@
+include LICENSE
+include MANIFEST.in
+include README.rst
+include setup.py
+include cornflow/migrations/*
+include cornflow/migrations/versions/*

cornflow-server/README.rst

@@ -18,7 +18,7 @@ Cornflow
 
 .. image:: https://img.shields.io/badge/License-Apache2.0-blue
 
-Cornflow is open source multi-solver optimization server with a REST API built using `flask <https://flask.palletsprojects.com>`_, `airflow <https://airflow.apache.org/>`_ and `pulp <https://coin-or.github.io/pulp/>`_.
+Cornflow is an open source multi-solver optimization server with a REST API built using `flask <https://flask.palletsprojects.com>`_, `airflow <https://airflow.apache.org/>`_ and `pulp <https://coin-or.github.io/pulp/>`_.
 
 While most deployment servers are based on the solving technique (MIP, CP, NLP, etc.), Cornflow focuses on the optimization problems themselves. However, it does not impose any constraint on the type of problem and solution method to use.
 

cornflow-server/cornflow/app.py

@@ -2,8 +2,8 @@
 Main file with the creation of the app logic
 """
 # Full imports
-import click
 import os
+import click
 
 # Partial imports
 from flask import Flask
@@ -15,31 +15,28 @@
 from logging.config import dictConfig
 
 # Module imports
-from cornflow.commands.access import access_init_command
-from cornflow.commands.actions import register_actions_command
-from cornflow.commands.dag import register_deployed_dags_command
-from cornflow.commands.permissions import (
-    register_base_permissions_command,
-    register_dag_permissions_command,
-)
-from cornflow.commands.roles import register_roles_command
-from cornflow.commands.users import (
+from cornflow.commands import (
+    create_service_user_command,
     create_admin_user_command,
     create_planner_user_command,
-    create_service_user_command,
+    register_roles_command,
+    register_actions_command,
+    register_views_command,
+    register_base_permissions_command,
+    access_init_command,
+    register_deployed_dags_command,
+    register_dag_permissions_command,
 )
-from cornflow.commands.views import register_views_command
 from cornflow.config import app_config
 from cornflow.endpoints import resources
 from cornflow.endpoints.login import LoginEndpoint, LoginOpenAuthEndpoint
 from cornflow.endpoints.signup import SignUpEndpoint
+from cornflow.shared.const import AUTH_DB, AUTH_LDAP, AUTH_OID
+from cornflow.shared.log_config import log_config
 from cornflow_core.compress import init_compress
 from cornflow_core.exceptions import initialize_errorhandlers
 from cornflow_core.shared import db, bcrypt
 
-from cornflow.shared.const import AUTH_DB, AUTH_LDAP, AUTH_OID
-from cornflow.shared.log_config import log_config
-
 
 def create_app(env_name="development", dataconn=None):
     """
@@ -159,21 +156,21 @@ def register_actions(verbose):
 @click.option("-v", "--verbose", is_flag=True, default=False)
 @with_appcontext
 def register_views(verbose):
-    register_views_command(verbose)
+    register_views_command(verbose=verbose)
 
 
 @click.command("register_base_assignations")
 @click.option("-v", "--verbose", is_flag=True, default=False)
 @with_appcontext
 def register_base_assignations(verbose):
-    register_base_permissions_command(verbose)
+    register_base_permissions_command(verbose=verbose)
 
 
 @click.command("access_init")
 @click.option("-v", "--verbose", is_flag=True, default=False)
 @with_appcontext
 def access_init(verbose):
-    access_init_command(verbose)
+    access_init_command(verbose=verbose)
 
 
 @click.command("register_deployed_dags")
@@ -196,6 +193,5 @@ def register_dag_permissions(open_deployment, verbose):
 
 if __name__ == "__main__":
     environment_name = os.getenv("FLASK_ENV", "development")
-    # env_name = 'development'
     flask_app = create_app(environment_name)
     flask_app.run()

cornflow-server/cornflow/cli/__init__.py

@@ -0,0 +1,28 @@
+"""
+init file to have this as a module
+"""
+
+import click
+from cornflow.cli.actions import actions
+from cornflow.cli.config import config
+from cornflow.cli.migrations import migrations
+from cornflow.cli.permissions import permissions
+from cornflow.cli.roles import roles
+from cornflow.cli.service import service
+from cornflow.cli.users import users
+from cornflow.cli.views import views
+
+
+@click.group(name="cornflow", help="Commands in the cornflow cli")
+def cli():
+    pass
+
+
+cli.add_command(actions)
+cli.add_command(config)
+cli.add_command(migrations)
+cli.add_command(permissions)
+cli.add_command(roles)
+cli.add_command(service)
+cli.add_command(users)
+cli.add_command(views)

cornflow-server/cornflow/cli/actions.py

@@ -0,0 +1,19 @@
+import os
+
+import click
+from cornflow.cli.utils import get_app
+from cornflow.commands import register_actions_command
+from .arguments import verbose
+
+
+@click.group(name="actions", help="Commands to manage the actions")
+def actions():
+    pass
+
+
+@actions.command(name="init", help="Initialize the actions")
+@verbose
+def init(verbose):
+    app = get_app()
+    with app.app_context():
+        register_actions_command(verbose)

cornflow-server/cornflow/cli/arguments.py

@@ -0,0 +1,24 @@
+import click
+
+verbose = click.option("-v", "--verbose", is_flag=True, default=False)
+app_name = click.option(
+    "-a",
+    "--app_name",
+    required=False,
+    type=str,
+    default="external_app",
+    help="The name of the external app",
+)
+
+username = click.option(
+    "-u", "--username", required=True, type=str, help="The username of the user"
+)
+password = click.option(
+    "-p", "--password", required=True, type=str, help="The password of the user"
+)
+email = click.option(
+    "-e", "--email", required=True, type=str, help="The email of the user"
+)
+path = click.option(
+    "-p", "--path", type=str, help="The path to the file to save the file"
+)

cornflow-server/cornflow/cli/config.py

@@ -0,0 +1,42 @@
+import click
+
+from cornflow.cli.utils import get_app
+from flask import current_app
+from .arguments import path
+
+
+@click.group(name="config", help="Commands to manage the configuration variables")
+def config():
+    pass
+
+
+@config.command(name="list", help="List the configuration variables")
+def config_list():
+    app = get_app()
+    with app.app_context():
+        for key, value in current_app.config.items():
+            click.echo(f"{key} = {value}")
+
+    return 1
+
+
+@config.command(name="save", help="Save the configuration variables to a file")
+@path
+def config_save(path):
+    app = get_app()
+    path = f"{path}config.cfg"
+    with app.app_context():
+        with open(path, "w") as f:
+            f.write("[configuration]\n\n")
+            for key, value in current_app.config.items():
+                f.write(f"{key} = {value}\n")
+
+    return 1
+
+
+@config.command(name="get", help="Get the value of a configuration variable")
+@click.option("--key", "-k", type=str, help="The key of the configuration variable")
+def config_get(key):
+    app = get_app()
+    with app.app_context():
+        click.echo(f"{current_app.config.get(key, None)}")

cornflow-server/cornflow/cli/migrations.py

@@ -0,0 +1,58 @@
+import os.path
+
+
+import click
+from cornflow_core.shared import db
+from flask_migrate import Migrate, migrate, upgrade, init
+
+from .utils import get_app
+
+
+@click.group(name="migrations", help="Commands to manage the migrations")
+def migrations():
+    pass
+
+
+@migrations.command(name="migrate", help="Calculate the migrations")
+def migrate_migrations():
+    app = get_app()
+    external = int(os.getenv("EXTERNAL_APP", 0))
+    if external == 0:
+        path = "./cornflow/migrations"
+    else:
+        path = f"./{os.getenv('EXTERNAL_APP_MODULE', 'external_app')}/migrations"
+
+    with app.app_context():
+        migration_client = Migrate(app=app, db=db, directory=path)
+        migrate()
+
+
+@migrations.command(name="upgrade", help="Apply migrations")
+def upgrade_migrations():
+    app = get_app()
+    external = int(os.getenv("EXTERNAL_APP", 0))
+    if external == 0:
+        path = "./cornflow/migrations"
+    else:
+        path = f"./{os.getenv('EXTERNAL_APP_MODULE', 'external_app')}/migrations"
+
+    with app.app_context():
+        migration_client = Migrate(app=app, db=db, directory=path)
+        upgrade()
+
+
+@migrations.command(
+    name="init",
+    help="Initialize the migrations for an external app. Creates the folder and copies the migrations from cornflow",
+)
+def init_migrations():
+    app = get_app()
+    external = int(os.getenv("EXTERNAL_APP", 0))
+    if external == 0:
+        path = "./cornflow/migrations"
+    else:
+        path = f"./{os.getenv('EXTERNAL_APP_MODULE', 'external_app')}/migrations"
+
+    with app.app_context():
+        migration_client = Migrate(app=app, db=db, directory=path)
+        init()