diff --git a/.github/workflows/cornflow-core-publish-to-pypi.yml b/.github/workflows/cornflow-core-publish-to-pypi.yml deleted file mode 100644 index 598c4bc1c..000000000 --- a/.github/workflows/cornflow-core-publish-to-pypi.yml +++ /dev/null @@ -1,53 +0,0 @@ -name: Publish cornflow-core Python ๐Ÿ distributions ๐Ÿ“ฆ to PyPI - -on: - push: - tags: - - "core*" - -jobs: - build-n-publish: - name: Build and publish cornflow-core Python ๐Ÿ distributions ๐Ÿ“ฆ to PyPI - defaults: - run: - working-directory: ./libs/core - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@master - - name: Set up Python 3.7 - uses: actions/setup-python@v1 - with: - python-version: 3.7 - - name: Install wheel - run: >- - python -m - pip install - wheel - --user - - name: Build a binary wheel and a source tarball - run: python setup.py sdist bdist_wheel - - name: Publish distribution ๐Ÿ“ฆ to Test PyPI - uses: pypa/gh-action-pypi-publish@master - with: - password: ${{ secrets.CORE_TEST_PYPI_TOKEN }} - repository_url: https://test.pypi.org/legacy/ - packages_dir: libs/core/dist/ - - name: Publish distribution ๐Ÿ“ฆ to PyPI - uses: pypa/gh-action-pypi-publish@master - with: - password: ${{ secrets.CORE_PYPI_TOKEN }} - packages_dir: libs/core/dist/ - - name: Get version number - uses: jungwinter/split@v2 - id: split - with: - msg: ${{ github.ref_name }} - separator: "e" - - name: Notify slack channel - uses: slackapi/slack-github-action@v1.23.0 - with: - slack-message: "A new version of cornflow core (v${{ steps.split.outputs._1 }}) has been deployed" - channel-id: ${{ secrets.SLACK_CHANNEL }} - env: - SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_USER_OAUTH_ACCESS_TOKEN }} - diff --git a/.github/workflows/test_cornflow_client.yml b/.github/workflows/test_cornflow_client.yml index 80b9084d5..22953acae 100644 --- a/.github/workflows/test_cornflow_client.yml +++ b/.github/workflows/test_cornflow_client.yml @@ -116,7 +116,6 @@ jobs: python -m pip install -U -r requirements.txt CLIENT_BRANCH="${{ github.head_ref || github.ref_name }}" python -m pip install -U "git+https://github.com/baobabsoluciones/cornflow@${CLIENT_BRANCH}#subdirectory=libs/client" - python -m pip install -U "git+https://github.com/baobabsoluciones/cornflow@${CLIENT_BRANCH}#subdirectory=libs/core" flask db upgrade -d cornflow/migrations/ flask access_init flask register_deployed_dags -r http://127.0.0.1:8080 -u admin -p admin diff --git a/.github/workflows/test_cornflow_core.yml b/.github/workflows/test_cornflow_core.yml deleted file mode 100644 index 37291e96d..000000000 --- a/.github/workflows/test_cornflow_core.yml +++ /dev/null @@ -1,53 +0,0 @@ -name: cornflow-core tests - -on: - pull_request: - types: [ opened, edited, synchronize, reopened ] - paths: - - 'libs/core/**' - - '!libs/core/README.rst' - - '!libs/core/LICENSE' - - '!libs/core/setup.py' - push: - branches: - - master - - development - paths: - - 'libs/core/**' - - '!libs/core/README.rst' - - '!libs/core/LICENSE' - - '!libs/core/setup.py' - -jobs: - testing: - - name: Run all test suites on cornflow-core - runs-on: ${{ matrix.os }} - defaults: - run: - working-directory: ./libs/core - strategy: - max-parallel: 21 - matrix: - python-version: [3.7, 3.8] - os: [ubuntu-latest] - - steps: - - uses: actions/checkout@v1 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install -U -r requirements-dev.txt - - name: Run unit tests - run: | - coverage run --source=./cornflow_core/ --omit="*/tests/*" -m unittest discover -s cornflow_core/tests - coverage report -m - coverage xml - - name: Upload coverage to codecov - uses: codecov/codecov-action@v3 - with: - flags: core-tests diff --git a/.github/workflows/test_cornflow_server.yml b/.github/workflows/test_cornflow_server.yml index bd030d687..7eabff50c 100644 --- a/.github/workflows/test_cornflow_server.yml +++ b/.github/workflows/test_cornflow_server.yml @@ -65,8 +65,6 @@ jobs: CLIENT_BRANCH="${{ github.head_ref || github.ref_name }}" python -m pip install --upgrade pip python -m pip install -U -r requirements-dev.txt - python -m pip uninstall cornflow-core -y - python -m pip install -U "git+https://github.com/baobabsoluciones/cornflow@${CLIENT_BRANCH}#subdirectory=libs/core" python -m pip uninstall cornflow-client -y python -m pip install -U "git+https://github.com/baobabsoluciones/cornflow@${CLIENT_BRANCH}#subdirectory=libs/client" - name: Install airflow @@ -104,7 +102,7 @@ jobs: AIRFLOW_CONN_CF_URI: http://airflow:Airflow_test_password1@localhost:5050 - name: Run unit tests run: | - coverage run --source=./cornflow/ -m unittest discover -s cornflow/tests/unit + coverage run --source=./cornflow/ --omit="*/tests/data/*" -m unittest discover -s cornflow/tests/unit coverage report -m env: FLASK_ENV: testing diff --git a/README.rst b/README.rst index 1d554f055..74f25b67f 100644 --- a/README.rst +++ b/README.rst @@ -5,14 +5,14 @@ Cornflow :alt: GitHub Workflow Status :target: https://github.com/baobabsoluciones/cornflow/actions +.. image:: https://img.shields.io/pypi/v/cornflow?label=cornflow&style=for-the-badge + :alt: PyPI + :target: https://pypi.python.org/pypi/cornflow + .. image:: https://img.shields.io/pypi/v/cornflow-client?label=cornflow-client&style=for-the-badge :alt: PyPI :target: https://pypi.python.org/pypi/cornflow-client -.. image:: https://img.shields.io/pypi/v/cornflow-core?label=cornflow-core&style=for-the-badge - :alt: PyPI - :target: https://pypi.python.org/pypi/cornflow-core - .. image:: https://img.shields.io/pypi/l/cornflow-client?color=blue&style=for-the-badge :alt: PyPI - License :target: https://github.com/baobabsoluciones/cornflow/blob/master/LICENSE @@ -25,10 +25,6 @@ Cornflow :alt: Codecov :target: https://app.codecov.io/gh/baobabsoluciones/cornflow -.. image:: https://img.shields.io/codecov/c/gh/baobabsoluciones/cornflow?flag=core-tests&label=Core&logo=codecov&logoColor=white&style=for-the-badge&token=H14UGPUQVL - :alt: Codecov - :target: https://app.codecov.io/gh/baobabsoluciones/cornflow - .. image:: https://img.shields.io/codecov/c/gh/baobabsoluciones/cornflow?flag=dags-tests&label=dags&logo=codecov&logoColor=white&style=for-the-badge&token=H14UGPUQVL :alt: Codecov :target: https://app.codecov.io/gh/baobabsoluciones/cornflow diff --git a/codecov.yml b/codecov.yml index b80925bd5..17e51c95c 100644 --- a/codecov.yml +++ b/codecov.yml @@ -15,10 +15,7 @@ flag_management: individual_flags: - name: server-tests paths: - - cornflow-server/ - - name: core-tests - paths: - - libs/core/ + - cornflow-server/ - name: client-tests paths: - libs/client/ diff --git a/cornflow-server/Dockerfile b/cornflow-server/Dockerfile index 48fcfe3e5..37c8df8ce 100644 --- a/cornflow-server/Dockerfile +++ b/cornflow-server/Dockerfile @@ -9,7 +9,7 @@ ENV DEBIAN_FRONTEND noninteractive ENV TERM linux # CORNFLOW vars -ARG CORNFLOW_VERSION=1.0.4 +ARG CORNFLOW_VERSION=1.0.5 # install linux pkg RUN apt update -y && apt-get install -y --no-install-recommends \ diff --git a/cornflow-server/changelog.rst b/cornflow-server/changelog.rst index 6d50fd83d..406557eef 100644 --- a/cornflow-server/changelog.rst +++ b/cornflow-server/changelog.rst @@ -1,3 +1,15 @@ +version 1.0.5 +-------------- + +- released: 2023-05-04 +- description: first version of cornflow without cornflow core +- changelog: + - removed cornflow core from dependencies. + - moved all cornflow core code to cornflow. + - added new error handling for InternalServerErrors. + - updated version of flask to 2.3.2 due to security reasons. + - updated version of other libraries due to upgrade on flask version. + version 1.0.4 --------------- diff --git a/cornflow-server/cornflow/app.py b/cornflow-server/cornflow/app.py index 56f8e5513..613d9e2a2 100644 --- a/cornflow-server/cornflow/app.py +++ b/cornflow-server/cornflow/app.py @@ -31,11 +31,11 @@ from cornflow.endpoints import resources, alarms_resources from cornflow.endpoints.login import LoginEndpoint, LoginOpenAuthEndpoint from cornflow.endpoints.signup import SignUpEndpoint +from cornflow.shared import db, bcrypt +from cornflow.shared.compress import init_compress from cornflow.shared.const import AUTH_DB, AUTH_LDAP, AUTH_OID +from cornflow.shared.exceptions import initialize_errorhandlers from cornflow.shared.log_config import log_config -from cornflow_core.compress import init_compress -from cornflow_core.exceptions import initialize_errorhandlers -from cornflow_core.shared import db, bcrypt def create_app(env_name="development", dataconn=None): diff --git a/libs/core/README.rst b/cornflow-server/cornflow/cli/README.rst similarity index 56% rename from libs/core/README.rst rename to cornflow-server/cornflow/cli/README.rst index b02011518..a159d3c34 100644 --- a/libs/core/README.rst +++ b/cornflow-server/cornflow/cli/README.rst @@ -2,7 +2,7 @@ Cornflow-tools ============== -Cornflow-core is a library that contains modules to help you create REST APIs in an easier and faster way. +Cornflow contains commands to help you create REST APIs in an easier and faster way. It includes a set of modules that can be used to start the creation of your flask REST API and some command line interface commands that let you create a full REST API from a JSONSchema file that represent your data or create a JSONSchema file representing your REST API. @@ -10,19 +10,22 @@ create a JSONSchema file representing your REST API. ---------------------------------------------------- Command line interface :code:`generate_from_schema` ---------------------------------------------------- -The cli :code:`generate_from_schema` allows you to automatically generate models, endpoints and schemas +The cli :code:`cornflow schemas generate_from_schema` allows you to automatically generate models, endpoints and schemas from a JSONSchema. The generated files can then be added to your flask (RestFul) REST API. How to use? =========== To start, you need to have a json file containing the schema of the tables you want to create. +It is possible to generate this json schema from an Excel file with the function +:code:`schema_from_excel` from :code:`cornflow-client`. + Let's assume that this file is stored on your computer as :code:`C:/Users/User/instance.json` Open the terminal, then run: .. code-block:: console - generate_from_schema -p C:/Users/User/instance.json -a application_name + cornflow schemas generate_from_schema -p C:/Users/User/instance.json -a application_name The argument :code:`application_name` will be the prefix of the name used for the generated files, classes and tables. It is an optional argument @@ -43,7 +46,7 @@ Example: .. code-block:: console - generate_from_schema -p C:/Users/User/instance.json -a application_name --output-path C:/Users/User/output_files + cornflow schemas generate_from_schema -p C:/Users/User/instance.json -a application_name --output-path C:/Users/User/output_files Remove methods @@ -51,14 +54,17 @@ Remove methods By default, two endpoints are created: -- A global endpoint, with three methods: +- A global endpoint, with two methods: - :code:`get()`, that returns all the element of the table. - :code:`post(**kwargs)`, that adds a new row to the table. -- A detail endpoint, with three methods: +- A detail endpoint, with four methods: - :code:`get(idx)`, that returns the entry with the given id. - :code:`put(idx, **kwargs)`, that updates the entry with the given id with the given data. - :code:`patch(idx, **kwargs)` that patches the entry with the given id with the given oatch. - :code:`delete(idx)`, that deletes the entry with the given id. +- A bulk endpoint, with two methods: + - :code:`post(**kwargs)`, that adds several new rows to the table. + - :code:`put(**kwargs)`, that adds or updates several rows in the table. If one or several of those methods are not necessary, the option :code:`--remove-methods` or :code:`-r` allows to not generate some of those methods. @@ -67,11 +73,12 @@ Example: .. code-block:: console - generate_from_schema -p C:/Users/User/instance.json -a application_name --remove-methods get-list -r delete-detail + cornflow schemas generate_from_schema -p C:/Users/User/instance.json -a application_name --remove-methods get-list -r delete-detail In that example, for each table, the detail endpoint will not contain the :code:`delete()` method and the list endpoint will not contain the :code:`get()` method. The choices for this method are -:code:`get-list`, :code:`post-list`, :code:`get-detail`, :code:`put-detail`, :code:`delete-detail` and :code:`patch-detail`. +:code:`get-list`, :code:`post-list`, :code:`get-detail`, :code:`put-detail`, :code:`delete-detail`, +:code:`patch-detail`, :code:`post-bulk` and :code:`put-bulk`. One table --------- @@ -112,10 +119,55 @@ Example: .. code-block:: console - generate_from_schema -p C:/Users/User/instance.json -a application_name --one table_name + cornflow schemas generate_from_schema -p C:/Users/User/instance.json -a application_name --one table_name In that case, only one table will be created. +Endpoints methods +----------------- +Use the :code:`-m` or :code:`--endpoints_methods` to pass an optional json file to the function. +In this file, you may list the methods you want to create for every table. + +Example: + +.. code-block:: console + + cornflow schemas generate_from_schema -p C:/Users/User/instance.json -m C:/Users/User/endpoints_methods.json + +The format of the json file must be the following: + +.. code-block:: json + + { + "table1":["get_list", "post_list"], + "table2":["post_list", "get_detail", "put_detail"] + } + +Roles whith access to the endpoints +----------------- +Use the :code:`-e` or :code:`--endpoints_access` to pass an optional json file to the function. +In this file, you may list the roles which should be able to access each table endpoint. +The available roles are: + +.. code-block:: python + + ["VIEWER_ROLE", "PLANNER_ROLE", "ADMIN_ROLE", "SERVICE_ROLE"] + +Example: + +.. code-block:: console + + cornflow schemas generate_from_schema -p C:/Users/User/instance.json -e C:/Users/User/endpoints_access.json + +The format of the json file must be the following: + +.. code-block:: json + + { + "table1":["VIEWER_ROLE", "SERVICE_ROLE"], + "table2":["VIEWER_ROLE", "PLANNER_ROLE", "ADMIN_ROLE", "SERVICE_ROLE"] + } + Notes ===== Primary keys @@ -149,10 +201,29 @@ as follows: If the property :code:`foreign_key` is left empty, it is assumed that the key is not a foreign key. +Date formats +------------- +By default, json doesn't accept datetimes formats, they must be passed as strings. +However, it is possible to pass this information through the format property and have flask models created +with date columns. +You only need to add the property :code:`format` in the information about the property. +The formats which are currently taken into account are "date", "time" and "datetime". + +.. code-block:: + + { + ..., + "current_date": { + "type": "string", + "format": "datetime", + }, + ... + } + ----------------------------------- Module :code:`schema_from_models` ----------------------------------- -The cli :code:`schema_from_models` allows you to automatically generate a JSONSchema based on +The cli :code:`cornflow schemas schema_from_models` allows you to automatically generate a JSONSchema based on a set of models. How to use? @@ -164,7 +235,7 @@ Open the terminal and run: .. code-block:: console - schema_from_models -p C:/Users/User/models + cornflow schemas schema_from_models -p C:/Users/User/models This command will create a new :code:`output_schema.json` directory in the directory from where it was executed, containing the generated schema. @@ -188,5 +259,44 @@ with their extension. Example: .. code-block:: console - schema_from_models -p C:/Users/User/models --ignore-files instance.py -i execution.py + cornflow schemas schema_from_models -p C:/Users/User/models --ignore-files instance.py -i execution.py + +---------------------------------------------------- +Generating endpoints, models and schemas from Excel +---------------------------------------------------- + +cornflow_client library include a function to generate schema from an Excel file containing example data. +Using this function in combination with :code`generate_from_schema` it is possible to generate all the +database structure directly from Excel. +This can be achieved using the following script: + +.. code-block:: + + from click.testing import CliRunner + from cornflow_client.schema.tools import schema_from_excel + from cornflow.cli.schemas import generate_from_schema + + path = "../data/" + excel_path = path + "table_structure.xlsx" + schema_path = path + "data_schema.json" + path_methods = path + "endpoints_methods.json" + path_access = path + "endpoints_access.json" + path_output = "../project_name/" + + # create schema from excel + schema = schema_from_excel( + excel_path, + path_out=schema_path, + fk=True, + format=True, + path_access=path_access, + path_methods=path_methods, + ) + + # create endpoints from schema + runner = CliRunner() + result = runner.invoke( + generate_from_schema, + ["-p", schema_path, "-o", path_output, "-m", path_methods, "-e", path_access], + ) diff --git a/cornflow-server/cornflow/cli/__init__.py b/cornflow-server/cornflow/cli/__init__.py index 8707da9af..75046784b 100644 --- a/cornflow-server/cornflow/cli/__init__.py +++ b/cornflow-server/cornflow/cli/__init__.py @@ -8,6 +8,7 @@ from cornflow.cli.migrations import migrations from cornflow.cli.permissions import permissions from cornflow.cli.roles import roles +from cornflow.cli.schemas import schemas from cornflow.cli.service import service from cornflow.cli.users import users from cornflow.cli.views import views @@ -23,6 +24,7 @@ def cli(): cli.add_command(migrations) cli.add_command(permissions) cli.add_command(roles) +cli.add_command(schemas) cli.add_command(service) cli.add_command(users) cli.add_command(views) diff --git a/cornflow-server/cornflow/cli/migrations.py b/cornflow-server/cornflow/cli/migrations.py index 708deb841..8444b1f7c 100644 --- a/cornflow-server/cornflow/cli/migrations.py +++ b/cornflow-server/cornflow/cli/migrations.py @@ -2,7 +2,7 @@ import click -from cornflow_core.shared import db +from cornflow.shared import db from flask_migrate import Migrate, migrate, upgrade, init from .utils import get_app diff --git a/cornflow-server/cornflow/cli/schemas.py b/cornflow-server/cornflow/cli/schemas.py new file mode 100644 index 000000000..5f523a217 --- /dev/null +++ b/cornflow-server/cornflow/cli/schemas.py @@ -0,0 +1,184 @@ +""" +File that implements the generate from schema cli command +""" +import click +from .tools.api_generator import APIGenerator +from .tools.schema_generator import SchemaGenerator +import json + +METHOD_OPTIONS = [ + "get_list", + "post_list", + "get_detail", + "put_detail", + "patch_detail", + "delete_detail", + "post_bulk", + "put_bulk", +] + + +@click.group(name="schemas", help="Commands to manage the schemas") +def schemas(): + pass + + +@schemas.command( + name="generate_from_schema", + help="Command to generate models, endpoints and schemas from a jsonschema" +) +@click.option( + "--path", "-p", type=str, help="The absolute path to the JSONSchema", required=True +) +@click.option("--app-name", "-a", type=str, help="The name of the application") +@click.option( + "--output-path", + "-o", + type=str, + default="output", + help="The output path", + required=False, +) +@click.option( + "--remove-methods", + "-r", + type=click.Choice(METHOD_OPTIONS, case_sensitive=False), + help="Methods that will NOT be added to the new endpoints", + multiple=True, + required=False, +) +@click.option( + "--one", + type=str, + help="If your schema describes only one table, use this option to indicate the name of the table", + required=False, +) +@click.option( + "--endpoints-methods", + "-m", + type=str, + default=None, + help="json file with dict of methods that will be added to each new endpoints", + required=False, +) +@click.option( + "--endpoints-access", + "-e", + type=str, + default=None, + help="json file with dict of roles access that will be added to each new endpoints", + required=False, +) +def generate_from_schema( + path, app_name, output_path, remove_methods, one, endpoints_methods, endpoints_access +): + """ + This method is executed for the command and creates all the files for the REST API from the provided JSONSchema + + :param str path: the path to the JSONSchema file. + :param str app_name: the name of the application. + :param str output_path: the output path. + :param tuple remove_methods: the methods that will not be added to the new endpoints. + :param str one: if your schema describes only one table, use this option to indicate the name of the table. + :param str endpoints_methods: json file with dict of methods that will be added to each new endpoints. + :param str endpoints_access: json file with dict of roles access that will be added to each new endpoints. + :return: a click status code + :rtype: int + """ + + path = path.replace("\\", "/") + output = None + if output_path != "output": + output = output_path.replace("\\", "/") + + if remove_methods is not None: + methods_to_add = {"default": list(set(METHOD_OPTIONS) - set(remove_methods))} + else: + methods_to_add = {"default": list(set(METHOD_OPTIONS))} + + if endpoints_methods is not None: + endpoints_methods = endpoints_methods.replace("\\", "/") + with open(endpoints_methods, "r") as file: + methods_to_add.update(json.load(file)) + + dict_endpoints_access = {"default": ["SERVICE_ROLE"]} + if endpoints_access is not None: + endpoints_access = endpoints_access.replace("\\", "/") + with open(endpoints_access, "r") as file: + dict_endpoints_access.update(json.load(file)) + + name_table = None + if one: + name_table = one + + click.echo("Generating REST API components from the provided JSONSchema") + click.echo(f"The path to the JSONSchema is {path}") + click.echo(f"The app_name is {app_name}") + click.echo(f"The output_path is {output}") + click.echo(f"The method to add are obtained from {endpoints_methods}") + click.echo(f"The methods to add are {methods_to_add}") + click.echo(f"The roles to add are {endpoints_access}") + click.echo(f"The name_table is {name_table}") + + APIGenerator( + path, + app_name=app_name, + output_path=output_path, + options=methods_to_add, + name_table=name_table, + endpoints_access=dict_endpoints_access + ).main() + + +@schemas.command(name="schema_from_models", help="Command to generate a jsonschema from a set of models") +@click.option( + "--path", + "-p", + type=str, + help="The absolute path to folder containing the models", + required=True, +) +@click.option("--output-path", "-o", type=str, help="The output path", required=False) +@click.option( + "--ignore-files", + "-i", + type=str, + help="Files that will be ignored (with the .py extension). " + "__init__.py files are automatically ignored. Ex: 'instance.py'", + multiple=True, + required=False, +) +@click.option( + "--leave-bases/--no-leave-bases", + "-l/-nl", + default=False, + help="Use this option to leave the bases classes BaseDataModel, " + "EmptyModel and TraceAttributes in the schema. By default, they will be deleted", +) +def schema_from_models(path, output_path, ignore_files, leave_bases): + """ + + :param str path: the path to the folder that contains the models + :param output_path: the output path where the JSONSchema should be placed + :param str ignore_files: files to be ignored. + :param str leave_bases: if the JSONSchema should have abstract classes used as the base for other clases. + :return: a click status code + :rtype: int + """ + path = path.replace("\\", "/") + output = None + if output_path: + output = output_path.replace("\\", "/") + + if ignore_files: + ignore_files = list(ignore_files) + + click.echo("Generating JSONSchema file from the REST API") + click.echo(f"The path to the JSONSchema is {path}") + click.echo(f"The output_path is {output}") + click.echo(f"The ignore_files is {ignore_files}") + click.echo(f"The leave_bases is {leave_bases}") + + SchemaGenerator( + path, output_path=output, ignore_files=ignore_files, leave_bases=leave_bases + ).main() diff --git a/cornflow-server/cornflow/cli/service.py b/cornflow-server/cornflow/cli/service.py index b763b31ac..7c93355ad 100644 --- a/cornflow-server/cornflow/cli/service.py +++ b/cornflow-server/cornflow/cli/service.py @@ -16,7 +16,7 @@ update_schemas_command, ) from cornflow.shared.const import AUTH_DB, ADMIN_ROLE, SERVICE_ROLE -from cornflow_core.shared import db +from cornflow.shared import db from cryptography.fernet import Fernet from flask_migrate import Migrate, upgrade diff --git a/libs/core/cornflow_core/cli/tools/__init__.py b/cornflow-server/cornflow/cli/tools/__init__.py similarity index 100% rename from libs/core/cornflow_core/cli/tools/__init__.py rename to cornflow-server/cornflow/cli/tools/__init__.py diff --git a/cornflow-server/cornflow/cli/tools/api_generator.py b/cornflow-server/cornflow/cli/tools/api_generator.py new file mode 100644 index 000000000..d42f0b7e1 --- /dev/null +++ b/cornflow-server/cornflow/cli/tools/api_generator.py @@ -0,0 +1,443 @@ +""" +This file has the class that creates the new API +""" +import json +import os +import re + +from .endpoint_tools import EndpointGenerator +from .models_tools import ModelGenerator, model_shared_imports +from .schemas_tools import SchemaGenerator, schemas_imports +from .tools import generate_class_def + + +class APIGenerator: + """ + This class is used to create the new API + """ + + def __init__( + self, + schema_path, + app_name, + output_path=None, + options=None, + name_table=None, + endpoints_access=None, + ): + self.path = schema_path + self.name = app_name + if self.name is not None: + self.prefix = self.name + "_" + else: + self.prefix = "" + if options is None: + options = {} + self.options = {**{"default": []}, **options} + if endpoints_access is None: + endpoints_access = {} + self.endpoints_access = {**{"default": ["SERVICE_ROLE"]}, **endpoints_access} + self.schema = self.import_schema() + if self.schema["type"] == "array" and not name_table: + self.schema = {"properties": {"data": self.schema}} + elif self.schema["type"] == "array" and name_table: + self.schema = {"properties": {name_table: self.schema}} + elif self.schema["type"] != "array" and name_table: + print( + "The JSONSchema does not contain only one table. The --one option will be ignored" + ) + + self.output_path = output_path or "output" + self.model_path = os.path.join(self.output_path, "models") + self.endpoint_path = os.path.join(self.output_path, "endpoints") + self.schema_path = os.path.join(self.output_path, "schemas") + self.init_resources = [] + self.init_file = os.path.join(self.endpoint_path, "__init__.py") + + def import_schema(self) -> dict: + """ + This method imports the JSONSchema file + + :return: the read schema + :rtype: dict + """ + with open(self.path, "r") as fd: + schema = json.load(fd) + return schema + + def prepare_dirs(self): + """ + This method creates all the folders needed + + :return: None + :rtype: None + """ + if not os.path.isdir(self.output_path): + os.mkdir(self.output_path) + if not os.path.isdir(self.model_path): + os.mkdir(self.model_path) + + init_path = os.path.join(self.model_path, "__init__.py") + with open(init_path, "w") as file: + file.write(f'"""\nThis file exposes the models\n"""\n') + + if not os.path.isdir(self.endpoint_path): + os.mkdir(self.endpoint_path) + + init_path = os.path.join(self.endpoint_path, "__init__.py") + with open(init_path, "w") as file: + file.write(f'"""\nThis file exposes the endpoints\n"""\n') + + if not os.path.isdir(self.schema_path): + os.mkdir(self.schema_path) + + init_path = os.path.join(self.schema_path, "__init__.py") + with open(init_path, "w") as file: + file.write(f'"""\nThis file exposes the schemas\n"""\n') + + def main(self): + """ + This is the main method that gets executed + + :return: None + :rtype: None + """ + self.prepare_dirs() + tables = self.schema["properties"].keys() + for table in tables: + if self.schema["properties"][table]["type"] != "array": + print( + f'\nThe table "{table}" does not have the correct format. ' + f"The structures will not be generated for this table" + ) + continue + model_name = self.new_model(table) + schemas_names = self.new_schemas(table) + self.new_endpoint(table, model_name, schemas_names) + + self.write_resources() + print( + f"The generated files will be stored in {os.path.join(os.getcwd(), self.output_path)}\n" + ) + return 0 + + def new_model(self, table_name): + """ + This method takes a table name and creates a flask database model with the fields os said table + + :param str table_name: the name of the table to create + :return: the name of the created model + :rtype: str + """ + filename = os.path.join(self.model_path, f"{self.prefix}{table_name}.py") + class_name = self.format_name(table_name, "_model") + + parents_class = ["TraceAttributesModel"] + mg = ModelGenerator( + class_name, self.schema, parents_class, table_name, self.name + ) + with open(filename, "w") as fd: + fd.write(model_shared_imports) + fd.write("\n") + fd.write(generate_class_def(class_name, parents_class)) + fd.write(mg.generate_model_description()) + fd.write("\n") + fd.write(mg.generate_table_name()) + fd.write("\n") + fd.write(mg.generate_model_fields()) + fd.write("\n") + fd.write(mg.generate_model_init()) + fd.write("\n") + fd.write(mg.generate_model_repr_str()) + fd.write("\n") + + init_file = os.path.join(self.model_path, "__init__.py") + + with open(init_file, "a") as file: + file.write(f"from .{self.prefix}{table_name} import {class_name}\n") + + return class_name + + def new_schemas(self, table_name: str) -> dict: + """ + This method takes a table name and creates a flask database model with the fields os said table + + :param str table_name: the name of the table to create + :return: the dictionary with the names of the schemas created + :rtype: dict + """ + filename = os.path.join(self.schema_path, self.prefix + table_name + ".py") + class_name_one = self.format_name(table_name, "_response") + class_name_edit = self.format_name(table_name, "_edit_request") + class_name_post = self.format_name(table_name, "_post_request") + class_name_post_bulk = self.format_name(table_name, "_post_bulk_request") + class_name_put_bulk = self.format_name(table_name, "_put_bulk_request") + class_name_put_bulk_one = class_name_put_bulk + "One" + + parents_class = ["Schema"] + partial_schema = self.schema["properties"][table_name]["items"] + sg = SchemaGenerator(partial_schema, table_name, self.name) + with open(filename, "w") as fd: + fd.write(sg.generate_schema_file_description()) + fd.write(schemas_imports) + fd.write("\n") + fd.write(generate_class_def(class_name_edit, parents_class)) + fd.write(sg.generate_edit_schema()) + fd.write("\n\n") + + fd.write(generate_class_def(class_name_post, parents_class)) + fd.write(sg.generate_post_schema()) + fd.write("\n\n") + + fd.write(generate_class_def(class_name_post_bulk, parents_class)) + fd.write(sg.generate_bulk_schema(class_name_post)) + fd.write("\n\n") + + parents_class = [class_name_edit] + fd.write(generate_class_def(class_name_put_bulk_one, parents_class)) + fd.write(sg.generate_put_bulk_schema_one()) + fd.write("\n\n") + + parents_class = ["Schema"] + fd.write(generate_class_def(class_name_put_bulk, parents_class)) + fd.write(sg.generate_bulk_schema(class_name_put_bulk_one)) + fd.write("\n\n") + + parents_class = [class_name_post] + fd.write(generate_class_def(class_name_one, parents_class)) + fd.write(sg.generate_schema()) + + init_file = os.path.join(self.schema_path, "__init__.py") + with open(init_file, "a") as file: + file.write( + f"from .{self.prefix}{table_name} import {class_name_one}, " + f"{class_name_edit}, {class_name_post}, {class_name_post_bulk}, {class_name_put_bulk}\n" + ) + + return { + "one": class_name_one, + "editRequest": class_name_edit, + "postRequest": class_name_post, + "postBulkRequest": class_name_post_bulk, + "putBulkRequest": class_name_put_bulk, + } + + def new_endpoint( + self, table_name: str, model_name: str, schemas_names: dict + ) -> None: + """ + This method takes a table name, a model_name and the names of the marshmallow schemas and + creates a flask endpoint with the methods passed + + :param str table_name: the name of the table to create + :param str model_name: the name of the model that have been created + :param dict schemas_names: the names of the schemas that have been created + :return: None + :rtype: None + """ + methods_to_add = self.get_methods(table_name) + roles_with_access = self.endpoints_access.get( + table_name, self.endpoints_access["default"] + ) + + # set names + filename = os.path.join(self.endpoint_path, self.prefix + table_name + ".py") + class_name_all = self.format_name(table_name, "_endpoint") + class_name_details = self.format_name(table_name, "_details_endpoint") + class_name_bulk = self.format_name(table_name, "_bulk_endpoint") + + eg = EndpointGenerator(table_name, self.name, model_name, schemas_names) + class_imports = [] + with open(filename, "w") as fd: + if any(len(v) for v in methods_to_add.values()): + fd.write(eg.generate_endpoints_imports(roles_with_access)) + fd.write("\n\n") + # Global + if len(methods_to_add["base"]): + self.create_endpoint_class( + class_name_all, + eg, + fd, + "base", + methods_to_add["base"], + roles_with_access, + ) + class_imports += [class_name_all] + # Details + if len(methods_to_add["detail"]): + self.create_endpoint_class( + class_name_details, + eg, + fd, + "detail", + methods_to_add["detail"], + roles_with_access, + ) + class_imports += [class_name_details] + # Bulk + if len(methods_to_add["bulk"]): + self.create_endpoint_class( + class_name_bulk, + eg, + fd, + "bulk", + methods_to_add["bulk"], + roles_with_access, + ) + class_imports += [class_name_bulk] + + # Write the imports in init + if len(class_imports): + with open(self.init_file, "a") as file: + file.write( + f"from .{self.prefix}{table_name} import {', '.join(class_imports)}\n" + ) + + id_type = self.get_id_type(table_name) + print(class_imports) + for res in class_imports: + self.init_resources += [ + dict( + resource=res, + urls=f'"{self.camel_to_url(res, id_type=id_type)}"', + endpoint=f'"{self.camel_to_ep(res)}"', + ) + ] + print(self.init_resources) + + def write_resources(self): + """ + Write the list of endpoints resources in __init__ file. + + :return: Nothing + """ + with open(self.init_file, "a") as file: + file.write("\n\n") + file.write("resources = [\n ") + file.write(",\n ".join(self.write_dict(d) for d in self.init_resources)) + file.write("\n]\n") + + @staticmethod + def write_dict(dic): + """ + Format a dict to be written in a python file. + Return dict {k1:v1, k2:v2} as the following string: + "dict( + k1 = v1, + k2 = v2 + )" + + :param dic: the dict + :return: a string + """ + content = ",\n ".join([f"{k}={v}" for k, v in dic.items()]) + return "dict(\n " + content + "\n )" + + @staticmethod + def create_endpoint_class( + class_name, eg, file, ep_type, methods, roles_with_access + ): + """ + Write an endpoint in a file + + :param class_name: the name of the class of the endpoint + :param eg: an instance of EndpointGenerator + :param file: the file + :param ep_type: the type of endpoint (base, bulk, detail) + :param methods: the methods to add to the endpoint. + :param roles_with_access: the roles which can access this endpoint. + :return: nothing (write in the file) + """ + parents_class = ["BaseMetaResource"] + file.write(generate_class_def(class_name, parents_class)) + file.write(eg.generate_endpoint_description(methods, ep_type)) + file.write("\n") + file.write(f' ROLES_WITH_ACCESS = [{", ".join(roles_with_access)}]\n') + file.write("\n") + file.write(eg.generate_endpoint_init()) + file.write("\n") + for m in methods: + file.write(eg.generate_endpoint(m)) + file.write("\n") + file.write("\n") + + @staticmethod + def snake_to_camel(name: str) -> str: + """ + This static method takes a name with underscores in it and changes it to camelCase + + :param str name: the name to mutate + :return: the mutated name + :rtype: str + """ + return "".join(word.title() for word in name.split("_")) + + @staticmethod + def camel_to_url(name: str, id_type) -> str: + """ + Transform a camelCase name into endpoint url: + NewTableEndpoint -> /new/table/ + The endpoint word is always removed in the url. + + :param name: name of the endpoint. + :param id_type: type of the primary key of the table. + :return: str url of the endpoint + """ + words = [w for w in re.findall("[A-Z][^A-Z]*", name) if w != "Endpoint"] + url = "/" + "/".join(w.lower() for w in words) + "/" + return url.replace("details", id_type) + + @staticmethod + def camel_to_ep(name: str) -> str: + """ + Transform a camelCase name into endpoint name: + NewTableEndpoint -> new-table + The endpoint word is always removed in the url. + + :param name: name of the endpoint + :return: str url of the endpoint + """ + words = [w for w in re.findall("[A-Z][^A-Z]*", name) if w != "Endpoint"] + return "-".join(w.lower() for w in words) + + def get_methods(self, table_name): + """ + Get the methods which will be used in each type of endpoint for a given table. + + :param str table_name: name of the table + :return: a dict in format {type: [list of methods] + """ + methods = self.options.get(table_name, self.options["default"]) + name_types = dict(base="list", bulk="bulk", detail="detail") + return { + t: [m for m in methods if m.split("_")[1] == ext] + for t, ext in name_types.items() + } + + def format_name(self, table_name, extension): + """ + Format the name of a schema, model or endpoint. + + :param table_name: The name of the table. + :param extension: the extension for the name of the object. + :return: the object name. + """ + return self.snake_to_camel(self.prefix + table_name + extension) + + def get_id_type(self, table_name): + """ + Get the type of the primary key of the table (id) + + :param table_name: name of the table in the schema. + :return: str: the type in format "" + """ + schema_table = self.schema["properties"][table_name]["items"]["properties"] + id_type=None + if "id" in schema_table.keys(): + id_type = schema_table["id"]["type"] + if id_type == "string" or isinstance(id_type, list): + return "" + elif id_type == "integer" or id_type == "number" or id_type is None: + return "" + else: + raise NotImplementedError(f"Unknown type for primary key: {id_type}") diff --git a/libs/core/cornflow_core/cli/tools/endpoint_tools.py b/cornflow-server/cornflow/cli/tools/endpoint_tools.py similarity index 77% rename from libs/core/cornflow_core/cli/tools/endpoint_tools.py rename to cornflow-server/cornflow/cli/tools/endpoint_tools.py index f03932609..d39c52dd0 100644 --- a/libs/core/cornflow_core/cli/tools/endpoint_tools.py +++ b/cornflow-server/cornflow/cli/tools/endpoint_tools.py @@ -9,22 +9,56 @@ def __init__(self, table_name, app_name, model_name, schemas_names): self.app_name = app_name self.model_name = model_name self.schemas_names = schemas_names + self.descriptions = { + "base": "Endpoint used to manage the table", + "bulk": "Endpoint used to perform bulk operations on the table", + "detail": "Endpoint used to perform detail operations on the table" + } - def generate_endpoints_imports(self): + def generate_endpoints_imports(self, roles): + """ + Generate the import text for an endpoint. + + :param roles: list of roles to import + :return: import text + """ return ( "# Imports from libraries\n" - "from flask_apispec import doc, marshal_with, use_kwargs\n" - "from cornflow_core.authentication import authenticate, BaseAuth\n" - "from cornflow_core.resources import BaseMetaResource\n\n" - "from cornflow_core.constants import SERVICE_ROLE\n" + "from flask_apispec import doc, marshal_with, use_kwargs\n\n" "# Import from internal modules\n" + "from cornflow.endpoints.meta_resource import BaseMetaResource\n\n" f"from ..models import {self.model_name}\n" f"from ..schemas import {', '.join(self.schemas_names.values())}\n\n" + "from cornflow.shared.authentication import authenticate, Auth\n" + f"from cornflow.shared.const import {', '.join(roles)}\n" ) - def generate_endpoint_description(self): + def get_type_methods(self, methods, ep_type): + """ + Select the methods of the table to use in the type of endpoint. + + :param methods: list of methods used for this table + :param ep_type: type of endpoint (base, bulk or detail) + :return: + """ + name_types = dict(base="list", bulk ="bulk", detail ="detail") + return [v[0] for v in [m.split("_") for m in methods] if v[1] == name_types[ep_type]] + + def generate_endpoint_description(self, methods, ep_type="base"): + """ + Generate the description of an endpoint. + + :param methods: list of available methods. + :param ep_type: type of endpoint (base, bulk or detail) + + :return: the description text + """ + type_methods = self.get_type_methods(methods, ep_type) + description = self.descriptions[ep_type] + app_name = f' of app {self.app_name}' if self.app_name is not None else "" res = ' """\n' - res += f" Endpoint used to manage the table {self.table_name} of app {self.app_name}\n" + res += f" {description} {self.table_name}{app_name}.\n\n" + res += f" Available methods: [{', '.join(type_methods)}]\n" res += ' """\n' return res @@ -41,7 +75,7 @@ def generate_endpoint_get_all(self): res += SP8 + 'description="Get list of all the elements in the table",\n' res += SP8 + f'tags=["{self.app_name}"],\n' res += " )\n" - res += " @authenticate(auth_class=BaseAuth())\n" + res += " @authenticate(auth_class=Auth())\n" res += f" @marshal_with({schema_name}(many=True))\n" res += " def get(self, **kwargs):\n" res += SP8 + '"""\n' @@ -66,7 +100,7 @@ def generate_endpoint_get_one(self): res += SP8 + 'description="Get one element of the table",\n' res += SP8 + f'tags=["{self.app_name}"],\n' res += " )\n" - res += " @authenticate(auth_class=BaseAuth())\n" + res += " @authenticate(auth_class=Auth())\n" res += f" @marshal_with({schema_name})\n" res += " def get(self, idx):\n" res += SP8 + '"""\n' @@ -93,7 +127,7 @@ def generate_endpoint_post(self): res += SP8 + 'description="Add a new row to the table",\n' res += SP8 + f'tags=["{self.app_name}"],\n' res += " )\n" - res += " @authenticate(auth_class=BaseAuth())\n" + res += " @authenticate(auth_class=Auth())\n" res += f" @marshal_with({schema_marshal})\n" res += f' @use_kwargs({schema_kwargs}, location="json")\n' res += " def post(self, **kwargs):\n" @@ -116,7 +150,7 @@ def generate_endpoint_delete_one(self): res += SP8 + 'description="Delete one row of the table",\n' res += SP8 + f'tags=["{self.app_name}"], \n' res += " )\n" - res += " @authenticate(auth_class=BaseAuth())\n" + res += " @authenticate(auth_class=Auth())\n" res += " def delete(self, idx):\n" res += SP8 + '"""\n' res += SP8 + "API method to delete a row of the table.\n" @@ -143,7 +177,7 @@ def generate_endpoint_put(self): res += SP8 + 'description="Edit one row of the table",\n' res += SP8 + f'tags=["{self.app_name}"], \n' res += " )\n" - res += " @authenticate(auth_class=BaseAuth())\n" + res += " @authenticate(auth_class=Auth())\n" res += f' @use_kwargs({schema_name}, location="json")\n' res += " def put(self, idx, **data):\n" res += SP8 + '"""\n' @@ -171,7 +205,7 @@ def generate_endpoint_patch(self): res += SP8 + 'description="Patch one row of the table",\n' res += SP8 + f'tags=["{self.app_name}"], \n' res += " )\n" - res += " @authenticate(auth_class=BaseAuth())\n" + res += " @authenticate(auth_class=Auth())\n" res += f' @use_kwargs({schema_name}, location="json")\n' res += " def patch(self, idx, **data):\n" res += SP8 + '"""\n' @@ -200,7 +234,7 @@ def generate_endpoint_post_bulk(self): res += SP8 + 'description="Add several new rows to the table",\n' res += SP8 + f'tags=["{self.app_name}"],\n' res += " )\n" - res += " @authenticate(auth_class=BaseAuth())\n" + res += " @authenticate(auth_class=Auth())\n" res += f" @marshal_with({schema_marshal}(many=True))\n" res += f' @use_kwargs({schema_kwargs}, location="json")\n' res += " def post(self, **kwargs):\n" @@ -225,7 +259,7 @@ def generate_endpoint_put_bulk(self): res += SP8 + 'description="Updates several rows of the table or adds them if they do not exist",\n' res += SP8 + f'tags=["{self.app_name}"],\n' res += " )\n" - res += " @authenticate(auth_class=BaseAuth())\n" + res += " @authenticate(auth_class=Auth())\n" res += f" @marshal_with({schema_marshal}(many=True))\n" res += f' @use_kwargs({schema_kwargs}, location="json")\n' res += " def put(self, **kwargs):\n" @@ -241,4 +275,14 @@ def generate_endpoint_put_bulk(self): res += SP8 + ":rtype: Tuple(dict, integer)\n" res += SP8 + '"""\n' res += SP8 + "return self.post_bulk_update(data=kwargs)\n" - return res \ No newline at end of file + return res + + def generate_endpoint(self, method): + ep_map = dict(get_list=self.generate_endpoint_get_all, post_list=self.generate_endpoint_post, get_detail=self.generate_endpoint_get_one, + put_detail=self.generate_endpoint_put, + patch_detail=self.generate_endpoint_patch, + delete_detail= self.generate_endpoint_delete_one, + post_bulk=self.generate_endpoint_post_bulk, + put_bulk=self.generate_endpoint_put_bulk + ) + return ep_map[method]() diff --git a/libs/core/cornflow_core/cli/tools/models_tools.py b/cornflow-server/cornflow/cli/tools/models_tools.py similarity index 92% rename from libs/core/cornflow_core/cli/tools/models_tools.py rename to cornflow-server/cornflow/cli/tools/models_tools.py index a446b81d2..1be105a2c 100644 --- a/libs/core/cornflow_core/cli/tools/models_tools.py +++ b/cornflow-server/cornflow/cli/tools/models_tools.py @@ -1,8 +1,10 @@ +from .tools import get_type + # Models model_shared_imports = ( "# Import from libraries\n" - "from cornflow_core.shared import db\n" - "from cornflow_core.models import TraceAttributesModel\n" + "from cornflow.shared import db\n" + "from cornflow.models.meta_models import TraceAttributesModel\n" "from sqlalchemy.dialects.postgresql import ARRAY\n\n" ) SP8 = 8 * " " @@ -13,6 +15,9 @@ "number": "db.Float", "boolean": "db.Boolean", "array": "ARRAY", + "date": "db.Date", + "datetime": "db.DateTime", + "time": "db.Time", } @@ -80,16 +85,9 @@ def has_id(schema): if not has_id(schema_table["properties"]): res += f" id = db.Column(db.Integer, primary_key=True, autoincrement=True)\n" for key, val in schema_table["properties"].items(): - nullable = False res += f" {key} = db.Column(" - types = val["type"] - if isinstance(types, list): - nullable = True - if types[0] == "null": - types = types[1] - else: - types = types[0] - res += JSON_TYPES_TO_SQLALCHEMY[types] + ty, nullable = get_type(val) + res += JSON_TYPES_TO_SQLALCHEMY[ty] if val.get("foreign_key"): foreign_table, foreign_prop = val["foreign_key"].split(".") if self.app_name is not None: diff --git a/libs/core/cornflow_core/cli/tools/schema_generator.py b/cornflow-server/cornflow/cli/tools/schema_generator.py similarity index 100% rename from libs/core/cornflow_core/cli/tools/schema_generator.py rename to cornflow-server/cornflow/cli/tools/schema_generator.py diff --git a/libs/core/cornflow_core/cli/tools/schemas_tools.py b/cornflow-server/cornflow/cli/tools/schemas_tools.py similarity index 71% rename from libs/core/cornflow_core/cli/tools/schemas_tools.py rename to cornflow-server/cornflow/cli/tools/schemas_tools.py index 77283e1c7..fc027a4f1 100644 --- a/libs/core/cornflow_core/cli/tools/schemas_tools.py +++ b/cornflow-server/cornflow/cli/tools/schemas_tools.py @@ -1,4 +1,6 @@ # Schemas +from .tools import get_type + schemas_imports = "from marshmallow import fields, Schema\n\n" JSON_TYPES_TO_FIELDS = { @@ -7,6 +9,9 @@ "number": "fields.Number", "boolean": "fields.Boolean", "array": "fields.List", + "date": "fields.Date", + "datetime": "fields.DateTime", + "time": "fields.Time", } @@ -30,13 +35,8 @@ def generate_edit_schema(self): for key, val in self.schema["properties"].items(): if key == "id": continue - val_type = val["type"] - if isinstance(val_type, list): - if val_type[0] == "null": - val_type = val_type[1] - else: - val_type = val_type[0] - res += f" {key} = {JSON_TYPES_TO_FIELDS[val_type]}(" + ty, nullable = get_type(val) + res += f" {key} = {JSON_TYPES_TO_FIELDS[ty]}(" res += "required=False" res += ")\n" return res @@ -44,13 +44,8 @@ def generate_edit_schema(self): def generate_post_schema(self): res = "" for key, val in self.schema["properties"].items(): - val_type = val["type"] - if isinstance(val_type, list): - if val_type[0] == "null": - val_type = val_type[1] - else: - val_type = val_type[0] - res += f" {key} = {JSON_TYPES_TO_FIELDS[val_type]}(" + ty, nullable = get_type(val) + res += f" {key} = {JSON_TYPES_TO_FIELDS[ty]}(" if key in self.schema["required"]: res += "required=True" else: @@ -64,7 +59,12 @@ def generate_bulk_schema(one_schema): return res def generate_put_bulk_schema_one(self): - return " id = fields.Int(required=True)\n" + if not self.schema["properties"].get("id"): + return " id = fields.Int(required=True)\n" + else: + id_type=self.schema["properties"].get("id")["type"] + return f' id = {JSON_TYPES_TO_FIELDS[id_type]}(required=True)\n' + def generate_schema(self): if not self.schema["properties"].get("id"): diff --git a/cornflow-server/cornflow/cli/tools/tools.py b/cornflow-server/cornflow/cli/tools/tools.py new file mode 100644 index 000000000..57847c9c8 --- /dev/null +++ b/cornflow-server/cornflow/cli/tools/tools.py @@ -0,0 +1,31 @@ +# Shared +TYPE_PRIORITIES = ["array", "string", "number", "integer", "boolean"] + + +def generate_class_def(class_name, parent_class): + return f'class {class_name}({", ".join(parent_class)}):\n' + + +def get_type(prop): + """ + Translate json schema types into python type functions. + ex: string -> str + + :param prop: str properties of the json schema. + :return: a tuple with the main type (str) and nullable (bool) + """ + nullable = False + ty = prop["type"] + fmt = prop.get("format", None) + # manage list of types + if isinstance(ty, list): + if "null" in ty: + nullable = True + if any(t in TYPE_PRIORITIES for t in ty): + ty = [t for t in TYPE_PRIORITIES if t in ty][0] + else: + raise ValueError("unknown type: " + str(ty)) + # manage format + if ty == "string" and fmt in ["date", "datetime", "time"]: + ty = fmt + return ty, nullable diff --git a/cornflow-server/cornflow/commands/actions.py b/cornflow-server/cornflow/commands/actions.py index 6fbd92ee9..441ee4e4e 100644 --- a/cornflow-server/cornflow/commands/actions.py +++ b/cornflow-server/cornflow/commands/actions.py @@ -2,14 +2,14 @@ def register_actions_command(verbose: bool = True): from flask import current_app from sqlalchemy.exc import DBAPIError, IntegrityError - from cornflow_core.models import ActionBaseModel + from cornflow.models import ActionModel from cornflow.shared.const import ACTIONS_MAP - from cornflow_core.shared import db + from cornflow.shared import db - actions_registered = [ac.name for ac in ActionBaseModel.get_all_objects()] + actions_registered = [ac.name for ac in ActionModel.get_all_objects()] actions_to_register = [ - ActionBaseModel(id=key, name=value) + ActionModel(id=key, name=value) for key, value in ACTIONS_MAP.items() if value not in actions_registered ] diff --git a/cornflow-server/cornflow/commands/dag.py b/cornflow-server/cornflow/commands/dag.py index e38488abe..03995ba87 100644 --- a/cornflow-server/cornflow/commands/dag.py +++ b/cornflow-server/cornflow/commands/dag.py @@ -11,7 +11,7 @@ def register_deployed_dags_command( # Internal modules imports from cornflow_client.airflow.api import Airflow from cornflow.models import DeployedDAG - from cornflow_core.shared import db + from cornflow.shared import db af_client = Airflow(url, user, pwd) max_attempts = 20 diff --git a/cornflow-server/cornflow/commands/permissions.py b/cornflow-server/cornflow/commands/permissions.py index ea8532afa..2ed923fb4 100644 --- a/cornflow-server/cornflow/commands/permissions.py +++ b/cornflow-server/cornflow/commands/permissions.py @@ -5,8 +5,8 @@ BASE_PERMISSION_ASSIGNATION, EXTRA_PERMISSION_ASSIGNATION, ) -from cornflow_core.models import ViewBaseModel, PermissionViewRoleBaseModel -from cornflow_core.shared import db +from cornflow.models import ViewModel, PermissionViewRoleModel +from cornflow.shared import db from flask import current_app from sqlalchemy.exc import DBAPIError, IntegrityError @@ -25,8 +25,8 @@ def register_base_permissions_command(external_app: str = None, verbose: bool = resources_to_register = [] exit() - views_in_db = {view.name: view.id for view in ViewBaseModel.get_all_objects()} - permissions_in_db = [perm for perm in PermissionViewRoleBaseModel.get_all_objects()] + views_in_db = {view.name: view.id for view in ViewModel.get_all_objects()} + permissions_in_db = [perm for perm in PermissionViewRoleModel.get_all_objects()] permissions_in_db_keys = [ (perm.role_id, perm.action_id, perm.api_view_id) for perm in permissions_in_db ] @@ -34,7 +34,7 @@ def register_base_permissions_command(external_app: str = None, verbose: bool = # Create base permissions permissions_in_app = [ - PermissionViewRoleBaseModel( + PermissionViewRoleModel( { "role_id": role, "action_id": action, @@ -45,7 +45,7 @@ def register_base_permissions_command(external_app: str = None, verbose: bool = for view in resources_to_register if role in view["resource"].ROLES_WITH_ACCESS ] + [ - PermissionViewRoleBaseModel( + PermissionViewRoleModel( { "role_id": role, "action_id": action, @@ -129,7 +129,7 @@ def register_dag_permissions_command( from sqlalchemy.exc import DBAPIError, IntegrityError from cornflow.models import DeployedDAG, PermissionsDAG, UserModel - from cornflow_core.shared import db + from cornflow.shared import db if open_deployment is None: open_deployment = int(current_app.config["OPEN_DEPLOYMENT"]) diff --git a/cornflow-server/cornflow/commands/roles.py b/cornflow-server/cornflow/commands/roles.py index ebb8ae512..2fdc24197 100644 --- a/cornflow-server/cornflow/commands/roles.py +++ b/cornflow-server/cornflow/commands/roles.py @@ -3,14 +3,14 @@ def register_roles_command(verbose: bool = True): from sqlalchemy.exc import DBAPIError, IntegrityError from flask import current_app - from cornflow_core.models import RoleBaseModel + from cornflow.models import RoleModel from cornflow.shared.const import ROLES_MAP - from cornflow_core.shared import db + from cornflow.shared import db - roles_registered = [role.name for role in RoleBaseModel.get_all_objects()] + roles_registered = [role.name for role in RoleModel.get_all_objects()] roles_to_register = [ - RoleBaseModel({"id": key, "name": value}) + RoleModel({"id": key, "name": value}) for key, value in ROLES_MAP.items() if value not in roles_registered ] diff --git a/cornflow-server/cornflow/commands/users.py b/cornflow-server/cornflow/commands/users.py index 65ac3d38f..8aaa937f5 100644 --- a/cornflow-server/cornflow/commands/users.py +++ b/cornflow-server/cornflow/commands/users.py @@ -1,8 +1,7 @@ def create_user_with_role( username, email, password, role_name, role, verbose: bool = False ): - from cornflow.models import UserModel, UserRoleModel - from cornflow_core.models import RoleBaseModel + from cornflow.models import UserModel, UserRoleModel, RoleModel from flask import current_app user = UserModel.get_one_user_by_username(username) @@ -23,7 +22,7 @@ def create_user_with_role( user_actual_roles = [ur.role for ur in user_roles] if ( user_roles is not None - and RoleBaseModel.get_one_object(role) in user_actual_roles + and RoleModel.get_one_object(role) in user_actual_roles ): if verbose: current_app.logger.info( diff --git a/cornflow-server/cornflow/commands/views.py b/cornflow-server/cornflow/commands/views.py index 8afac5c43..b9c01e4a4 100644 --- a/cornflow-server/cornflow/commands/views.py +++ b/cornflow-server/cornflow/commands/views.py @@ -1,10 +1,13 @@ -import sys -from importlib import import_module -from cornflow_core.models import ViewBaseModel -from cornflow_core.shared import db +# Imports from external libraries from flask import current_app +from importlib import import_module from sqlalchemy.exc import DBAPIError, IntegrityError +import sys + +# Imports from internal libraries +from cornflow.models import ViewModel +from cornflow.shared import db def register_views_command(external_app: str = None, verbose: bool = False): @@ -22,10 +25,10 @@ def register_views_command(external_app: str = None, verbose: bool = False): resources_to_register = [] exit() - views_registered = [view.name for view in ViewBaseModel.get_all_objects()] + views_registered = [view.name for view in ViewModel.get_all_objects()] views_to_register = [ - ViewBaseModel( + ViewModel( { "name": view["endpoint"], "url_rule": view["urls"], diff --git a/cornflow-server/cornflow/config.py b/cornflow-server/cornflow/config.py index f99c7f243..81efbd5b6 100644 --- a/cornflow-server/cornflow/config.py +++ b/cornflow-server/cornflow/config.py @@ -77,12 +77,16 @@ class DefaultConfig(object): class Development(DefaultConfig): + """ """ + ENV = "development" + class Testing(DefaultConfig): """ """ + ENV = "testing" SQLALCHEMY_TRACK_MODIFICATIONS = False DEBUG = False TESTING = True @@ -100,6 +104,7 @@ class Testing(DefaultConfig): class Production(DefaultConfig): """ """ + ENV = "production" SQLALCHEMY_TRACK_MODIFICATIONS = False DEBUG = False TESTING = False diff --git a/cornflow-server/cornflow/endpoints/action.py b/cornflow-server/cornflow/endpoints/action.py index 339efce87..ed5b39302 100644 --- a/cornflow-server/cornflow/endpoints/action.py +++ b/cornflow-server/cornflow/endpoints/action.py @@ -1,16 +1,15 @@ """ """ - -from cornflow_core.authentication import authenticate -from cornflow_core.resources import BaseMetaResource -from cornflow_core.schemas import ActionsResponse +# Imports from external libraries from flask_apispec import marshal_with, doc from flask import current_app # Import from internal modules -from cornflow_core.models import ActionBaseModel -from cornflow.shared.authentication import Auth +from cornflow.endpoints.meta_resource import BaseMetaResource +from cornflow.models import ActionModel +from cornflow.schemas.action import ActionsResponse +from cornflow.shared.authentication import Auth, authenticate from cornflow.shared.const import ADMIN_ROLE @@ -20,7 +19,7 @@ class ActionListEndpoint(BaseMetaResource): def __init__(self): super().__init__() - self.data_model = ActionBaseModel + self.data_model = ActionModel @doc(description="Get all the actions", tags=["Actions"]) @authenticate(auth_class=Auth()) diff --git a/cornflow-server/cornflow/endpoints/alarms.py b/cornflow-server/cornflow/endpoints/alarms.py index 5d0689fb1..74e67e287 100644 --- a/cornflow-server/cornflow/endpoints/alarms.py +++ b/cornflow-server/cornflow/endpoints/alarms.py @@ -1,16 +1,15 @@ # Imports from libraries from flask_apispec import doc, marshal_with, use_kwargs -from cornflow_core.authentication import authenticate -from cornflow_core.resources import BaseMetaResource # Import from internal modules +from cornflow.endpoints.meta_resource import BaseMetaResource from cornflow.models import AlarmsModel from cornflow.schemas.alarms import ( AlarmsResponse, AlarmsPostRequest, QueryFiltersAlarms ) -from cornflow.shared.authentication import Auth +from cornflow.shared.authentication import Auth, authenticate class AlarmsEndpoint(BaseMetaResource): diff --git a/cornflow-server/cornflow/endpoints/apiview.py b/cornflow-server/cornflow/endpoints/apiview.py index bb3d098a9..fda4ed633 100644 --- a/cornflow-server/cornflow/endpoints/apiview.py +++ b/cornflow-server/cornflow/endpoints/apiview.py @@ -1,20 +1,17 @@ """ """ -from cornflow_core.authentication import authenticate - # Import from internal modules -from cornflow_core.models import ViewBaseModel -from cornflow_core.resources import BaseMetaResource -from cornflow_core.schemas import ViewResponse +from cornflow.endpoints.meta_resource import BaseMetaResource +from cornflow.models import ViewModel +from cornflow.schemas.view import ViewResponse +from cornflow.shared.authentication import Auth, authenticate +from cornflow.shared.const import ADMIN_ROLE -# Import from libraries +# Import from external libraries from flask_apispec import marshal_with, doc from flask import current_app -from cornflow.shared.authentication import Auth -from cornflow.shared.const import ADMIN_ROLE - class ApiViewListEndpoint(BaseMetaResource): ROLES_WITH_ACCESS = [ADMIN_ROLE] @@ -24,7 +21,7 @@ class ApiViewListEndpoint(BaseMetaResource): def __init__(self): super().__init__() - self.data_model = ViewBaseModel + self.data_model = ViewModel @doc(description="Get all the api views", tags=["ApiViews"]) @authenticate(auth_class=Auth()) diff --git a/cornflow-server/cornflow/endpoints/case.py b/cornflow-server/cornflow/endpoints/case.py index 17788eef8..f01b37825 100644 --- a/cornflow-server/cornflow/endpoints/case.py +++ b/cornflow-server/cornflow/endpoints/case.py @@ -4,6 +4,10 @@ These endpoints have different access url, but manage the same data entities """ # Import from libraries +from cornflow_client.constants import ( + INSTANCE_SCHEMA, + SOLUTION_SCHEMA +) from flask import current_app from flask_apispec import marshal_with, use_kwargs, doc from flask_inflate import inflate @@ -11,8 +15,13 @@ # Import from internal modules +from cornflow.endpoints.meta_resource import BaseMetaResource from cornflow.models import CaseModel, ExecutionModel, DeployedDAG, InstanceModel - +from cornflow.shared.authentication import Auth, authenticate +from cornflow.shared.compress import compressed +from cornflow.shared.const import VIEWER_ROLE, PLANNER_ROLE, ADMIN_ROLE +from cornflow.shared.exceptions import InvalidData, ObjectDoesNotExist +from cornflow.shared.validators import json_schema_validate_as_string from cornflow.schemas.case import ( CaseBase, CaseFromInstanceExecution, @@ -26,22 +35,6 @@ CaseListAllWithIndicators, ) -from cornflow.schemas.model_json import DataSchema -from cornflow.shared.authentication import Auth -from cornflow.shared.const import VIEWER_ROLE, PLANNER_ROLE, ADMIN_ROLE - -from cornflow_client.constants import ( - INSTANCE_SCHEMA, - SOLUTION_SCHEMA -) - -from cornflow_core.authentication import authenticate -from cornflow_core.compress import compressed -from cornflow_core.exceptions import InvalidData, ObjectDoesNotExist -from cornflow_core.resources import BaseMetaResource -from cornflow_core.shared import json_schema_validate_as_string - - class CaseEndpoint(BaseMetaResource): """ diff --git a/cornflow-server/cornflow/endpoints/dag.py b/cornflow-server/cornflow/endpoints/dag.py index 67cae816a..4ae29de62 100644 --- a/cornflow-server/cornflow/endpoints/dag.py +++ b/cornflow-server/cornflow/endpoints/dag.py @@ -8,6 +8,7 @@ from flask_apispec import use_kwargs, doc, marshal_with # Import from internal modules +from cornflow.endpoints.meta_resource import BaseMetaResource from cornflow.models import DeployedDAG, ExecutionModel, InstanceModel, CaseModel from cornflow.schemas import DeployedDAGSchema, DeployedDAGEditSchema from cornflow.schemas.case import CaseCheckRequest @@ -19,8 +20,7 @@ ExecutionSchema, ) -from cornflow.schemas.model_json import DataSchema -from cornflow.shared.authentication import Auth +from cornflow.shared.authentication import Auth, authenticate from cornflow.shared.const import ( ADMIN_ROLE, EXEC_STATE_CORRECT, @@ -29,14 +29,8 @@ SERVICE_ROLE, PLANNER_ROLE, ) - -from cornflow_core.exceptions import ObjectDoesNotExist, InvalidData -from cornflow_core.authentication import authenticate -from cornflow_core.resources import BaseMetaResource -from cornflow_core.shared import json_schema_validate_as_string - - -execution_schema = ExecutionSchema() +from cornflow.shared.exceptions import ObjectDoesNotExist, InvalidData +from cornflow.shared.validators import json_schema_validate_as_string class DAGDetailEndpoint(BaseMetaResource): diff --git a/cornflow-server/cornflow/endpoints/data_check.py b/cornflow-server/cornflow/endpoints/data_check.py index 92a7fb57f..0a0f29c73 100644 --- a/cornflow-server/cornflow/endpoints/data_check.py +++ b/cornflow-server/cornflow/endpoints/data_check.py @@ -5,18 +5,14 @@ # Import from libraries from cornflow_client.airflow.api import Airflow from cornflow_client.constants import INSTANCE_SCHEMA, SOLUTION_SCHEMA -from cornflow_core.authentication import authenticate -from cornflow_core.exceptions import AirflowError, ObjectDoesNotExist, InvalidUsage, InvalidData -from cornflow_core.resources import BaseMetaResource -from cornflow_core.shared import json_schema_validate_as_string from flask import request, current_app from flask_apispec import marshal_with, doc # Import from internal modules +from cornflow.endpoints.meta_resource import BaseMetaResource from cornflow.models import InstanceModel, ExecutionModel, CaseModel, DeployedDAG from cornflow.schemas.execution import ExecutionDetailsEndpointResponse -from cornflow.shared.authentication import Auth - +from cornflow.shared.authentication import Auth, authenticate from cornflow.shared.const import ( EXEC_STATE_QUEUED, EXEC_STATE_ERROR, @@ -24,7 +20,13 @@ EXEC_STATE_NOT_RUN, EXECUTION_STATE_MESSAGE_DICT, VIEWER_ROLE, PLANNER_ROLE, ADMIN_ROLE, ) - +from cornflow.shared.exceptions import ( + AirflowError, + ObjectDoesNotExist, + InvalidUsage, + InvalidData +) +from cornflow.shared.validators import json_schema_validate_as_string class DataCheckExecutionEndpoint(BaseMetaResource): diff --git a/cornflow-server/cornflow/endpoints/example_data.py b/cornflow-server/cornflow/endpoints/example_data.py index bf4e5ff4f..90739ef06 100644 --- a/cornflow-server/cornflow/endpoints/example_data.py +++ b/cornflow-server/cornflow/endpoints/example_data.py @@ -6,17 +6,15 @@ from cornflow_client.airflow.api import Airflow from flask import current_app, request from flask_apispec import marshal_with, doc -from cornflow_core.authentication import authenticate import json # Import from internal modules +from cornflow.endpoints.meta_resource import BaseMetaResource from cornflow.models import PermissionsDAG -from cornflow.shared.authentication import Auth -from cornflow_core.exceptions import AirflowError, NoPermission from cornflow.schemas.example_data import ExampleData -from cornflow_core.resources import BaseMetaResource - +from cornflow.shared.authentication import Auth, authenticate from cornflow.shared.const import VIEWER_ROLE, PLANNER_ROLE, ADMIN_ROLE +from cornflow.shared.exceptions import AirflowError, NoPermission class ExampleDataDetailsEndpoint(BaseMetaResource): diff --git a/cornflow-server/cornflow/endpoints/execution.py b/cornflow-server/cornflow/endpoints/execution.py index f650a839c..8f6ddd7f4 100644 --- a/cornflow-server/cornflow/endpoints/execution.py +++ b/cornflow-server/cornflow/endpoints/execution.py @@ -6,15 +6,12 @@ # Import from libraries from cornflow_client.airflow.api import Airflow -from cornflow_core.resources import BaseMetaResource -from cornflow_core.shared import ( - json_schema_validate_as_string, -) from cornflow_client.constants import INSTANCE_SCHEMA, CONFIG_SCHEMA, SOLUTION_SCHEMA from flask import request, current_app from flask_apispec import marshal_with, use_kwargs, doc # Import from internal modules +from cornflow.endpoints.meta_resource import BaseMetaResource from cornflow.models import InstanceModel, DeployedDAG, ExecutionModel from cornflow.schemas.execution import ( ExecutionDetailsEndpointResponse, @@ -28,8 +25,8 @@ QueryFiltersExecution, ReLaunchExecutionRequest, ) - -from cornflow.shared.authentication import Auth +from cornflow.shared.authentication import Auth, authenticate +from cornflow.shared.compress import compressed from cornflow.shared.const import ( EXEC_STATE_RUNNING, EXEC_STATE_ERROR, @@ -41,10 +38,8 @@ EXEC_STATE_STOPPED, EXEC_STATE_QUEUED, ) -from cornflow_core.authentication import authenticate -from cornflow_core.exceptions import AirflowError, ObjectDoesNotExist, InvalidData -from cornflow_core.compress import compressed - +from cornflow.shared.exceptions import AirflowError, ObjectDoesNotExist, InvalidData +from cornflow.shared.validators import json_schema_validate_as_string class ExecutionEndpoint(BaseMetaResource): """ diff --git a/cornflow-server/cornflow/endpoints/health.py b/cornflow-server/cornflow/endpoints/health.py index de828dba1..699cd005e 100644 --- a/cornflow-server/cornflow/endpoints/health.py +++ b/cornflow-server/cornflow/endpoints/health.py @@ -10,10 +10,10 @@ from flask_apispec import marshal_with, doc # Import from internal modules +from cornflow.endpoints.meta_resource import BaseMetaResource +from cornflow.models import UserModel from cornflow.schemas.health import HealthResponse from cornflow.shared.const import STATUS_HEALTHY, STATUS_UNHEALTHY -from cornflow_core.resources import BaseMetaResource -from cornflow.models import UserModel class HealthEndpoint(BaseMetaResource): diff --git a/cornflow-server/cornflow/endpoints/instance.py b/cornflow-server/cornflow/endpoints/instance.py index fc125183e..3d46f33b9 100644 --- a/cornflow-server/cornflow/endpoints/instance.py +++ b/cornflow-server/cornflow/endpoints/instance.py @@ -5,8 +5,7 @@ """ # Import from libraries -from cornflow_core.resources import BaseMetaResource -from cornflow_core.shared import json_schema_validate_as_string +from cornflow_client.constants import INSTANCE_SCHEMA from flask import request, current_app from flask_apispec import marshal_with, use_kwargs, doc from flask_inflate import inflate @@ -14,9 +13,9 @@ import os import pulp from werkzeug.utils import secure_filename -from cornflow_core.authentication import authenticate # Import from internal modules +from cornflow.endpoints.meta_resource import BaseMetaResource from cornflow.models import InstanceModel, DeployedDAG from cornflow.schemas.instance import ( InstanceSchema, @@ -29,10 +28,11 @@ QueryFiltersInstance, ) -from cornflow.shared.authentication import Auth -from cornflow_core.compress import compressed -from cornflow_core.exceptions import InvalidUsage, InvalidData -from cornflow_client.constants import INSTANCE_SCHEMA +from cornflow.shared.authentication import Auth, authenticate +from cornflow.shared.compress import compressed +from cornflow.shared.exceptions import InvalidUsage, InvalidData +from cornflow.shared.validators import json_schema_validate_as_string + # Initialize the schema that all endpoints are going to use diff --git a/cornflow-server/cornflow/endpoints/licenses.py b/cornflow-server/cornflow/endpoints/licenses.py index 9eda53c28..e0b04c9d1 100644 --- a/cornflow-server/cornflow/endpoints/licenses.py +++ b/cornflow-server/cornflow/endpoints/licenses.py @@ -1,10 +1,11 @@ # Imports from libraries -from cornflow_core.authentication import authenticate -from cornflow_core.resources import BaseMetaResource from flask_apispec import doc + +# Imports from internal modules +from cornflow.endpoints.meta_resource import BaseMetaResource +from cornflow.shared.authentication import Auth, authenticate from cornflow.shared.const import VIEWER_ROLE, PLANNER_ROLE, ADMIN_ROLE from cornflow.shared.licenses import get_licenses_summary -from cornflow.shared.authentication import Auth class LicensesEndpoint(BaseMetaResource): diff --git a/cornflow-server/cornflow/endpoints/login.py b/cornflow-server/cornflow/endpoints/login.py index 442c07d39..09adab30d 100644 --- a/cornflow-server/cornflow/endpoints/login.py +++ b/cornflow-server/cornflow/endpoints/login.py @@ -2,17 +2,178 @@ External endpoint for the user to login to the cornflow webserver """ -# Full import from libraries -from cornflow_core.resources import LoginBaseEndpoint - # Partial imports from flask import current_app from flask_apispec import use_kwargs, doc +from sqlalchemy.exc import IntegrityError, DBAPIError # Import from internal modules +from cornflow.endpoints.meta_resource import BaseMetaResource from cornflow.models import PermissionsDAG, UserModel, UserRoleModel -from cornflow_core.schemas import LoginEndpointRequest, LoginOpenAuthRequest -from cornflow.shared.authentication import Auth +from cornflow.schemas.user import LoginEndpointRequest, LoginOpenAuthRequest +from cornflow.shared import db +from cornflow.shared.authentication import Auth, LDAPBase +from cornflow.shared.const import ( + AUTH_DB, + AUTH_LDAP, + AUTH_OID, + OID_AZURE, + OID_GOOGLE, + OID_NONE, +) +from cornflow.shared.exceptions import ( + ConfigurationError, + InvalidCredentials, + InvalidUsage, + EndpointNotImplemented, +) + + +class LoginBaseEndpoint(BaseMetaResource): + """ + Base endpoint to perform a login action from a user + """ + def __init__(self): + super().__init__() + self.ldap_class = LDAPBase + + def log_in(self, **kwargs): + """ + This method is in charge of performing the log in of the user + + :param kwargs: keyword arguments passed for the login, these can be username, password or a token + :return: the response of the login or it raises an error. The correct response is a dict + with the newly issued token and the user id, and a status code of 200 + :rtype: dict + """ + auth_type = current_app.config["AUTH_TYPE"] + + if auth_type == AUTH_DB: + user = self.auth_db_authenticate(**kwargs) + elif auth_type == AUTH_LDAP: + user = self.auth_ldap_authenticate(**kwargs) + elif auth_type == AUTH_OID: + user = self.auth_oid_authenticate(**kwargs) + else: + raise ConfigurationError() + + try: + token = self.auth_class.generate_token(user.id) + except Exception as e: + raise InvalidUsage(f"Error in generating user token: {str(e)}", 400) + + return {"token": token, "id": user.id}, 200 + + def auth_db_authenticate(self, username, password): + """ + Method in charge of performing the authentication against the database + + :param str username: the username of the user to log in + :param str password: the password of the user to log in + :return: the user object or it raises an error if it has not been possible to log in + :rtype: :class:`UserModel` + """ + user = self.data_model.get_one_object(username=username) + + if not user: + raise InvalidCredentials() + + if not user.check_hash(password): + raise InvalidCredentials() + + return user + + def auth_ldap_authenticate(self, username, password): + """ + Method in charge of performing the authentication against the ldap server + + :param str username: the username of the user to log in + :param str password: the password of the user to log in + :return: the user object or it raises an error if it has not been possible to log in + :rtype: :class:`UserModel` + """ + ldap_obj = self.ldap_class(current_app.config) + if not ldap_obj.authenticate(username, password): + raise InvalidCredentials() + user = self.data_model.get_one_object(username=username) + if not user: + current_app.logger.info(f"LDAP user {username} does not exist and is created") + email = ldap_obj.get_user_email(username) + if not email: + email = "" + data = {"username": username, "email": email} + user = self.data_model(data=data) + user.save() + + roles = ldap_obj.get_user_roles(username) + + try: + self.user_role_association.del_one_user(user.id) + for role in roles: + user_role = self.user_role_association( + data={"user_id": user.id, "role_id": role} + ) + user_role.save() + + except IntegrityError as e: + db.session.rollback() + current_app.logger.error(f"Integrity error on user role assignment on log in: {e}") + except DBAPIError as e: + db.session.rollback() + current_app.logger.error(f"Unknown error on user role assignment on log in: {e}") + + return user + + def auth_oid_authenticate(self, token): + """ + Method in charge of performing the log in with the token issued by an Open ID provider + + :param str token: the token that the user has obtained from the Open ID provider + :return: the user object or it raises an error if it has not been possible to log in + :rtype: :class:`UserModel` + """ + oid_provider = int(current_app.config["OID_PROVIDER"]) + + client_id = current_app.config["OID_CLIENT_ID"] + tenant_id = current_app.config["OID_TENANT_ID"] + issuer = current_app.config["OID_ISSUER"] + + if client_id is None or tenant_id is None or issuer is None: + raise ConfigurationError("The OID provider configuration is not valid") + + if oid_provider == OID_AZURE: + decoded_token = self.auth_class().validate_oid_token( + token, client_id, tenant_id, issuer, oid_provider + ) + + elif oid_provider == OID_GOOGLE: + raise EndpointNotImplemented("The selected OID provider is not implemented") + elif oid_provider == OID_NONE: + raise EndpointNotImplemented("The OID provider configuration is not valid") + else: + raise EndpointNotImplemented("The OID provider configuration is not valid") + + username = decoded_token["preferred_username"] + + user = self.data_model.get_one_object(username=username) + + if not user: + current_app.logger.info(f"OpenID user {username} does not exist and is created") + + data = {"username": username, "email": username} + + user = self.data_model(data=data) + user.save() + + self.user_role_association(user.id) + + user_role = self.user_role_association( + {"user_id": user.id, "role_id": int(current_app.config["DEFAULT_ROLE"])} + ) + + user_role.save() + + return user class LoginEndpoint(LoginBaseEndpoint): diff --git a/cornflow-server/cornflow/endpoints/main_alarms.py b/cornflow-server/cornflow/endpoints/main_alarms.py index ce2a8e137..ea2f266fa 100644 --- a/cornflow-server/cornflow/endpoints/main_alarms.py +++ b/cornflow-server/cornflow/endpoints/main_alarms.py @@ -1,16 +1,15 @@ # Imports from libraries from flask_apispec import doc, marshal_with, use_kwargs -from cornflow_core.authentication import authenticate -from cornflow_core.resources import BaseMetaResource # Import from internal modules +from cornflow.endpoints.meta_resource import BaseMetaResource from cornflow.models import MainAlarmsModel from cornflow.schemas.main_alarms import ( MainAlarmsResponse, MainAlarmsPostRequest, QueryFiltersMainAlarms ) -from cornflow.shared.authentication import Auth +from cornflow.shared.authentication import Auth, authenticate class MainAlarmsEndpoint(BaseMetaResource): diff --git a/libs/core/cornflow_core/resources/meta_resource.py b/cornflow-server/cornflow/endpoints/meta_resource.py similarity index 96% rename from libs/core/cornflow_core/resources/meta_resource.py rename to cornflow-server/cornflow/endpoints/meta_resource.py index b28c7d88a..2fa194769 100644 --- a/libs/core/cornflow_core/resources/meta_resource.py +++ b/cornflow-server/cornflow/endpoints/meta_resource.py @@ -1,18 +1,16 @@ """ This file has all the logic shared for all the resources """ -# Import from python standard libraries -from functools import wraps -from pytups import SuperDict - # Import from external libraries from flask_restful import Resource from flask import g, request from flask_apispec.views import MethodResource +from functools import wraps +from pytups import SuperDict # Import from internal modules -from cornflow_core.constants import ALL_DEFAULT_ROLES -from cornflow_core.exceptions import InvalidUsage, ObjectDoesNotExist, NoPermission +from cornflow.shared.const import ALL_DEFAULT_ROLES +from cornflow.shared.exceptions import InvalidUsage, ObjectDoesNotExist, NoPermission class BaseMetaResource(Resource, MethodResource): @@ -88,8 +86,8 @@ def post_bulk(self, data, trace_field="user_id"): {**el, **{trace_field: self.get_user_id()}} for el in dict(data)["data"] ] - instances = self.data_model.create_bulk(data) - return instances, 201 + self.data_model.create_bulk(data) + return {"message": "Created correctly"}, 201 def post_bulk_update(self, data, trace_field="user_id"): """""" @@ -109,7 +107,7 @@ def post_bulk_update(self, data, trace_field="user_id"): instances.append(instance) self.data_model.create_update_bulk(instances) - return instances, 201 + return {"message": "Updated correctly"}, 201 def put_detail(self, data, track_user: bool = True, **kwargs): """ diff --git a/cornflow-server/cornflow/endpoints/permission.py b/cornflow-server/cornflow/endpoints/permission.py index 3b3406fc4..4d4ca1310 100644 --- a/cornflow-server/cornflow/endpoints/permission.py +++ b/cornflow-server/cornflow/endpoints/permission.py @@ -1,25 +1,22 @@ """ """ - -# Import from internal modules -from cornflow_core.authentication import authenticate -from cornflow_core.compress import compressed -from cornflow_core.exceptions import ObjectAlreadyExists -from cornflow_core.models import PermissionViewRoleBaseModel -from cornflow_core.resources import BaseMetaResource -from cornflow_core.schemas import ( - PermissionViewRoleBaseEditRequest, - PermissionViewRoleBaseRequest, - PermissionViewRoleBaseResponse, -) - # Import from libraries from flask_apispec import doc, marshal_with, use_kwargs from flask import current_app -from cornflow.shared.authentication import Auth +# Import from internal modules +from cornflow.endpoints.meta_resource import BaseMetaResource +from cornflow.models import PermissionViewRoleModel +from cornflow.schemas.permissions import ( + PermissionViewRoleEditRequest, + PermissionViewRoleRequest, + PermissionViewRoleResponse, +) +from cornflow.shared.authentication import Auth, authenticate +from cornflow.shared.compress import compressed from cornflow.shared.const import ADMIN_ROLE +from cornflow.shared.exceptions import ObjectAlreadyExists class PermissionsViewRoleEndpoint(BaseMetaResource): @@ -27,14 +24,14 @@ class PermissionsViewRoleEndpoint(BaseMetaResource): def __init__(self): super().__init__() - self.data_model = PermissionViewRoleBaseModel + self.data_model = PermissionViewRoleModel @doc( description="Get all the permissions assigned to the roles", tags=["PermissionViewRole"], ) @authenticate(auth_class=Auth()) - @marshal_with(PermissionViewRoleBaseResponse(many=True)) + @marshal_with(PermissionViewRoleResponse(many=True)) @compressed def get(self): current_app.logger.info( @@ -44,10 +41,10 @@ def get(self): @doc(description="Create a new permission", tags=["PermissionViewRole"]) @authenticate(auth_class=Auth()) - @use_kwargs(PermissionViewRoleBaseRequest, location="json") - @marshal_with(PermissionViewRoleBaseResponse) + @use_kwargs(PermissionViewRoleRequest, location="json") + @marshal_with(PermissionViewRoleResponse) def post(self, **kwargs): - if PermissionViewRoleBaseModel.get_permission( + if PermissionViewRoleModel.get_permission( role_id=kwargs.get("role_id"), api_view_id=kwargs.get("api_view_id"), action_id=kwargs.get("action_id"), @@ -66,11 +63,11 @@ class PermissionsViewRoleDetailEndpoint(BaseMetaResource): def __init__(self): super().__init__() - self.data_model = PermissionViewRoleBaseModel + self.data_model = PermissionViewRoleModel @doc(description="Get one permission", tags=["PermissionViewRole"]) @authenticate(auth_class=Auth()) - @marshal_with(PermissionViewRoleBaseResponse) + @marshal_with(PermissionViewRoleResponse) @BaseMetaResource.get_data_or_404 def get(self, idx): """ @@ -90,7 +87,7 @@ def get(self, idx): @doc(description="Edit a permission", tags=["PermissionViewRole"]) @authenticate(auth_class=Auth()) - @use_kwargs(PermissionViewRoleBaseEditRequest, location="json") + @use_kwargs(PermissionViewRoleEditRequest, location="json") def put(self, idx, **kwargs): response = self.put_detail(kwargs, idx=idx, track_user=False) current_app.logger.info(f"User {self.get_user()} edits permission {idx}") diff --git a/cornflow-server/cornflow/endpoints/roles.py b/cornflow-server/cornflow/endpoints/roles.py index e5e96f2f9..08de73eb5 100644 --- a/cornflow-server/cornflow/endpoints/roles.py +++ b/cornflow-server/cornflow/endpoints/roles.py @@ -2,23 +2,20 @@ Endpoints to manage the roles of the application and the assignation fo roles to users. Some of this endpoints are disable in case that the authentication is not performed over AUTH DB """ - -from cornflow_core.authentication import authenticate -from cornflow_core.exceptions import EndpointNotImplemented -from cornflow_core.resources import BaseMetaResource - # Import from libraries from flask import current_app from flask_apispec import doc, marshal_with, use_kwargs # Import from internal modules -from cornflow_core.models import RoleBaseModel -from cornflow_core.schemas import ( +from cornflow.endpoints.meta_resource import BaseMetaResource +from cornflow.models import RoleModel +from cornflow.schemas.role import ( RolesRequest, RolesResponse, ) -from cornflow.shared.authentication import Auth +from cornflow.shared.authentication import Auth, authenticate from cornflow.shared.const import ADMIN_ROLE, AUTH_LDAP +from cornflow.shared.exceptions import EndpointNotImplemented class RolesListEndpoint(BaseMetaResource): @@ -27,7 +24,7 @@ class RolesListEndpoint(BaseMetaResource): def __init__(self): super().__init__() - self.data_model = RoleBaseModel + self.data_model = RoleModel @doc(description="Gets all the roles", tags=["Roles"]) @authenticate(auth_class=Auth()) @@ -79,7 +76,7 @@ class RoleDetailEndpoint(BaseMetaResource): def __init__(self): super().__init__() - self.data_model = RoleBaseModel + self.data_model = RoleModel @doc(description="Gets one role", tags=["Roles"]) @authenticate(auth_class=Auth()) diff --git a/cornflow-server/cornflow/endpoints/schemas.py b/cornflow-server/cornflow/endpoints/schemas.py index eda001d46..8301e8987 100644 --- a/cornflow-server/cornflow/endpoints/schemas.py +++ b/cornflow-server/cornflow/endpoints/schemas.py @@ -3,17 +3,16 @@ """ # Import from libraries -from cornflow_core.authentication import authenticate -from cornflow_core.exceptions import NoPermission -from cornflow_core.resources import BaseMetaResource from flask import current_app, request from flask_apispec import marshal_with, doc # Import from internal modules +from cornflow.endpoints.meta_resource import BaseMetaResource from cornflow.models import PermissionsDAG, DeployedDAG from cornflow.schemas.schemas import SchemaOneApp, SchemaListApp -from cornflow.shared.authentication import Auth +from cornflow.shared.authentication import Auth, authenticate from cornflow.shared.const import ALL_DEFAULT_ROLES +from cornflow.shared.exceptions import NoPermission class SchemaEndpoint(BaseMetaResource): diff --git a/cornflow-server/cornflow/endpoints/signup.py b/cornflow-server/cornflow/endpoints/signup.py index 0a7d3de5c..b8720efc4 100644 --- a/cornflow-server/cornflow/endpoints/signup.py +++ b/cornflow-server/cornflow/endpoints/signup.py @@ -2,17 +2,23 @@ External endpoint for the user to signup """ # Import from libraries -from cornflow_core.resources import SignupBaseEndpoint from flask import current_app from flask_apispec import use_kwargs, doc # Import from internal modules +from cornflow.endpoints.meta_resource import BaseMetaResource from cornflow.models import PermissionsDAG, UserRoleModel, UserModel -from cornflow_core.schemas import SignupRequest +from cornflow.schemas.user import SignupRequest from cornflow.shared.authentication import Auth +from cornflow.shared.const import AUTH_LDAP, AUTH_OID +from cornflow.shared.exceptions import ( + EndpointNotImplemented, + InvalidCredentials, + InvalidUsage, +) -class SignUpEndpoint(SignupBaseEndpoint): +class SignUpEndpoint(BaseMetaResource): """ Endpoint used to sign up to the cornflow web server. """ @@ -39,3 +45,57 @@ def post(self, **kwargs): PermissionsDAG.add_all_permissions_to_user(content["id"]) return content, status + + def sign_up(self, **kwargs): + """ + The method in charge of performing the sign up of users + + :param kwargs: the keyword arguments needed to perform the sign up + :return: a dictionary with the newly issued token and the user id, and a status code + """ + auth_type = current_app.config["AUTH_TYPE"] + if auth_type == AUTH_LDAP: + err = "The user has to sign up on the active directory" + raise EndpointNotImplemented( + err, + log_txt="Error while user tries to sign up. " + err + ) + elif auth_type == AUTH_OID: + err = "The user has to sign up with the OpenID protocol" + raise EndpointNotImplemented( + err, + log_txt="Error while user tries to sign up. " + err + ) + + user = self.data_model(kwargs) + + if user.check_username_in_use(): + raise InvalidCredentials( + error="Username already in use, please supply another username", + log_txt="Error while user tries to sign up. Username already in use." + + ) + + if user.check_email_in_use(): + raise InvalidCredentials( + error="Email already in use, please supply another email address", + log_txt="Error while user tries to sign up. Email already in use." + ) + + user.save() + + user_role = self.user_role_association( + {"user_id": user.id, "role_id": current_app.config["DEFAULT_ROLE"]} + ) + + user_role.save() + + try: + token = self.auth_class.generate_token(user.id) + except Exception as e: + raise InvalidUsage( + error="Error in generating user token: " + str(e), status_code=400, + log_txt="Error while user tries to sign up. Unable to generate token." + ) + current_app.logger.info(f"New user created: {user}") + return {"token": token, "id": user.id}, 201 diff --git a/cornflow-server/cornflow/endpoints/tables.py b/cornflow-server/cornflow/endpoints/tables.py index df9b061c4..e20c89620 100644 --- a/cornflow-server/cornflow/endpoints/tables.py +++ b/cornflow-server/cornflow/endpoints/tables.py @@ -1,15 +1,14 @@ # Import from libraries -from cornflow_core.authentication import authenticate -from cornflow_core.exceptions import InvalidUsage, ObjectDoesNotExist -from cornflow_core.resources import BaseMetaResource from flask_apispec import doc, use_kwargs from flask import current_app # Import from internal modules +from cornflow.endpoints.meta_resource import BaseMetaResource +from cornflow.schemas.common import QueryFilters +from cornflow.shared.authentication import Auth, authenticate from cornflow.shared.const import SERVICE_ROLE -from cornflow.shared.authentication import Auth +from cornflow.shared.exceptions import InvalidUsage, ObjectDoesNotExist from cornflow.shared.utils_tables import get_all_tables, item_as_dict, items_as_dict_list -from cornflow.schemas.common import QueryFilters class TablesEndpoint(BaseMetaResource): diff --git a/cornflow-server/cornflow/endpoints/token.py b/cornflow-server/cornflow/endpoints/token.py index 113c4f838..1847c7e3d 100644 --- a/cornflow-server/cornflow/endpoints/token.py +++ b/cornflow-server/cornflow/endpoints/token.py @@ -1,13 +1,13 @@ # Import from libraries -from cornflow_core.resources import BaseMetaResource from flask import request from flask_apispec import marshal_with, doc # Import from internal modules +from cornflow.endpoints.meta_resource import BaseMetaResource from cornflow.schemas.user import TokenEndpointResponse -from cornflow_core.exceptions import InvalidCredentials, ObjectDoesNotExist from cornflow.shared.authentication import Auth from cornflow.shared.const import ALL_DEFAULT_ROLES +from cornflow.shared.exceptions import InvalidCredentials, ObjectDoesNotExist class TokenEndpoint(BaseMetaResource): diff --git a/cornflow-server/cornflow/endpoints/user.py b/cornflow-server/cornflow/endpoints/user.py index 6aebd5eb2..e347af7f5 100644 --- a/cornflow-server/cornflow/endpoints/user.py +++ b/cornflow-server/cornflow/endpoints/user.py @@ -1,27 +1,13 @@ """ Endpoints for the user profiles """ -# Full imports - -from cornflow_core.authentication import authenticate -from cornflow_core.exceptions import ( - EndpointNotImplemented, - InvalidCredentials, - InvalidUsage, - NoPermission, - ObjectDoesNotExist, -) - -# Partial imports from libraries -from cornflow_core.resources import BaseMetaResource -from cornflow_core.resources.recover_password import RecoverPasswordBaseEndpoint -from cornflow_core.shared import check_email_pattern, check_password_pattern -from cornflow_core.shared import db +# Imports from external libraries from flask import current_app from flask_apispec import marshal_with, use_kwargs, doc from sqlalchemy.exc import DBAPIError, IntegrityError -# Import from internal modules +# Imports from internal modules +from cornflow.endpoints.meta_resource import BaseMetaResource from cornflow.models import UserModel, UserRoleModel from cornflow.schemas.user import ( RecoverPasswordRequest, @@ -30,8 +16,19 @@ UserEndpointResponse, UserSchema, ) -from cornflow.shared.authentication import Auth +from cornflow.shared import db +from cornflow.shared.authentication import Auth, authenticate from cornflow.shared.const import ADMIN_ROLE, AUTH_LDAP, ALL_DEFAULT_ROLES, AUTH_OID +from cornflow.shared.exceptions import ( + ConfigurationError, + EndpointNotImplemented, + InvalidCredentials, + InvalidUsage, + NoPermission, + ObjectDoesNotExist, +) +from cornflow.shared.email import get_password_recover_email, send_email_to +from cornflow.shared.validators import check_email_pattern, check_password_pattern class UserEndpoint(BaseMetaResource): @@ -240,7 +237,7 @@ def put(self, user_id, make_admin): return user_obj, 200 -class RecoverPassword(RecoverPasswordBaseEndpoint): +class RecoverPassword(BaseMetaResource): """ Endpoint to recover the password """ @@ -261,9 +258,47 @@ def put(self, **kwargs): :rtype: Tuple(dict, integer) """ - response, status = self.recover_password(kwargs.get("email")) + sender = current_app.config["SERVICE_EMAIL_ADDRESS"] + password = current_app.config["SERVICE_EMAIL_PASSWORD"] + smtp_server = current_app.config["SERVICE_EMAIL_SERVER"] + port = current_app.config["SERVICE_EMAIL_PORT"] + service_name = current_app.config["SERVICE_NAME"] + receiver = kwargs.get("email") + + if sender is None or password is None or smtp_server is None or port is None: + raise ConfigurationError( + "This functionality is not available. Check that cornflow's email is correctly configured" + ) + + message = "The password recovery process has started. Check the email inbox." + + user_obj = self.data_model({"email": receiver}) + if not user_obj.check_email_in_use(): + return {"message": message}, 200 + + new_password = self.data_model.generate_random_password() + + text_email = get_password_recover_email( + temp_password=new_password, + service_name=service_name, + sender=sender, + receiver=receiver, + ) + + send_email_to( + email=text_email, + smtp_server=smtp_server, + port=port, + sender=sender, + password=password, + receiver=receiver, + ) + + data = {"password": new_password} + user_obj = self.data_model.get_one_user_by_email(receiver) + user_obj.update(data) current_app.logger.info( - f"User with email {kwargs.get('email')} has requested a new password" + f"User with email {receiver} has requested a new password" ) - return response, status + return {"message": message}, 200 diff --git a/cornflow-server/cornflow/endpoints/user_role.py b/cornflow-server/cornflow/endpoints/user_role.py index 40eec567f..b76efec1f 100644 --- a/cornflow-server/cornflow/endpoints/user_role.py +++ b/cornflow-server/cornflow/endpoints/user_role.py @@ -1,24 +1,19 @@ """ Endpoints to manage the roles of the application and the assignation fo roles to users. -Some of this endpoints are disable in case that the authentication is not performed over AUTH DB +Some of this endpoints are disabled in case that the authentication is not performed over AUTH DB """ -from cornflow_core.authentication import authenticate -from cornflow_core.exceptions import EndpointNotImplemented, ObjectAlreadyExists -from cornflow_core.resources import BaseMetaResource - # Import from libraries from flask import current_app from flask_apispec import doc, marshal_with, use_kwargs # Import from internal modules +from cornflow.endpoints.meta_resource import BaseMetaResource from cornflow.models import UserRoleModel -from cornflow_core.schemas import ( - UserRoleRequest, - UserRoleResponse, -) -from cornflow.shared.authentication import Auth +from cornflow.schemas.user_role import UserRoleRequest, UserRoleResponse +from cornflow.shared.authentication import Auth, authenticate from cornflow.shared.const import ADMIN_ROLE, AUTH_LDAP +from cornflow.shared.exceptions import EndpointNotImplemented, ObjectAlreadyExists class UserRoleListEndpoint(BaseMetaResource): diff --git a/cornflow-server/cornflow/models/__init__.py b/cornflow-server/cornflow/models/__init__.py index 70ef89056..d36c34ffa 100644 --- a/cornflow-server/cornflow/models/__init__.py +++ b/cornflow-server/cornflow/models/__init__.py @@ -1,13 +1,16 @@ """ Initialization file for the models module """ - +from .action import ActionModel +from .alarms import AlarmsModel from .case import CaseModel from .dag import DeployedDAG from .dag_permissions import PermissionsDAG from .execution import ExecutionModel from .instance import InstanceModel +from .main_alarms import MainAlarmsModel +from .permissions import PermissionViewRoleModel +from .role import RoleModel from .user import UserModel from .user_role import UserRoleModel -from .alarms import AlarmsModel -from .main_alarms import MainAlarmsModel +from .view import ViewModel diff --git a/libs/core/cornflow_core/models/action.py b/cornflow-server/cornflow/models/action.py similarity index 74% rename from libs/core/cornflow_core/models/action.py rename to cornflow-server/cornflow/models/action.py index 54988b681..7e867bfb6 100644 --- a/libs/core/cornflow_core/models/action.py +++ b/cornflow-server/cornflow/models/action.py @@ -1,11 +1,11 @@ """ This file contains the model that has the actions that can be performed on an REST API endpoint """ -from cornflow_core.models import EmptyBaseModel -from cornflow_core.shared import db +from cornflow.models.meta_models import EmptyBaseModel +from cornflow.shared import db -class ActionBaseModel(EmptyBaseModel): +class ActionModel(EmptyBaseModel): """ Model to store the actions that can be performed over the REST API: get, patch, post, put, delete. This model inherits from :class:`EmptyBaseModel` and does not have traceability @@ -21,11 +21,11 @@ class ActionBaseModel(EmptyBaseModel): name = db.Column(db.String(128), unique=True, nullable=False) permissions = db.relationship( - "PermissionViewRoleBaseModel", + "PermissionViewRoleModel", backref="actions", lazy=True, - primaryjoin="and_(ActionBaseModel.id==PermissionViewRoleBaseModel.action_id, " - "PermissionViewRoleBaseModel.deleted_at==None)", + primaryjoin="and_(ActionModel.id==PermissionViewRoleModel.action_id, " + "PermissionViewRoleModel.deleted_at==None)", cascade="all,delete", ) diff --git a/cornflow-server/cornflow/models/alarms.py b/cornflow-server/cornflow/models/alarms.py index b3eaec71f..26909b5fa 100644 --- a/cornflow-server/cornflow/models/alarms.py +++ b/cornflow-server/cornflow/models/alarms.py @@ -1,8 +1,8 @@ """ Model for the alarms """ # Import from internal modules -from cornflow_core.shared import db -from cornflow_core.models import TraceAttributesModel +from cornflow.shared import db +from cornflow.models.meta_models import TraceAttributesModel # Imports from external libraries from sqlalchemy.dialects.postgresql import TEXT diff --git a/cornflow-server/cornflow/models/base_data_model.py b/cornflow-server/cornflow/models/base_data_model.py index cc49235bc..086de155b 100644 --- a/cornflow-server/cornflow/models/base_data_model.py +++ b/cornflow-server/cornflow/models/base_data_model.py @@ -2,16 +2,15 @@ """ # Import from libraries -from cornflow_core.models import TraceAttributesModel - -# Import from internal modules -from cornflow_core.shared import db from flask import current_app from sqlalchemy import desc from sqlalchemy.dialects.postgresql import JSON from sqlalchemy.dialects.postgresql import TEXT from sqlalchemy.ext.declarative import declared_attr +# Import from internal modules +from cornflow.models.meta_models import TraceAttributesModel +from cornflow.shared import db from cornflow.shared.utils import hash_json_256 diff --git a/cornflow-server/cornflow/models/case.py b/cornflow-server/cornflow/models/case.py index f7245933b..94975d6f4 100644 --- a/cornflow-server/cornflow/models/case.py +++ b/cornflow-server/cornflow/models/case.py @@ -9,9 +9,9 @@ from sqlalchemy.exc import DBAPIError, IntegrityError # Import from internal modules -from .base_data_model import BaseDataModel -from cornflow_core.exceptions import InvalidPatch, ObjectDoesNotExist, InvalidData -from cornflow_core.shared import db +from cornflow.models.base_data_model import BaseDataModel +from cornflow.shared import db +from cornflow.shared.exceptions import InvalidPatch, ObjectDoesNotExist, InvalidData from cornflow.shared.utils import hash_json_256 diff --git a/cornflow-server/cornflow/models/dag.py b/cornflow-server/cornflow/models/dag.py index 021472dc7..a45aeaf3d 100644 --- a/cornflow-server/cornflow/models/dag.py +++ b/cornflow-server/cornflow/models/dag.py @@ -2,11 +2,6 @@ """ # Import from libraries -from sqlalchemy.dialects.postgresql import TEXT, JSON - -# Import from internal modules -from cornflow_core.models import TraceAttributesModel -from cornflow_core.shared import db from cornflow_client.airflow.api import Airflow from cornflow_client.constants import ( INSTANCE_SCHEMA, @@ -14,7 +9,12 @@ INSTANCE_CHECKS_SCHEMA, SOLUTION_CHECKS_SCHEMA ) -from cornflow_core.exceptions import ObjectDoesNotExist +from sqlalchemy.dialects.postgresql import TEXT, JSON + +# Import from internal modules +from cornflow.models.meta_models import TraceAttributesModel +from cornflow.shared import db +from cornflow.shared.exceptions import ObjectDoesNotExist class DeployedDAG(TraceAttributesModel): diff --git a/cornflow-server/cornflow/models/dag_permissions.py b/cornflow-server/cornflow/models/dag_permissions.py index 987310727..985447a75 100644 --- a/cornflow-server/cornflow/models/dag_permissions.py +++ b/cornflow-server/cornflow/models/dag_permissions.py @@ -1,8 +1,6 @@ -from cornflow_core.models import TraceAttributesModel -from cornflow_core.shared import db - -# from .meta_model import TraceAttributes -from .dag import DeployedDAG +from cornflow.models.dag import DeployedDAG +from cornflow.models.meta_models import TraceAttributesModel +from cornflow.shared import db class PermissionsDAG(TraceAttributesModel): diff --git a/cornflow-server/cornflow/models/execution.py b/cornflow-server/cornflow/models/execution.py index b2fa3fa39..680923ddf 100644 --- a/cornflow-server/cornflow/models/execution.py +++ b/cornflow-server/cornflow/models/execution.py @@ -4,14 +4,11 @@ # Import from libraries import hashlib - -# Imports from sqlalchemy -from sqlalchemy.dialects.postgresql import JSON -from sqlalchemy.dialects.postgresql import TEXT +from sqlalchemy.dialects.postgresql import JSON, TEXT # Imports from internal modules -from cornflow_core.shared import db -from .base_data_model import BaseDataModel +from cornflow.models.base_data_model import BaseDataModel +from cornflow.shared import db from cornflow.shared.const import DEFAULT_EXECUTION_CODE, EXECUTION_STATE_MESSAGE_DICT diff --git a/cornflow-server/cornflow/models/instance.py b/cornflow-server/cornflow/models/instance.py index 932d20b29..1551c4a16 100644 --- a/cornflow-server/cornflow/models/instance.py +++ b/cornflow-server/cornflow/models/instance.py @@ -4,8 +4,8 @@ import hashlib # Imported from internal models -from .base_data_model import BaseDataModel -from cornflow_core.shared import db +from cornflow.models.base_data_model import BaseDataModel +from cornflow.shared import db class InstanceModel(BaseDataModel): diff --git a/cornflow-server/cornflow/models/main_alarms.py b/cornflow-server/cornflow/models/main_alarms.py index 89adb13bd..070ed55bf 100644 --- a/cornflow-server/cornflow/models/main_alarms.py +++ b/cornflow-server/cornflow/models/main_alarms.py @@ -1,8 +1,8 @@ """ Model for the alarms """ # Import from internal modules -from cornflow_core.shared import db -from cornflow_core.models import TraceAttributesModel +from cornflow.shared import db +from cornflow.models.meta_models import TraceAttributesModel # Imports from external libraries diff --git a/libs/core/cornflow_core/models/meta_models.py b/cornflow-server/cornflow/models/meta_models.py similarity index 98% rename from libs/core/cornflow_core/models/meta_models.py rename to cornflow-server/cornflow/models/meta_models.py index 0a679d938..119cf7b2c 100644 --- a/libs/core/cornflow_core/models/meta_models.py +++ b/cornflow-server/cornflow/models/meta_models.py @@ -1,15 +1,15 @@ """ This file contains the base abstract models from which the rest of the models inherit """ +# Imports from libraries from datetime import datetime -from typing import Dict, List - -from sqlalchemy.exc import DBAPIError, IntegrityError -from sqlalchemy import desc from flask import current_app +from sqlalchemy.exc import DBAPIError, IntegrityError +from typing import Dict, List -from cornflow_core.exceptions import InvalidData -from cornflow_core.shared import db +# Imports from internal modules +from cornflow.shared import db +from cornflow.shared.exceptions import InvalidData class EmptyBaseModel(db.Model): diff --git a/libs/core/cornflow_core/models/permissions.py b/cornflow-server/cornflow/models/permissions.py similarity index 83% rename from libs/core/cornflow_core/models/permissions.py rename to cornflow-server/cornflow/models/permissions.py index 1789bdb22..cec790c92 100644 --- a/libs/core/cornflow_core/models/permissions.py +++ b/cornflow-server/cornflow/models/permissions.py @@ -1,17 +1,17 @@ """ -This file contains the PermissionViewRoleBaseModel +This file contains the PermissionViewRoleModel """ +# Imports from internal modules +from cornflow.models.meta_models import TraceAttributesModel +from cornflow.shared import db -from cornflow_core.models import TraceAttributesModel -from cornflow_core.shared import db - -class PermissionViewRoleBaseModel(TraceAttributesModel): +class PermissionViewRoleModel(TraceAttributesModel): """ This model has the permissions that can be defined between an action, a view and a role It inherits from :class:`TraceAttributesModel` to have trace fields - The :class:`PermissionViewRoleBaseModel` has the following fields: + The :class:`PermissionViewRoleModel` has the following fields: - **id**: int, the primary key of the table, an integer value that is auto incremented - **action_id**: the id of the action @@ -34,13 +34,13 @@ class PermissionViewRoleBaseModel(TraceAttributesModel): id = db.Column(db.Integer, primary_key=True, autoincrement=True) action_id = db.Column(db.Integer, db.ForeignKey("actions.id"), nullable=False) - action = db.relationship("ActionBaseModel", viewonly=True) + action = db.relationship("ActionModel", viewonly=True) api_view_id = db.Column(db.Integer, db.ForeignKey("api_view.id"), nullable=False) - api_view = db.relationship("ViewBaseModel", viewonly=True) + api_view = db.relationship("ViewModel", viewonly=True) role_id = db.Column(db.Integer, db.ForeignKey("roles.id"), nullable=False) - role = db.relationship("RoleBaseModel", viewonly=True) + role = db.relationship("RoleModel", viewonly=True) def __init__(self, data): super().__init__() diff --git a/libs/core/cornflow_core/models/role.py b/cornflow-server/cornflow/models/role.py similarity index 73% rename from libs/core/cornflow_core/models/role.py rename to cornflow-server/cornflow/models/role.py index 332f4831d..ac7f8499e 100644 --- a/libs/core/cornflow_core/models/role.py +++ b/cornflow-server/cornflow/models/role.py @@ -1,17 +1,17 @@ """ -This file contains the RoleBaseModel +This file contains the RoleModel """ +# Imports from internal modules +from cornflow.models.meta_models import TraceAttributesModel +from cornflow.shared import db -from cornflow_core.models import TraceAttributesModel -from cornflow_core.shared import db - -class RoleBaseModel(TraceAttributesModel): +class RoleModel(TraceAttributesModel): """ This model has the roles that are defined on the REST API It inherits from :class:`TraceAttributesModel` to have trace fields - The :class:`RoleBaseModel` has the following fields: + The :class:`RoleModel` has the following fields: - **id**: int, the primary key of the table, an integer value that is auto incremented - **name**: str, the name of the role @@ -29,20 +29,20 @@ class RoleBaseModel(TraceAttributesModel): name = db.Column(db.String(128), nullable=False) user_roles = db.relationship( - "UserRoleBaseModel", + "UserRoleModel", backref="roles", lazy=True, - primaryjoin="and_(RoleBaseModel.id==UserRoleBaseModel.role_id, " - "UserRoleBaseModel.deleted_at==None)", + primaryjoin="and_(RoleModel.id==UserRoleModel.role_id, " + "UserRoleModel.deleted_at==None)", cascade="all,delete", ) permissions = db.relationship( - "PermissionViewRoleBaseModel", + "PermissionViewRoleModel", backref="roles", lazy=True, - primaryjoin="and_(RoleBaseModel.id==PermissionViewRoleBaseModel.role_id, " - "PermissionViewRoleBaseModel.deleted_at==None)", + primaryjoin="and_(RoleModel.id==PermissionViewRoleModel.role_id, " + "PermissionViewRoleModel.deleted_at==None)", cascade="all,delete", ) diff --git a/cornflow-server/cornflow/models/user.py b/cornflow-server/cornflow/models/user.py index 99f5061bf..4b1f185a1 100644 --- a/cornflow-server/cornflow/models/user.py +++ b/cornflow-server/cornflow/models/user.py @@ -1,10 +1,25 @@ -from cornflow_core.models import UserBaseModel -from cornflow_core.shared import db +""" +This file contains the UserModel +""" +# Imports from external libraries +import random +import string -from .user_role import UserRoleModel +# Imports from internal modules +from cornflow.models.meta_models import TraceAttributesModel +from cornflow.models.user_role import UserRoleModel +from cornflow.shared import ( + bcrypt, + db, +) +from cornflow.shared.exceptions import InvalidCredentials +from cornflow.shared.validators import ( + check_password_pattern, + check_email_pattern, +) -class UserModel(UserBaseModel): +class UserModel(TraceAttributesModel): """ Model class for the Users. It inherits from :class:`TraceAttributes` to have trace fields. @@ -31,7 +46,16 @@ class UserModel(UserBaseModel): """ __tablename__ = "users" - __table_args__ = {"extend_existing": True} + id = db.Column(db.Integer, primary_key=True, autoincrement=True) + first_name = db.Column(db.String(128), nullable=True) + last_name = db.Column(db.String(128), nullable=True) + username = db.Column(db.String(128), nullable=False, unique=True) + password = db.Column(db.String(128), nullable=True) + email = db.Column(db.String(128), nullable=False, unique=True) + + user_roles = db.relationship( + "UserRoleModel", cascade="all,delete", backref="users" + ) instances = db.relationship( "InstanceModel", @@ -57,6 +81,161 @@ class UserModel(UserBaseModel): primaryjoin="and_(UserModel.id==PermissionsDAG.user_id)", ) + @property + def roles(self): + """ + This property gives back the roles assigned to the user + """ + return {r.role.id: r.role.name for r in self.user_roles} + + def __init__(self, data): + super().__init__() + self.first_name = data.get("first_name") + self.last_name = data.get("last_name") + self.username = data.get("username") + # TODO: handle better None passwords that can be found when using ldap + check_pass, msg = check_password_pattern(data.get("password")) + if check_pass: + self.password = self.__generate_hash(data.get("password")) + else: + raise InvalidCredentials( + msg, + log_txt="Error while trying to create a new user. " + msg + ) + + check_email, msg = check_email_pattern(data.get("email")) + if check_email: + self.email = data.get("email") + else: + raise InvalidCredentials( + msg, + log_txt="Error while trying to create a new user. " + msg + ) + + def update(self, data): + """ + Updates the user information in the database + + :param dict data: the data to update the user + """ + # First we create the hash of the new password and then we update the object + new_password = data.get("password") + if new_password: + new_password = self.__generate_hash(new_password) + data["password"] = new_password + super().update(data) + + def comes_from_external_provider(self): + """ + Returns a boolean if the user comes from an external_provider or not + """ + return self.password is None + + @staticmethod + def __generate_hash(password): + """ + Method to generate the hash from the password. + + :param str password: the password given by the user . + :return: the hashed password. + :rtype: str + """ + if password is None: + return None + return bcrypt.generate_password_hash(password, rounds=10).decode("utf8") + + def check_hash(self, password): + """ + Method to check if the hash stored in the database is the same as the password given by the user + + :param str password: the password given by the user. + :return: if the password is the same or not. + :rtype: bool + """ + return bcrypt.check_password_hash(self.password, password) + + @classmethod + def get_all_users(cls): + """ + Query to get all users + + :return: a list with all the users. + :rtype: list(:class:`UserModel`) + """ + return cls.get_all_objects() + + @classmethod + def get_one_user(cls, idx): + """ + Query to get the information of one user + + :param int idx: ID of the user + :return: the user object + :rtype: :class:`UserModel` + """ + return cls.get_one_object(idx=idx) + + @classmethod + def get_one_user_by_email(cls, email): + """ + Query to get one user from the email + + :param str email: User email + :return: the user object + :rtype: :class:`UserModel` + """ + return cls.get_one_object(email=email) + + @classmethod + def get_one_user_by_username(cls, username): + """ + Returns one user (object) given a username + + :param str username: the user username that we want to query for + :return: the user object + :rtype: :class:`UserModel` + """ + return cls.get_one_object(username=username) + + def check_username_in_use(self): + """ + Checks if a username is already in use + + :return: a boolean if the username is in use + :rtype: bool + """ + return self.query.filter_by(username=self.username).first() is not None + + def check_email_in_use(self): + """ + Checks if a email is already in use + + :return: a boolean if the username is in use + :rtype: bool + """ + return self.query.filter_by(email=self.email).first() is not None + + @staticmethod + def generate_random_password() -> str: + """ + Method to generate a new random password for the user + + :return: the newly generated password + :rtype: str + """ + nb_lower = random.randint(1, 9) + nb_upper = random.randint(10 - nb_lower, 11) + nb_numbers = random.randint(1, 3) + nb_special_char = random.randint(1, 3) + upper_letters = random.sample(string.ascii_uppercase, nb_upper) + lower_letters = random.sample(string.ascii_lowercase, nb_lower) + numbers = random.sample(list(map(str, list(range(10)))), nb_numbers) + symbols = random.sample("!ยก?ยฟ#$%&'()*+-_./:;,<>=@[]^`{}|~\"\\", nb_special_char) + chars = upper_letters + lower_letters + numbers + symbols + random.shuffle(chars) + pwd = "".join(chars) + return pwd + def is_admin(self): """ Returns a boolean if a user is an admin or not @@ -68,3 +247,15 @@ def is_service_user(self): Returns a boolean if a user is a service user or not """ return UserRoleModel.is_service_user(self.id) + + def __repr__(self): + """ + Representation method of the class + + :return: the representation of the class + :rtype: str + """ + return "".format(self.username) + + def __str__(self): + return self.__repr__() diff --git a/cornflow-server/cornflow/models/user_role.py b/cornflow-server/cornflow/models/user_role.py index 613291c07..a721be779 100644 --- a/cornflow-server/cornflow/models/user_role.py +++ b/cornflow-server/cornflow/models/user_role.py @@ -1,23 +1,89 @@ """ -a +Model for the relationship between users and roles """ -from cornflow_core.models import UserRoleBaseModel -from cornflow_core.shared import db - +from cornflow.models.meta_models import TraceAttributesModel +from cornflow.shared import db from cornflow.shared.const import ADMIN_ROLE, SERVICE_ROLE -class UserRoleModel(UserRoleBaseModel): +class UserRoleModel(TraceAttributesModel): """ Model for the relationship between user and roles + It inherits from :class:`TraceAttributesModel` to have trace fields + + The :class:`UserRoleModel` has the following fields: + + - **id**: int, the primary key of the assignation, an integer value that is auto incremented + - **user_id**: the id of the user. + - **role_id**: the id of the assigned role. + - **created_at**: datetime, the datetime when the user was created (in UTC). + This datetime is generated automatically, the user does not need to provide it. + - **updated_at**: datetime, the datetime when the user was last updated (in UTC). + This datetime is generated automatically, the user does not need to provide it. + - **deleted_at**: datetime, the datetime when the user was deleted (in UTC). + This field is used only if we deactivate instead of deleting the record. + This datetime is generated automatically, the user does not need to provide it. """ # TODO: Should have a user_id to store the user that defined the assignation? __tablename__ = "user_role" - __table_args__ = ({"extend_existing": True},) + __table_args__ = (db.UniqueConstraint("user_id", "role_id"),) + + id = db.Column(db.Integer, primary_key=True, autoincrement=True) + + user_id = db.Column(db.Integer, db.ForeignKey("users.id"), nullable=False) + user = db.relationship("UserModel", viewonly=True, lazy=False) + + role_id = db.Column(db.Integer, db.ForeignKey("roles.id"), nullable=False) + role = db.relationship("RoleModel", viewonly=True, lazy=False) + + def __init__(self, data): + """ + Method to initialize th assignation of a role to a user that + + :param dict data: dict with the information needed to create a new assignation of a role to a user + """ + super().__init__() + self.user_id = data.get("user_id") + self.role_id = data.get("role_id") + + @classmethod + def check_if_role_assigned(cls, user_id, role_id): + """ + Method to check if a user has a given role assigned + + :param int user_id: id of the specific user + :param int role_id: id of the specific role + :return: a boolean if the user has the role assigned + :rtype: bool + """ + return cls.get_one_object(user_id=user_id, role_id=role_id) is not None + + @classmethod + def check_if_role_assigned_disabled(cls, user_id, role_id): + """ + Method to check if a user has a given role assigned but disabled - user = db.relationship("UserBaseModel", viewonly=True, lazy=False) - role = db.relationship("RoleBaseModel", viewonly=True, lazy=False) + :param user_id: id of the specific user + :param role_id: id of the specific role + :return: a boolean if the user has the role assigned but disabled + :rtype: bool + """ + user_role = cls.query.filter( + cls.user_id == user_id, cls.role_id == role_id, cls.deleted_at != None + ).first() + return user_role is not None + + @classmethod + def del_one_user(cls, user_id): + """ + Method to delete all the assigned roles to one user + + :param int user_id: the ID of the user + :return: a list with all the deleted objects. + :rtype: list + """ + return cls.query.filter_by(user_id=user_id).delete(synchronize_session=False) @classmethod def is_admin(cls, user_id): @@ -50,3 +116,25 @@ def is_service_user(cls, user_id): return True return False + + def __repr__(self): + """ + Method for the representation of the assigned roles + + :return: the representation + :rtype: str + """ + try: + return f"{self.user.username} has role {self.role.name}" + except AttributeError: + return f"{self.user_id} has role {self.role_id}" + + def __str__(self): + """ + Method for the string representation of the assigned roles + + :return: the string representation + :rtype: str + """ + return self.__repr__() + diff --git a/libs/core/cornflow_core/models/view.py b/cornflow-server/cornflow/models/view.py similarity index 80% rename from libs/core/cornflow_core/models/view.py rename to cornflow-server/cornflow/models/view.py index 31caca3d0..1bb2301ef 100644 --- a/libs/core/cornflow_core/models/view.py +++ b/cornflow-server/cornflow/models/view.py @@ -1,14 +1,15 @@ """ This file contains the view model """ - +# Imports from libraries from sqlalchemy.dialects.postgresql import TEXT -from cornflow_core.models import EmptyBaseModel -from cornflow_core.shared import db +# Imports from internal modules +from cornflow.models.meta_models import EmptyBaseModel +from cornflow.shared import db -class ViewBaseModel(EmptyBaseModel): +class ViewModel(EmptyBaseModel): """ This model stores the views / endpoints / resources of the API This model inherits from :class:`EmptyBaseModel` so it has no traceability @@ -28,11 +29,11 @@ class ViewBaseModel(EmptyBaseModel): description = db.Column(TEXT, nullable=True) permissions = db.relationship( - "PermissionViewRoleBaseModel", + "PermissionViewRoleModel", backref="api_views", lazy=True, - primaryjoin="and_(ViewBaseModel.id==PermissionViewRoleBaseModel.api_view_id, " - "PermissionViewRoleBaseModel.deleted_at==None)", + primaryjoin="and_(ViewModel.id==PermissionViewRoleModel.api_view_id, " + "PermissionViewRoleModel.deleted_at==None)", cascade="all,delete", ) @@ -57,7 +58,7 @@ def get_one_by_name(cls, name: str): This methods queries the model to search for a view with a given name. :param str name: The name that the view has - :return: The found result, either an object :class:`ViewBaseModel` or None - :rtype: None or :class:`ViewBaseModel` + :return: The found result, either an object :class:`ViewModel` or None + :rtype: None or :class:`ViewModel` """ return cls.query.filter_by(name=name).first() diff --git a/cornflow-server/cornflow/schemas/__init__.py b/cornflow-server/cornflow/schemas/__init__.py index edcd154dd..8ba1690d0 100644 --- a/cornflow-server/cornflow/schemas/__init__.py +++ b/cornflow-server/cornflow/schemas/__init__.py @@ -6,6 +6,3 @@ from .execution import ExecutionSchema from .instance import InstanceSchema from .user import UserSchema - - -# from .model_json import DataSchema diff --git a/libs/core/cornflow_core/schemas/action.py b/cornflow-server/cornflow/schemas/action.py similarity index 100% rename from libs/core/cornflow_core/schemas/action.py rename to cornflow-server/cornflow/schemas/action.py diff --git a/cornflow-server/cornflow/schemas/case.py b/cornflow-server/cornflow/schemas/case.py index 07300b582..84068f4b0 100644 --- a/cornflow-server/cornflow/schemas/case.py +++ b/cornflow-server/cornflow/schemas/case.py @@ -3,13 +3,12 @@ and to serialize the response data given by the same endpoints. """ -from cornflow_core.schemas import BasePatchOperation - # Imports from marshmallow library from marshmallow import fields, Schema # Import from internal modules from .common import BaseDataEndpointResponse, QueryFilters +from .patch import BasePatchOperation class CaseRawRequest(Schema): diff --git a/cornflow-server/cornflow/schemas/common.py b/cornflow-server/cornflow/schemas/common.py index 7953b23fd..35fa8f25f 100644 --- a/cornflow-server/cornflow/schemas/common.py +++ b/cornflow-server/cornflow/schemas/common.py @@ -2,7 +2,7 @@ File with the common schemas used in cornflow """ from marshmallow import fields, Schema -from cornflow_core.schemas import BaseQueryFilters +from .query import BaseQueryFilters class QueryFilters(BaseQueryFilters): diff --git a/cornflow-server/cornflow/schemas/execution.py b/cornflow-server/cornflow/schemas/execution.py index 872dbc137..cd48c5776 100644 --- a/cornflow-server/cornflow/schemas/execution.py +++ b/cornflow-server/cornflow/schemas/execution.py @@ -1,7 +1,10 @@ +# Imports from libraries from marshmallow import fields, Schema, validate -from .solution_log import LogSchema + +# Imports from internal modules from cornflow.shared.const import MIN_EXECUTION_STATUS_CODE, MAX_EXECUTION_STATUS_CODE from .common import QueryFilters, BaseDataEndpointResponse +from .solution_log import LogSchema class QueryFiltersExecution(QueryFilters): diff --git a/libs/core/cornflow_core/schemas/patch.py b/cornflow-server/cornflow/schemas/patch.py similarity index 100% rename from libs/core/cornflow_core/schemas/patch.py rename to cornflow-server/cornflow/schemas/patch.py diff --git a/libs/core/cornflow_core/schemas/permissions.py b/cornflow-server/cornflow/schemas/permissions.py similarity index 89% rename from libs/core/cornflow_core/schemas/permissions.py rename to cornflow-server/cornflow/schemas/permissions.py index 9f2f8d0ac..21f72e1dd 100644 --- a/libs/core/cornflow_core/schemas/permissions.py +++ b/cornflow-server/cornflow/schemas/permissions.py @@ -6,7 +6,7 @@ from marshmallow import fields, Schema -class PermissionViewRoleBaseRequest(Schema): +class PermissionViewRoleRequest(Schema): """ Schema used for the permissions """ @@ -16,7 +16,7 @@ class PermissionViewRoleBaseRequest(Schema): api_view_id = fields.Int() -class PermissionViewRoleBaseResponse(Schema): +class PermissionViewRoleResponse(Schema): """ Schema used for the get methods """ @@ -30,7 +30,7 @@ class PermissionViewRoleBaseResponse(Schema): role = fields.Function(lambda obj: obj.role.name) -class PermissionViewRoleBaseEditRequest(Schema): +class PermissionViewRoleEditRequest(Schema): """ Schema used for the edition request of the permissions """ diff --git a/libs/core/cornflow_core/schemas/query.py b/cornflow-server/cornflow/schemas/query.py similarity index 100% rename from libs/core/cornflow_core/schemas/query.py rename to cornflow-server/cornflow/schemas/query.py diff --git a/libs/core/cornflow_core/schemas/role.py b/cornflow-server/cornflow/schemas/role.py similarity index 100% rename from libs/core/cornflow_core/schemas/role.py rename to cornflow-server/cornflow/schemas/role.py diff --git a/cornflow-server/cornflow/schemas/user.py b/cornflow-server/cornflow/schemas/user.py index b4fe72814..8a4f568a5 100644 --- a/cornflow-server/cornflow/schemas/user.py +++ b/cornflow-server/cornflow/schemas/user.py @@ -1,12 +1,20 @@ -from cornflow_core.schemas import BaseUserSchema +""" +This file contains the schemas used for the users defined in the application +""" from marshmallow import fields, Schema - from .instance import InstanceSchema -class UserSchema(BaseUserSchema): +class UserSchema(Schema): """ """ - + id = fields.Int(dump_only=True) + first_name = fields.Str() + last_name = fields.Str() + username = fields.Str(required=True) + email = fields.Email(required=True) + password = fields.Str(required=True, load_only=True) + created_at = fields.DateTime(dump_only=True) + modified_at = fields.DateTime(dump_only=True) instances = fields.Nested(InstanceSchema, many=True) @@ -41,3 +49,32 @@ class UserEditRequest(Schema): last_name = fields.Str(required=False) email = fields.Str(required=False) password = fields.Str(required=False) + + +class LoginEndpointRequest(Schema): + """ + This is the schema used by the login endpoint with auth db or ldap + """ + + username = fields.Str(required=True) + password = fields.Str(required=True) + + +class LoginOpenAuthRequest(Schema): + """ + This is the schema used by the login endpoint with Open ID protocol + """ + + token = fields.Str(required=True) + + +class SignupRequest(Schema): + """ + This is the schema used by the sign up + """ + + username = fields.Str(required=True) + email = fields.Email(required=True) + password = fields.Str(required=True, load_only=True) + first_name = fields.Str(required=False) + last_name = fields.Str(required=False) diff --git a/libs/core/cornflow_core/schemas/user_role.py b/cornflow-server/cornflow/schemas/user_role.py similarity index 100% rename from libs/core/cornflow_core/schemas/user_role.py rename to cornflow-server/cornflow/schemas/user_role.py diff --git a/libs/core/cornflow_core/schemas/view.py b/cornflow-server/cornflow/schemas/view.py similarity index 100% rename from libs/core/cornflow_core/schemas/view.py rename to cornflow-server/cornflow/schemas/view.py diff --git a/cornflow-server/cornflow/shared/__init__.py b/cornflow-server/cornflow/shared/__init__.py index e69de29bb..1c8153a97 100644 --- a/cornflow-server/cornflow/shared/__init__.py +++ b/cornflow-server/cornflow/shared/__init__.py @@ -0,0 +1 @@ +from .utils import db, bcrypt \ No newline at end of file diff --git a/cornflow-server/cornflow/shared/authentication.py b/cornflow-server/cornflow/shared/authentication.py deleted file mode 100644 index 16d17dcd2..000000000 --- a/cornflow-server/cornflow/shared/authentication.py +++ /dev/null @@ -1,121 +0,0 @@ -""" - -""" - -# Global imports -from functools import wraps - -from cornflow_core.authentication import BaseAuth -from cornflow_core.exceptions import InvalidData, NoPermission -from cornflow_core.models import ViewBaseModel, PermissionViewRoleBaseModel - -# Partial imports -from flask import request, g, current_app - -# Internal modules imports -from .const import PERMISSION_METHOD_MAP -from cornflow.models import UserModel, PermissionsDAG - - -class Auth(BaseAuth): - def __init__(self, user_model=UserModel): - super().__init__(user_model) - - def authenticate(self): - user = self.get_user_from_header(request.headers) - check = Auth._get_permission_for_request(request, user.id) - g.user = user - return True - - @staticmethod - def dag_permission_required(func): - """ - DAG permission decorator - :param func: - :return: - """ - - @wraps(func) - def dag_decorator(*args, **kwargs): - if int(current_app.config["OPEN_DEPLOYMENT"]) == 0: - user_id = g.user.id - dag_id = request.json.get("schema", None) - if dag_id is None: - raise InvalidData( - error="The request does not specify a schema to use", - status_code=400, - log_txt=f"Error while user {g.user} tries to access a dag. " - f"The schema is not specified in the request.", - ) - else: - if PermissionsDAG.check_if_has_permissions(user_id, dag_id): - # We have permissions - return func(*args, **kwargs) - else: - raise NoPermission( - error="You do not have permission to use this DAG", - status_code=403, - log_txt=f"Error while user {g.user} tries to access dag {dag_id}. " - f"The user does not have permission to access the dag.", - ) - else: - return func(*args, **kwargs) - - return dag_decorator - - @staticmethod - def return_user_from_token(token): - """ - Function used for internal testing. Given a token gives back the user_id encoded in it. - - :param str token: the given token - :return: the user id code. - :rtype: int - """ - user_id = Auth.decode_token(token)["user_id"] - return user_id - - """ - START OF INTERNAL PROTECTED METHODS - """ - - @staticmethod - def _get_permission_for_request(req, user_id): - method, url = Auth._get_request_info(req) - user_roles = UserModel.get_one_user(user_id).roles - if user_roles is None or user_roles == {}: - raise NoPermission( - error="You do not have permission to access this endpoint", - status_code=403, - log_txt=f"Error while user {user_id} tries to access an endpoint. " - f"The user does not have any role assigned. ", - ) - - action_id = PERMISSION_METHOD_MAP[method] - try: - view_id = ViewBaseModel.query.filter_by(url_rule=url).first().id - except AttributeError: - current_app.logger.error( - "The permission for this endpoint is not in the database." - ) - raise NoPermission( - error="You do not have permission to access this endpoint", - status_code=403, - log_txt=f"Error while user {user_id} tries to access endpoint. " - f"The user does not have permission to access. ", - ) - - for role in user_roles: - has_permission = PermissionViewRoleBaseModel.get_permission( - role_id=role, api_view_id=view_id, action_id=action_id - ) - - if has_permission: - return True - - raise NoPermission( - error="You do not have permission to access this endpoint", - status_code=403, - log_txt=f"Error while user {user_id} tries to access endpoint {view_id} with action {action_id}. " - f"The user does not permission to access. ", - ) diff --git a/libs/core/cornflow_core/authentication/__init__.py b/cornflow-server/cornflow/shared/authentication/__init__.py similarity index 80% rename from libs/core/cornflow_core/authentication/__init__.py rename to cornflow-server/cornflow/shared/authentication/__init__.py index d3becfa91..954875461 100644 --- a/libs/core/cornflow_core/authentication/__init__.py +++ b/cornflow-server/cornflow/shared/authentication/__init__.py @@ -1,6 +1,6 @@ """ Exposes the different classes and methods """ -from .auth import BaseAuth +from .auth import Auth from .decorators import authenticate from .ldap import LDAPBase diff --git a/libs/core/cornflow_core/authentication/auth.py b/cornflow-server/cornflow/shared/authentication/auth.py similarity index 74% rename from libs/core/cornflow_core/authentication/auth.py rename to cornflow-server/cornflow/shared/authentication/auth.py index 86cb9ea04..584041f0e 100644 --- a/libs/core/cornflow_core/authentication/auth.py +++ b/cornflow-server/cornflow/shared/authentication/auth.py @@ -1,49 +1,92 @@ """ -This file contains the basic auth class that can be used for authentication on the request to the REST API +This file contains the auth class that can be used for authentication on the request to the REST API """ -# Imports from python base libraries -import base64 -from datetime import datetime, timedelta -from typing import Union, Tuple -# Import from external libraries +# Imports from external libraries +import base64 import jwt import requests + from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric.rsa import RSAPublicNumbers -from flask import current_app, g, request, Request -from jwt import decode, encode, ExpiredSignatureError, InvalidTokenError +from datetime import datetime, timedelta +from flask import request, g, current_app, Request +from functools import wraps +from typing import Union, Tuple from werkzeug.datastructures import Headers # Imports from internal modules -from cornflow_core.constants import ( +from cornflow.models import ( + PermissionsDAG, + PermissionViewRoleModel, + UserModel, + ViewModel, +) +from cornflow.shared.const import ( OID_AZURE, OID_AZURE_DISCOVERY_TENANT_URL, OID_AZURE_DISCOVERY_COMMON_URL, OID_GOOGLE, + PERMISSION_METHOD_MAP, ) -from cornflow_core.exceptions import ( +from cornflow.shared.exceptions import ( + CommunicationError, + EndpointNotImplemented, InvalidCredentials, + InvalidData, InvalidUsage, + NoPermission, ObjectDoesNotExist, - EndpointNotImplemented, - CommunicationError, ) -from cornflow_core.models import UserBaseModel - -class BaseAuth: - """ - This class implements the basic auth class, with token generation and decoding, oid token validation - and an authenticate method. - """ - def __init__(self, user_model=None): - if user_model is None: - self.user_model = UserBaseModel +class Auth: + def __init__(self, user_model=UserModel): self.user_model = user_model + def authenticate(self): + user = self.get_user_from_header(request.headers) + Auth._get_permission_for_request(request, user.id) + g.user = user + return True + + @staticmethod + def dag_permission_required(func): + """ + DAG permission decorator + :param func: + :return: + """ + + @wraps(func) + def dag_decorator(*args, **kwargs): + if int(current_app.config["OPEN_DEPLOYMENT"]) == 0: + user_id = g.user.id + dag_id = request.json.get("schema", None) + if dag_id is None: + raise InvalidData( + error="The request does not specify a schema to use", + status_code=400, + log_txt=f"Error while user {g.user} tries to access a dag. " + f"The schema is not specified in the request.", + ) + else: + if PermissionsDAG.check_if_has_permissions(user_id, dag_id): + # We have permissions + return func(*args, **kwargs) + else: + raise NoPermission( + error="You do not have permission to use this DAG", + status_code=403, + log_txt=f"Error while user {g.user} tries to access dag {dag_id}. " + f"The user does not have permission to access the dag.", + ) + else: + return func(*args, **kwargs) + + return dag_decorator + @staticmethod def generate_token(user_id: int = None) -> str: """ @@ -66,7 +109,7 @@ def generate_token(user_id: int = None) -> str: "sub": user_id, } - return encode(payload, current_app.config["SECRET_KEY"], algorithm="HS256") + return jwt.encode(payload, current_app.config["SECRET_KEY"], algorithm="HS256") @staticmethod def decode_token(token: str = None) -> dict: @@ -84,23 +127,23 @@ def decode_token(token: str = None) -> dict: log_txt="Error while trying to decode token. " + err ) try: - payload = decode( + payload = jwt.decode( token, current_app.config["SECRET_KEY"], algorithms="HS256" ) return {"user_id": payload["sub"]} - except ExpiredSignatureError: + except jwt.ExpiredSignatureError: raise InvalidCredentials( "The token has expired, please login again", log_txt="Error while trying to decode token. The token has expired." ) - except InvalidTokenError: + except jwt.InvalidTokenError: raise InvalidCredentials( "Invalid token, please try again with a new token", log_txt="Error while trying to decode token. The token is invalid." ) def validate_oid_token( - self, token: str, client_id: str, tenant_id: str, issuer: str, provider: int + self, token: str, client_id: str, tenant_id: str, issuer: str, provider: int ) -> dict: """ This method takes a token issued by an OID provider, the relevant information about the OID provider @@ -167,7 +210,7 @@ def get_token_from_header(headers: Headers = None) -> str: log_txt=f"Error while trying to get a token from header. " + err ) - def get_user_from_header(self, headers: Headers = None) -> UserBaseModel: + def get_user_from_header(self, headers: Headers = None) -> UserModel: """ Gets the user represented by the token that has to be in the request headers. @@ -194,22 +237,63 @@ def get_user_from_header(self, headers: Headers = None) -> UserBaseModel: ) return user - def authenticate(self): + @staticmethod + def return_user_from_token(token): """ - Main method to perform the authentication of the user on the REST API. + Function used for internal testing. Given a token gives back the user_id encoded in it. - :return: True if the authentication was successful, it raises an exception in case the authentication failed - or the was an error - :rtype: bool + :param str token: the given token + :return: the user id code. + :rtype: int """ - user = self.get_user_from_header(request.headers) - g.user = user - return True + user_id = Auth.decode_token(token)["user_id"] + return user_id """ START OF INTERNAL PROTECTED METHODS """ + @staticmethod + def _get_permission_for_request(req, user_id): + method, url = Auth._get_request_info(req) + user_roles = UserModel.get_one_user(user_id).roles + if user_roles is None or user_roles == {}: + raise NoPermission( + error="You do not have permission to access this endpoint", + status_code=403, + log_txt=f"Error while user {user_id} tries to access an endpoint. " + f"The user does not have any role assigned. ", + ) + + action_id = PERMISSION_METHOD_MAP[method] + try: + view_id = ViewModel.query.filter_by(url_rule=url).first().id + except AttributeError: + current_app.logger.error( + "The permission for this endpoint is not in the database." + ) + raise NoPermission( + error="You do not have permission to access this endpoint", + status_code=403, + log_txt=f"Error while user {user_id} tries to access endpoint. " + f"The user does not permission to access. ", + ) + + for role in user_roles: + has_permission = PermissionViewRoleModel.get_permission( + role_id=role, api_view_id=view_id, action_id=action_id + ) + + if has_permission: + return True + + raise NoPermission( + error="You do not have permission to access this endpoint", + status_code=403, + log_txt=f"Error while user {user_id} tries to access endpoint {view_id} with action {action_id}. " + f"The user does not permission to access. ", + ) + @staticmethod def _get_request_info(req: Request) -> Tuple[str, str]: """ diff --git a/libs/core/cornflow_core/authentication/decorators.py b/cornflow-server/cornflow/shared/authentication/decorators.py similarity index 90% rename from libs/core/cornflow_core/authentication/decorators.py rename to cornflow-server/cornflow/shared/authentication/decorators.py index f1c8dca5c..8cb573793 100644 --- a/libs/core/cornflow_core/authentication/decorators.py +++ b/cornflow-server/cornflow/shared/authentication/decorators.py @@ -2,11 +2,11 @@ This file contains the decorator used for the authentication """ from functools import wraps -from .auth import BaseAuth -from cornflow_core.exceptions import InvalidCredentials +from .auth import Auth +from cornflow.shared.exceptions import InvalidCredentials -def authenticate(auth_class: BaseAuth): +def authenticate(auth_class: Auth): """ This is the decorator used for the authentication diff --git a/libs/core/cornflow_core/authentication/ldap.py b/cornflow-server/cornflow/shared/authentication/ldap.py similarity index 97% rename from libs/core/cornflow_core/authentication/ldap.py rename to cornflow-server/cornflow/shared/authentication/ldap.py index 27e50da1f..2c6b1b534 100644 --- a/libs/core/cornflow_core/authentication/ldap.py +++ b/cornflow-server/cornflow/shared/authentication/ldap.py @@ -6,8 +6,8 @@ from ldap3 import Server, Connection, ALL # Import from internal modules -from cornflow_core.constants import ALL_DEFAULT_ROLES, ROLES_MAP -from cornflow_core.exceptions import InvalidCredentials +from cornflow.shared.const import ALL_DEFAULT_ROLES, ROLES_MAP +from cornflow.shared.exceptions import InvalidCredentials class LDAPBase: diff --git a/libs/core/cornflow_core/compress/compress.py b/cornflow-server/cornflow/shared/compress.py similarity index 100% rename from libs/core/cornflow_core/compress/compress.py rename to cornflow-server/cornflow/shared/compress.py diff --git a/cornflow-server/cornflow/shared/const.py b/cornflow-server/cornflow/shared/const.py index 1decc9df8..e8fd2d591 100644 --- a/cornflow-server/cornflow/shared/const.py +++ b/cornflow-server/cornflow/shared/const.py @@ -1,22 +1,6 @@ """ In this files we import the values for different constants on cornflow server -This constants can be inherited from cornflow-core or can be overridden here """ -from cornflow_core.constants import ( - AUTH_DB, - AUTH_LDAP, - AUTH_OID, - AUTH_OAUTH, - OID_NONE, - OID_AZURE, - OID_GOOGLE, - VIEWER_ROLE, - PLANNER_ROLE, - ADMIN_ROLE, - SERVICE_ROLE, - ALL_DEFAULT_ROLES, - ROLES_MAP, -) # endpoints responses for health check STATUS_HEALTHY = "healthy" @@ -61,15 +45,15 @@ # These codes and names are inherited from flask app builder in order to have the same names and values # as this library that is the base of airflow -AUTH_DB = AUTH_DB -AUTH_LDAP = AUTH_LDAP -AUTH_OAUTH = AUTH_OAUTH -AUTH_OID = AUTH_OID +AUTH_DB = 1 +AUTH_LDAP = 2 +AUTH_OAUTH = 4 +AUTH_OID = 0 # Providers of open ID: -OID_NONE = OID_NONE -OID_AZURE = OID_AZURE -OID_GOOGLE = OID_GOOGLE +OID_NONE = 0 +OID_AZURE = 1 +OID_GOOGLE = 2 # AZURE OPEN ID URLS OID_AZURE_DISCOVERY_COMMON_URL = ( @@ -87,12 +71,12 @@ ALL_DEFAULT_ACTIONS = [GET_ACTION, PATCH_ACTION, POST_ACTION, PUT_ACTION, DELETE_ACTION] -VIEWER_ROLE = VIEWER_ROLE -PLANNER_ROLE = PLANNER_ROLE -ADMIN_ROLE = ADMIN_ROLE -SERVICE_ROLE = SERVICE_ROLE +VIEWER_ROLE = 1 +PLANNER_ROLE = 2 +ADMIN_ROLE = 3 +SERVICE_ROLE = 4 -ALL_DEFAULT_ROLES = ALL_DEFAULT_ROLES +ALL_DEFAULT_ROLES = [VIEWER_ROLE, PLANNER_ROLE, ADMIN_ROLE, SERVICE_ROLE] ACTIONS_MAP = { GET_ACTION: "can_get", @@ -110,7 +94,12 @@ "DELETE": DELETE_ACTION, } -ROLES_MAP = ROLES_MAP +ROLES_MAP = { + PLANNER_ROLE: "planner", + VIEWER_ROLE: "viewer", + ADMIN_ROLE: "admin", + SERVICE_ROLE: "service", +} BASE_PERMISSION_ASSIGNATION = [ (VIEWER_ROLE, GET_ACTION), diff --git a/libs/core/cornflow_core/messages/email.py b/cornflow-server/cornflow/shared/email.py similarity index 98% rename from libs/core/cornflow_core/messages/email.py rename to cornflow-server/cornflow/shared/email.py index 5fc146126..ad2db2060 100644 --- a/libs/core/cornflow_core/messages/email.py +++ b/cornflow-server/cornflow/shared/email.py @@ -7,7 +7,7 @@ from email.mime.text import MIMEText from smtplib import SMTP_SSL, SMTPAuthenticationError, SMTPRecipientsRefused -from cornflow_core.exceptions import InvalidData +from cornflow.shared.exceptions import InvalidData def get_email(text: str, subject: str, sender: str, receiver: str): diff --git a/libs/core/cornflow_core/exceptions/exceptions.py b/cornflow-server/cornflow/shared/exceptions.py similarity index 65% rename from libs/core/cornflow_core/exceptions/exceptions.py rename to cornflow-server/cornflow/shared/exceptions.py index be27a5095..de1099009 100644 --- a/libs/core/cornflow_core/exceptions/exceptions.py +++ b/cornflow-server/cornflow/shared/exceptions.py @@ -6,6 +6,7 @@ from webargs.flaskparser import parser from cornflow_client.constants import AirflowError from werkzeug.exceptions import HTTPException +import traceback class InvalidUsage(Exception): @@ -121,6 +122,12 @@ class ConfigurationError(InvalidUsage): error = "No authentication method configured on the server" +INTERNAL_SERVER_ERROR_MESSAGE = "500 Internal Server Error" +INTERNAL_SERVER_ERROR_MESSAGE_DETAIL = "The server encountered an internal error and was unable " \ + "to complete your request. Either the server is overloaded or " \ + "there is an error in the application." + + def initialize_errorhandlers(app): """ Function to register the different error handlers @@ -153,25 +160,60 @@ def handle_invalid_usage(error): response.status_code = error.status_code return response - if app.config["ENV"] in ["testing", "development"]: - - @app.errorhandler(Exception) - def handle_internal_server_error(error): - """ - Method to handle all the other exceptions - - :param error: the raised error - :type error: `Exception` - :return: an HTTP response - :rtype: `Response` - """ - if isinstance(error, HTTPException): - return error - error_str = f"{error.__class__.__name__}: {error}" + @app.errorhandler(Exception) + def handle_internal_server_error(error): + """ + Method to handle all the other exceptions + + :param error: the raised error + :type error: `Exception` + :return: an HTTP response + :rtype: `Response` + """ + error_msg = f"{error.__class__.__name__}: {error}" + error_str = f"{error.__class__.__name__}: {error}. {traceback.format_exc()}" + + status_code = 500 + + # ToDo: should we leave the default behavior for HTTPExceptions ? + if isinstance(error, HTTPException): + # Log only the name and description of the error since it's a HTTPException + app.logger.error(error_msg) + + # For HTTPExceptions we keep the associated messages + # but return them as json instead of html + + # HTTPExceptions sometimes already have an associated status code + status_code = error.code or status_code + error_msg = f"{status_code} {error.name or INTERNAL_SERVER_ERROR_MESSAGE}" + error_str = f"{error_msg}. {str(error.description or '') or INTERNAL_SERVER_ERROR_MESSAGE_DETAIL}" + response_dict = { + "message": error_msg, + "error": error_str + } + response = jsonify(response_dict) + + elif app.config["ENV"] == "production": + # Log the entire traceback app.logger.error(error_str) - response = jsonify(dict(error=error_str)) - response.status_code = 500 - return response + + # We are in production: we return generic messages + # to avoid giving away sensitive information + + response_dict = { + "message": INTERNAL_SERVER_ERROR_MESSAGE, + "error": INTERNAL_SERVER_ERROR_MESSAGE_DETAIL + } + response = jsonify(response_dict) + else: + # Log the entire traceback + app.logger.error(error_str) + + # Testing or development: we return the full error + response = jsonify(dict(message=error_msg, error=error_str)) + + response.status_code = status_code + return response return app diff --git a/cornflow-server/cornflow/shared/utils.py b/cornflow-server/cornflow/shared/utils.py index 12e90ec9b..472202c1d 100644 --- a/cornflow-server/cornflow/shared/utils.py +++ b/cornflow-server/cornflow/shared/utils.py @@ -1,11 +1,31 @@ """ - +This file defines the database session with SQLAlchemy and the password encryption with Bcrypt. +Additionally we add the option to have our database models inherit ABCMeta class so that abstract methods can be defined """ +from abc import ABCMeta +from flask_bcrypt import Bcrypt +from flask_sqlalchemy import SQLAlchemy +from flask_sqlalchemy.model import Model, DefaultMeta import hashlib import json +from sqlalchemy.ext.declarative import declarative_base def hash_json_256(data): return hashlib.sha256( json.dumps(data, sort_keys=True, separators=(",", ":")).encode("utf-8") - ).hexdigest() \ No newline at end of file + ).hexdigest() + + +class CustomABCMeta(DefaultMeta, ABCMeta): + """ + Custom meta class so that the models inherit ABCMeta + """ + + pass + + +db = SQLAlchemy( + model_class=declarative_base(cls=Model, metaclass=CustomABCMeta, name="Model") +) +bcrypt = Bcrypt() diff --git a/cornflow-server/cornflow/shared/utils_tables.py b/cornflow-server/cornflow/shared/utils_tables.py index ec73d095b..c9d05a03e 100644 --- a/cornflow-server/cornflow/shared/utils_tables.py +++ b/cornflow-server/cornflow/shared/utils_tables.py @@ -1,12 +1,16 @@ -from importlib import import_module -import sys -import os +# Imports from external libraries import inspect +import os +import sys + +from importlib import import_module from sqlalchemy.dialects.postgresql import TEXT -from cornflow_core.models import EmptyBaseModel -from cornflow_core.shared import db from sqlalchemy.sql.sqltypes import Integer + +# Imports from internal modules from cornflow.models import * +from cornflow.models.meta_models import EmptyBaseModel +from cornflow.shared import db def _import_file(filename): diff --git a/libs/core/cornflow_core/shared/validators.py b/cornflow-server/cornflow/shared/validators.py similarity index 98% rename from libs/core/cornflow_core/shared/validators.py rename to cornflow-server/cornflow/shared/validators.py index 0d116021f..49c6ae97d 100644 --- a/libs/core/cornflow_core/shared/validators.py +++ b/cornflow-server/cornflow/shared/validators.py @@ -1,5 +1,5 @@ """ -This file has several validators used on cornflow core +This file has several validators """ import re from typing import Tuple, Union diff --git a/cornflow-server/cornflow/tests/custom_liveServer.py b/cornflow-server/cornflow/tests/custom_liveServer.py index 8ea18a9ec..256950c82 100644 --- a/cornflow-server/cornflow/tests/custom_liveServer.py +++ b/cornflow-server/cornflow/tests/custom_liveServer.py @@ -2,7 +2,7 @@ import os import cornflow_client as cf -from cornflow_core.shared import db +from cornflow.shared import db # External libraries from flask import current_app diff --git a/cornflow-server/cornflow/tests/custom_test_case.py b/cornflow-server/cornflow/tests/custom_test_case.py index dd92121dc..3255f0340 100644 --- a/cornflow-server/cornflow/tests/custom_test_case.py +++ b/cornflow-server/cornflow/tests/custom_test_case.py @@ -18,7 +18,7 @@ from cornflow.commands.permissions import register_dag_permissions_command from cornflow.shared.authentication import Auth from cornflow.shared.const import ADMIN_ROLE, PLANNER_ROLE, SERVICE_ROLE -from cornflow_core.shared import db +from cornflow.shared import db from cornflow.tests.const import ( LOGIN_URL, SIGNUP_URL, diff --git a/cornflow-server/cornflow/tests/data/endpoints_access.json b/cornflow-server/cornflow/tests/data/endpoints_access.json new file mode 100644 index 000000000..f98a0b23b --- /dev/null +++ b/cornflow-server/cornflow/tests/data/endpoints_access.json @@ -0,0 +1,6 @@ +{ + "employees": [ + "ADMIN_ROLE", + "SERVICE_ROLE" + ] +} \ No newline at end of file diff --git a/cornflow-server/cornflow/tests/data/endpoints_methods.json b/cornflow-server/cornflow/tests/data/endpoints_methods.json new file mode 100644 index 000000000..66cde7736 --- /dev/null +++ b/cornflow-server/cornflow/tests/data/endpoints_methods.json @@ -0,0 +1,22 @@ +{ + "employees": [ + "get_list", + "post_list", + "get_detail", + "put_detail", + "patch_detail", + "delete_detail", + "post_bulk", + "put_bulk" + ], + "shifts": [ + "get_list", + "post_list", + "get_detail", + "put_detail", + "patch_detail", + "delete_detail", + "post_bulk", + "put_bulk" + ] +} \ No newline at end of file diff --git a/cornflow-server/cornflow/tests/data/instance2_gfs.json b/cornflow-server/cornflow/tests/data/instance2_gfs.json new file mode 100644 index 000000000..d8e33963d --- /dev/null +++ b/cornflow-server/cornflow/tests/data/instance2_gfs.json @@ -0,0 +1,388 @@ +{ + "$schema": "http://json-schema.org/schema#", + "type": "object", + "properties": { + "parameters": { + "type": "object", + "description": "Scalar values needed for the rostering problem", + "show": true, + "filterable": false, + "sortable": true, + "properties": { + "horizon": { + "type": "integer", + "title": "Horizon", + "description": "The number of weeks that are going to be solved", + "minimum": 1 + }, + "opening_days": { + "type": "integer", + "title": "Days open", + "description": "The number of days that the work center opens, the first day is always considered to be a Monday", + "minimum": 0, + "maximum": 7 + }, + "slot_length": { + "type": "integer", + "title": "Slot length", + "description": "The length of each time slot in minutes" + }, + "starting_hour": { + "type": "number", + "title": "Starting hour", + "description": "The hour the work center opens", + "minimum": 0, + "maximum": 23 + }, + "ending_hour": { + "type": "number", + "title": "Ending hour", + "description": "The hour the work center closes", + "minimum": 1, + "maximum": 24 + }, + "starting_date": { + "type": "string", + "title": "Starting date", + "description": "The first day that has to be solved", + "format": "date" + }, + "min_working_hours": { + "type": "integer", + "title": "Minimum hours work per day", + "description": "The minimum amount of hours that have to be worked each day that the employee works", + "minimum": 0, + "maximum": 24 + }, + "min_resting_hours": { + "type": "integer", + "title": "Minimum hours to rest", + "description": "The minimum amount of hours that have to be rested between the end of the shift on one day, and the start of the shift on the next day", + "minimum": 0 + } + }, + "required": [ + "horizon", + "opening_days", + "slot_length", + "starting_hour", + "ending_hour", + "starting_date", + "min_working_hours", + "min_resting_hours" + ] + }, + "employees": { + "type": "array", + "description": "Table with the employee master information", + "show": true, + "filterable": true, + "sortable": true, + "items": { + "type": "object", + "properties": { + "id": { + "description": "The unique identifier for each employee.", + "title": "Employee id.", + "type": "string", + "sortable": true, + "filterable": true + }, + "name": { + "description": "The name of each employee.", + "title": "Employee name", + "type": "string", + "sortable": true, + "filterable": true + }, + "manager": { + "description": "If the employee is a manager or not.", + "title": "Manager", + "type": "boolean", + "sortable": true, + "filterable": true + }, + "manager_tasks": { + "description": "If the employee can perform the tasks of a manager without being one. It is not required.", + "title": "Manager tasks", + "type": "boolean", + "sortable": true, + "filterable": false + } + }, + "required": [ + "id", + "name", + "manager" + ] + } + }, + "shifts": { + "type": "array", + "description": "Table with the shifts master information", + "show": true, + "filterable": true, + "sortable": true, + "items": { + "type": "object", + "properties": { + "id": { + "description": "The unique identifier of each shift.", + "title": "Shift id.", + "type": "integer", + "sortable": true, + "filterable": true + }, + "name": { + "description": "The name of the shift.", + "title": "Shift name", + "type": "string", + "sortable": true, + "filterable": true + }, + "start": { + "description": "The earliest hour that an employee assigned to this shift can start working.", + "title": "Starting hour", + "type": "integer", + "sortable": true, + "filterable": true + }, + "end": { + "description": "The latest hour that an employee assigned to this shift can stop working.", + "title": "Ending hour", + "type": "integer", + "sortable": true, + "filterable": true + } + }, + "required": [ + "id", + "name", + "start", + "end" + ] + } + }, + "contracts": { + "type": "array", + "description": "Table with the relationship between employees, shifts and their past, current and future contracts", + "show": true, + "filterable": true, + "sortable": true, + "items": { + "type": "object", + "properties": { + "id": { + "description": "The unique identifier of each contract.", + "title": "Contract id.", + "type": "integer", + "sortable": true, + "filterable": true + }, + "id_employee": { + "description": "The unique identifier of the employee assigned to the contract.", + "title": "Employee id.", + "type": "integer", + "sortable": true, + "filterable": true, + "foreign_key": "employees.id" + }, + "id_shift": { + "description": "The unique identifier of the shift assigned to the contract.", + "title": "Shift id.", + "type": "integer", + "sortable": true, + "filterable": true, + "foreign_key": "shifts.id" + }, + "start_contract": { + "description": "The starting date of the contract. Must be a Monday and have format YYYY-MM-DD.", + "title": "Contract start", + "type": "string", + "$comment": "date", + "sortable": true, + "filterable": true + }, + "end_contract": { + "description": "The ending date of the contract. Must be a Sunday and have format YYYY-MM-DD. It is not required and can be void.", + "title": "Contract end", + "type": [ + "string", + "null" + ], + "$comment": "date", + "sortable": true, + "filterable": false + }, + "weekly_hours": { + "description": "The number of hours that the employee has to work each week.", + "title": "Contract weekly hours", + "type": "number", + "sortable": true, + "filterable": true + }, + "days_worked": { + "description": "The number of days that the employee has to work per week.", + "title": "Contract weekly days", + "type": "integer", + "sortable": true, + "filterable": true + } + }, + "required": [ + "id", + "id_employee", + "id_shift", + "start_contract", + "weekly_hours", + "days_worked" + ] + } + }, + "demand": { + "type": "array", + "description": "Demand that needs to be covered and tried to make the same across all time slots", + "show": true, + "filterable": true, + "sortable": true, + "items": { + "type": "object", + "properties": { + "day": { + "description": "The date of the demand. Has to have format YYYY-MM-DD.", + "title": "Day", + "type": "string", + "$comment": "date", + "sortable": true, + "filterable": true + }, + "hour": { + "description": "The hour of the demand.", + "title": "Hour", + "type": "number", + "sortable": true, + "filterable": true + }, + "demand": { + "description": "The demand value.", + "title": "Demand", + "type": "number", + "sortable": true, + "filterable": true + } + }, + "required": [ + "day", + "hour", + "demand" + ] + } + }, + "skills": { + "type": "array", + "description": "Table with the information about each skill", + "show": true, + "filterable": true, + "sortable": true, + "items": { + "type": "object", + "properties": { + "id": { + "description": "The unique identifier of the skill", + "title": "Skill id.", + "type": "integer", + "sortable": true, + "filterable": true + }, + "name": { + "description": "The name of the skill", + "title": "Name", + "type": "string", + "sortable": true, + "filterable": true + } + }, + "required": ["id", "name"] + } + }, + "skills_employees": { + "type": "array", + "description": "Table with the skills of each employee", + "show": true, + "filterable": true, + "sortable": true, + "items": { + "type": "object", + "properties": { + "id_skill": { + "description": "The unique identifier of the skill", + "title": "Skill id.", + "type": "integer", + "sortable": true, + "filterable": true, + "foreign_key": "skills.id" + }, + "id_employee": { + "description": "The unique identifier of the employee", + "title": "Employee id.", + "type": "integer", + "sortable": true, + "filterable": true, + "foreign_key": "employees.id" + } + }, + "required": ["id_skill", "id_employee"] + } + }, + "skill_demand": { + "type": "array", + "description": "Demand for each skill, that needs to be covered", + "show": true, + "filterable": true, + "sortable": true, + "items": { + "type": "object", + "properties": { + "day": { + "description": "The date of the skill demand. Has to have format YYYY-MM-DD.", + "title": "Day", + "type": "string", + "$comment": "date", + "sortable": true, + "filterable": true + }, + "hour": { + "description": "The hour of the skill demand.", + "title": "Hour", + "type": "number", + "sortable": true, + "filterable": true + }, + "id_skill": { + "description": "The unique identifier of the skill", + "title": "Skill id.", + "type": "integer", + "sortable": true, + "filterable": true, + "foreign_key": "skills.id" + }, + "demand": { + "description": "The number of employees with the skill that are demanded.", + "title": "Demand", + "type": "integer", + "sortable": true, + "filterable": true + } + }, + "required": ["day", "hour", "id_skill", "demand"] + } + } + }, + "required": [ + "employees", + "shifts", + "contracts", + "demand", + "parameters" + ] +} \ No newline at end of file diff --git a/libs/core/cornflow_core/tests/data/instance.json b/cornflow-server/cornflow/tests/data/instance_gfs.json similarity index 100% rename from libs/core/cornflow_core/tests/data/instance.json rename to cornflow-server/cornflow/tests/data/instance_gfs.json diff --git a/libs/core/cornflow_core/tests/data/models/__init__.py b/cornflow-server/cornflow/tests/data/models/__init__.py similarity index 100% rename from libs/core/cornflow_core/tests/data/models/__init__.py rename to cornflow-server/cornflow/tests/data/models/__init__.py diff --git a/libs/core/cornflow_core/tests/data/models/action.py b/cornflow-server/cornflow/tests/data/models/action.py similarity index 77% rename from libs/core/cornflow_core/tests/data/models/action.py rename to cornflow-server/cornflow/tests/data/models/action.py index e70de6da5..e873e05c4 100644 --- a/libs/core/cornflow_core/tests/data/models/action.py +++ b/cornflow-server/cornflow/tests/data/models/action.py @@ -1,12 +1,11 @@ """ """ -from cornflow_core.models import ActionBaseModel +from cornflow.models import ActionModel -# from .meta_model import EmptyModel -from cornflow_core.shared import db +from cornflow.shared import db -class ActionModel(ActionBaseModel): +class ActionModel(ActionModel): """ This model contains the base actions over the REST API. These are: * can get diff --git a/libs/core/cornflow_core/tests/data/models/base_data_model.py b/cornflow-server/cornflow/tests/data/models/base_data_model.py similarity index 95% rename from libs/core/cornflow_core/tests/data/models/base_data_model.py rename to cornflow-server/cornflow/tests/data/models/base_data_model.py index 2e4f063bc..a2ef366c6 100644 --- a/libs/core/cornflow_core/tests/data/models/base_data_model.py +++ b/cornflow-server/cornflow/tests/data/models/base_data_model.py @@ -1,18 +1,15 @@ """ """ -# Import from libraries -from cornflow_core.models import TraceAttributesModel - # Import from internal modules -from cornflow_core.shared import db +from cornflow.shared import db +from cornflow.models.meta_models import TraceAttributesModel +from cornflow.shared.utils import hash_json_256 from flask import current_app from sqlalchemy import desc from sqlalchemy.dialects.postgresql import JSON from sqlalchemy.dialects.postgresql import TEXT from sqlalchemy.ext.declarative import declared_attr -from ..shared.utils import hash_json_256 - class BaseDataModel(TraceAttributesModel): """ """ diff --git a/libs/core/cornflow_core/tests/data/models/instance.py b/cornflow-server/cornflow/tests/data/models/instance.py similarity index 98% rename from libs/core/cornflow_core/tests/data/models/instance.py rename to cornflow-server/cornflow/tests/data/models/instance.py index 7e3856eb1..d1b87ed13 100644 --- a/libs/core/cornflow_core/tests/data/models/instance.py +++ b/cornflow-server/cornflow/tests/data/models/instance.py @@ -5,7 +5,7 @@ # Imported from internal models from .base_data_model import BaseDataModel -from cornflow_core.shared import db +from cornflow.shared import db class InstanceModel(BaseDataModel): diff --git a/libs/core/cornflow_core/tests/data/models/permission.py b/cornflow-server/cornflow/tests/data/models/permission.py similarity index 91% rename from libs/core/cornflow_core/tests/data/models/permission.py rename to cornflow-server/cornflow/tests/data/models/permission.py index f196b5bbf..6a88f0dfc 100644 --- a/libs/core/cornflow_core/tests/data/models/permission.py +++ b/cornflow-server/cornflow/tests/data/models/permission.py @@ -1,11 +1,11 @@ -from cornflow_core.models import PermissionViewRoleBaseModel, TraceAttributesModel +from cornflow.models import PermissionViewRoleModel +from cornflow.models.meta_models import TraceAttributesModel -# from .meta_model import TraceAttributes -from .dag import DeployedDAG -from cornflow_core.shared import db +from cornflow.models.dag import DeployedDAG +from cornflow.shared import db -class PermissionViewRoleModel(PermissionViewRoleBaseModel): +class PermissionViewRoleModel(PermissionViewRoleModel): # TODO: trace the user that modifies the permissions __tablename__ = "permission_view" __table_args__ = {"extend_existing": True} diff --git a/libs/core/cornflow_core/tests/data/one_table.json b/cornflow-server/cornflow/tests/data/one_table_gfs.json similarity index 100% rename from libs/core/cornflow_core/tests/data/one_table.json rename to cornflow-server/cornflow/tests/data/one_table_gfs.json diff --git a/cornflow-server/cornflow/tests/integration/test_cornflowclient.py b/cornflow-server/cornflow/tests/integration/test_cornflowclient.py index 31b13aaaa..76b8e9d08 100644 --- a/cornflow-server/cornflow/tests/integration/test_cornflowclient.py +++ b/cornflow-server/cornflow/tests/integration/test_cornflowclient.py @@ -346,7 +346,7 @@ def test_status_solving_timer(self): time.sleep(5) status = self.client.get_status(execution["id"]) self.assertEqual(status["state"], EXEC_STATE_RUNNING) - time.sleep(10) + time.sleep(12) status = self.client.get_status(execution["id"]) self.assertEqual(status["state"], EXEC_STATE_CORRECT) diff --git a/cornflow-server/cornflow/tests/unit/test_cli.py b/cornflow-server/cornflow/tests/unit/test_cli.py index 2f986c911..ba82442d9 100644 --- a/cornflow-server/cornflow/tests/unit/test_cli.py +++ b/cornflow-server/cornflow/tests/unit/test_cli.py @@ -5,13 +5,13 @@ from cornflow.app import create_app from cornflow.cli import cli from cornflow.models import UserModel -from cornflow_core.models import ( - ActionBaseModel, - RoleBaseModel, - ViewBaseModel, - PermissionViewRoleBaseModel, +from cornflow.models import ( + ActionModel, + RoleModel, + ViewModel, + PermissionViewRoleModel, ) -from cornflow_core.shared import db +from cornflow.shared import db from flask_testing import TestCase @@ -62,7 +62,7 @@ def test_actions(self): runner = CliRunner() result = runner.invoke(cli, ["actions", "init", "-v"]) self.assertEqual(result.exit_code, 0) - actions = ActionBaseModel.get_all_objects().all() + actions = ActionModel.get_all_objects().all() self.assertEqual(len(actions), 5) def test_config_entrypoint(self): @@ -113,7 +113,7 @@ def test_roles_init_command(self): runner = CliRunner() result = runner.invoke(cli, ["roles", "init", "-v"]) self.assertEqual(result.exit_code, 0) - roles = RoleBaseModel.get_all_objects().all() + roles = RoleModel.get_all_objects().all() self.assertEqual(len(roles), 4) def test_views_entrypoint(self): @@ -128,7 +128,7 @@ def test_views_init_command(self): runner = CliRunner() result = runner.invoke(cli, ["views", "init", "-v"]) self.assertEqual(result.exit_code, 0) - views = ViewBaseModel.get_all_objects().all() + views = ViewModel.get_all_objects().all() self.assertEqual(len(views), 48) def test_permissions_entrypoint(self): @@ -147,10 +147,10 @@ def test_permissions_init(self): runner = CliRunner() result = runner.invoke(cli, ["permissions", "init", "-v"]) self.assertEqual(result.exit_code, 0) - actions = ActionBaseModel.get_all_objects().all() - roles = RoleBaseModel.get_all_objects().all() - views = ViewBaseModel.get_all_objects().all() - permissions = PermissionViewRoleBaseModel.get_all_objects().all() + actions = ActionModel.get_all_objects().all() + roles = RoleModel.get_all_objects().all() + views = ViewModel.get_all_objects().all() + permissions = PermissionViewRoleModel.get_all_objects().all() self.assertEqual(len(actions), 5) self.assertEqual(len(roles), 4) self.assertEqual(len(views), 48) @@ -163,10 +163,10 @@ def test_permissions_base_command(self): runner.invoke(cli, ["views", "init", "-v"]) result = runner.invoke(cli, ["permissions", "base", "-v"]) self.assertEqual(result.exit_code, 0) - actions = ActionBaseModel.get_all_objects().all() - roles = RoleBaseModel.get_all_objects().all() - views = ViewBaseModel.get_all_objects().all() - permissions = PermissionViewRoleBaseModel.get_all_objects().all() + actions = ActionModel.get_all_objects().all() + roles = RoleModel.get_all_objects().all() + views = ViewModel.get_all_objects().all() + permissions = PermissionViewRoleModel.get_all_objects().all() self.assertEqual(len(actions), 5) self.assertEqual(len(roles), 4) self.assertEqual(len(views), 48) diff --git a/cornflow-server/cornflow/tests/unit/test_commands.py b/cornflow-server/cornflow/tests/unit/test_commands.py index 609495113..223bc9766 100644 --- a/cornflow-server/cornflow/tests/unit/test_commands.py +++ b/cornflow-server/cornflow/tests/unit/test_commands.py @@ -1,15 +1,5 @@ import json -from cornflow.tests.const import LOGIN_URL, INSTANCE_URL, INSTANCE_PATH -from cornflow.tests.integration.test_cornflowclient import load_file -from cornflow_core.models import ( - ActionBaseModel, - PermissionViewRoleBaseModel, - RoleBaseModel, - ViewBaseModel, -) - -from cornflow_core.shared import db from flask_testing import TestCase from cornflow.app import ( @@ -23,22 +13,27 @@ register_roles, register_views, ) - - from cornflow.commands.dag import register_deployed_dags_command_test from cornflow.endpoints import resources, alarms_resources - +from cornflow.models import ( + ActionModel, + PermissionViewRoleModel, + RoleModel, + ViewModel, +) from cornflow.models import ( DeployedDAG, PermissionsDAG, UserModel, ) - +from cornflow.shared import db from cornflow.shared.const import ( ACTIONS_MAP, ROLES_MAP, BASE_PERMISSION_ASSIGNATION, ) +from cornflow.tests.const import LOGIN_URL, INSTANCE_URL, INSTANCE_PATH +from cornflow.tests.integration.test_cornflowclient import load_file class TestCommands(TestCase): @@ -150,7 +145,7 @@ def test_base_user_command(self): def test_register_actions(self): self.runner.invoke(register_actions) - actions = ActionBaseModel.query.all() + actions = ActionModel.query.all() for a in actions: self.assertEqual(ACTIONS_MAP[a.id], a.name) @@ -158,7 +153,7 @@ def test_register_actions(self): def test_register_views(self): self.runner.invoke(register_views) - views = ViewBaseModel.query.all() + views = ViewModel.query.all() views_list = [v.name for v in views] resources_list = [ self.resources[i]["endpoint"] for i in range(len(self.resources)) @@ -167,7 +162,7 @@ def test_register_views(self): self.assertCountEqual(views_list, resources_list) def test_register_roles(self): - roles = RoleBaseModel.query.all() + roles = RoleModel.query.all() for r in roles: self.assertEqual(ROLES_MAP[r.id], r.name) @@ -177,10 +172,9 @@ def test_base_permissions_assignation(self): for base in BASE_PERMISSION_ASSIGNATION: for view in self.resources: if base[0] in view["resource"].ROLES_WITH_ACCESS: - - permission = PermissionViewRoleBaseModel.get_permission( + permission = PermissionViewRoleModel.get_permission( role_id=base[0], - api_view_id=ViewBaseModel.query.filter_by(name=view["endpoint"]) + api_view_id=ViewModel.query.filter_by(name=view["endpoint"]) .first() .id, action_id=base[1], diff --git a/cornflow-server/cornflow/tests/unit/test_dags.py b/cornflow-server/cornflow/tests/unit/test_dags.py index 2d5f58b60..c69ca9a43 100644 --- a/cornflow-server/cornflow/tests/unit/test_dags.py +++ b/cornflow-server/cornflow/tests/unit/test_dags.py @@ -14,7 +14,7 @@ from cornflow.shared.const import ADMIN_ROLE, SERVICE_ROLE from cornflow.models import DeployedDAG, PermissionsDAG, UserModel, UserRoleModel from cornflow.shared.const import EXEC_STATE_CORRECT, EXEC_STATE_MANUAL -from cornflow_core.shared import db +from cornflow.shared import db from cornflow.tests.const import ( CASE_PATH, DAG_URL, diff --git a/libs/core/cornflow_core/tests/test_generate_from_schema.py b/cornflow-server/cornflow/tests/unit/test_generate_from_schema.py similarity index 66% rename from libs/core/cornflow_core/tests/test_generate_from_schema.py rename to cornflow-server/cornflow/tests/unit/test_generate_from_schema.py index c9c70d59b..155d58457 100644 --- a/libs/core/cornflow_core/tests/test_generate_from_schema.py +++ b/cornflow-server/cornflow/tests/unit/test_generate_from_schema.py @@ -1,18 +1,22 @@ +# Imports from libraries import importlib.util -import unittest -from unittest.mock import MagicMock -import os -import sys import json +import os import shutil +import sys +import unittest from click.testing import CliRunner from flask_sqlalchemy import SQLAlchemy +from pytups import TupList, SuperDict from sqlalchemy.dialects.postgresql import TEXT, JSON from sqlalchemy.sql.sqltypes import Integer -from pytups import TupList, SuperDict -from cornflow_core.models import TraceAttributesModel -from cornflow_core.cli.generate_from_schema import generate_from_schema +from unittest.mock import MagicMock + +# Imports from internal modules +from cornflow.models.meta_models import TraceAttributesModel +from cornflow.cli import cli +from cornflow.cli.schemas import APIGenerator sys.modules["mockedpackage"] = MagicMock() path_to_tests = os.path.dirname(os.path.abspath(__file__)) @@ -22,24 +26,37 @@ class GenerationTests(unittest.TestCase): def setUp(self): super().setUp() - self.full_inst_path = self._get_path("./data/instance.json") + self.full_inst_path = self._get_path( + "../data/instance_gfs.json") + self.inst_path2 = self._get_path("../data/instance2_gfs.json") self.full_inst = SuperDict.from_dict(self.import_schema(self.full_inst_path)) # Removing parameter tables self.full_inst["properties"] = self.full_inst["properties"].vfilter( lambda v: v["type"] == "array" ) - self.one_tab_inst_path = self._get_path("./data/one_table.json") + self.one_tab_inst_path = self._get_path("../data/one_table_gfs.json") self.one_tab_inst = SuperDict.from_dict( self.import_schema(self.one_tab_inst_path) ) self.app_name = "test" self.second_app_name = "test_sec" - self.default_output_path = self._get_path("./data/output") - self.other_output_path = self._get_path("./data/output_path") + self.default_output_path = self._get_path("../data/output") + self.other_output_path = self._get_path("../data/output_path") self.last_path = self.default_output_path self.all_methods = TupList( - ["getOne", "getAll", "deleteOne", "deleteAll", "update", "post"] + [ + "get_detail", + "get_list", + "delete_detail", + "put_detail", + "patch_detail", + "post_list", + "put_bulk", + "post_bulk", + ] ) + self.endpoints_methods_path = self._get_path("../data/endpoints_methods.json") + self.endpoints_access_path = self._get_path("../data/endpoints_access.json") def tearDown(self): if os.path.isdir(self.last_path): @@ -58,15 +75,17 @@ def import_schema(path): def test_base(self): runner = CliRunner() result = runner.invoke( - generate_from_schema, + cli, [ + "schemas", + "generate_from_schema", "-p", self.full_inst_path, "-a", self.app_name, "-o", self.other_output_path, - ], + ] ) self.assertEqual(result.exit_code, 0) @@ -76,8 +95,10 @@ def test_base(self): def test_one_table_schema(self): runner = CliRunner() result = runner.invoke( - generate_from_schema, + cli, [ + "schemas", + "generate_from_schema", "-p", self.one_tab_inst_path, "-a", @@ -96,8 +117,10 @@ def test_one_table_schema(self): def test_one_table_one_option(self): runner = CliRunner() result = runner.invoke( - generate_from_schema, + cli, [ + "schemas", + "generate_from_schema", "-p", self.one_tab_inst_path, "-a", @@ -118,8 +141,10 @@ def test_one_table_one_option(self): def test_remove_method(self): runner = CliRunner() result = runner.invoke( - generate_from_schema, + cli, [ + "schemas", + "generate_from_schema", "-p", self.full_inst_path, "-a", @@ -140,7 +165,7 @@ def test_remove_method(self): self.assertEqual(result.exit_code, 0) include_methods = self.all_methods.vfilter( - lambda v: v not in ["deleteOne", "update", "getOne"] + lambda v: v not in ["delete_detail", "put_detail", "get_detail", "patch_detail"] ) self.last_path = self.other_output_path self.check( @@ -149,6 +174,49 @@ def test_remove_method(self): app_name=self.second_app_name, ) + def test_endpoints_methods(self): + + runner = CliRunner() + result = runner.invoke( + cli, + [ + "schemas", + "generate_from_schema", + "-p", + self.full_inst_path, + "-a", + "test3", + "-o", + self.other_output_path, + "-m", + self.endpoints_methods_path, + ], + ) + self.assertEqual(result.exit_code, 0) + self.last_path = self.other_output_path + self.check(output_path=self.other_output_path, app_name="test3") + + def test_endpoints_access(self): + runner = CliRunner() + result = runner.invoke( + cli, + [ + "schemas", + "generate_from_schema", + "-p", + self.full_inst_path, + "-a", + "test4", + "-o", + self.other_output_path, + "-e", + self.endpoints_access_path, + ], + ) + self.assertEqual(result.exit_code, 0) + self.last_path = self.other_output_path + self.check(output_path=self.other_output_path, app_name="test4") + def check( self, instance=None, output_path=None, include_methods=None, app_name=None ): @@ -183,16 +251,14 @@ def check( with open(path_file, "r") as fd: txt = fd.read() packages_to_mock = [ - "..shared.utils", - ".meta_model", - ".meta_resource", - "..shared.const", - "..shared.authentification", - "..models", - "..schemas", + " cornflow.models ", + " cornflow.schemas ", + " cornflow.shared.authentication ", + " ..models ", + " ..schemas " ] for package in packages_to_mock: - txt = txt.replace(package, "mockedpackage") + txt = txt.replace(package, " mockedpackage ") with open(path_file, "w") as fd: fd.write(txt) @@ -269,15 +335,16 @@ def check( # Checks that the endpoints have all the methods for file, table in files: mod_name = self.snake_to_camel(app_name + "_" + table + "_endpoint") - class_names = [self.snake_to_camel(app_name + "_" + table + "_endpoint")] - if ( - "getOne" in include_methods - or "deleteOne" in include_methods - or "update" in include_methods - ): - class_names.append( - self.snake_to_camel(app_name + "_" + table + "_details_endpoint") - ) + class_names = [] + base = self.snake_to_camel(app_name + "_" + table + "_endpoint") + details = self.snake_to_camel(app_name + "_" + table + "_details_endpoint") + bulk = self.snake_to_camel(app_name + "_" + table + "_bulk_endpoint") + if any(m in include_methods for m in ["get_list", "post_list"]): + class_names += [base] + if any(m in include_methods for m in ["get_detail", "delete_detail", "put_detail", "patch_detail"]): + class_names += [details] + if any(m in include_methods for m in ["put_bulk", "post_bulk"]): + class_names += [bulk] file_path = os.path.join(endpoints_dir, file) spec = importlib.util.spec_from_file_location(mod_name, file_path) mod = importlib.util.module_from_spec(spec) @@ -294,29 +361,49 @@ def check( "delete_detail": "DELETE", "put_detail": "PUT", "patch_detail": "PATCH", + "post_bulk": "POST", + "put_bulk": "PUT", } - # Checks the methods of the first endpoint - include_methods_e1 = [ - method_name - for method_name in include_methods - if method_name in ["get_list", "post_list"] - ] - props_and_methods = mod.__dict__[class_names[0]].methods - for method_name in include_methods_e1: - self.assertIn(api_methods[method_name], props_and_methods) + # Checks the methods of the base endpoint + if base in class_names: + include_methods_base = [ + method_name + for method_name in include_methods + if method_name in ["get_list", "post_list"] + ] + props_and_methods = mod.__dict__[base].methods + for method_name in include_methods_base: + self.assertIn(api_methods[method_name], props_and_methods) # Checks the methods of the details endpoint - if len(class_names) == 2: - include_methods_e2 = [ + if details in class_names: + include_methods_details = [ method_name for method_name in include_methods if method_name in ["get_detail", "put_detail", "delete_detail", "patch_detail"] ] - props_and_methods = mod.__dict__[class_names[1]].methods - for method_name in include_methods_e2: + props_and_methods = mod.__dict__[details].methods + for method_name in include_methods_details: self.assertIn(api_methods[method_name], props_and_methods) + if bulk in class_names: + include_methods_bulk = [ + method_name + for method_name in include_methods + if method_name in ["post_bulk", "put_bulk"] + ] + props_and_methods = mod.__dict__[bulk].methods + for method_name in include_methods_bulk: + self.assertIn(api_methods[method_name], props_and_methods) + + def test_get_id_type(self): + api_gen = APIGenerator(schema_path=self.inst_path2, app_name=None) + self.assertEqual(api_gen.get_id_type("employees"), "") + self.assertEqual(api_gen.get_id_type("shifts"), "") + self.assertEqual(api_gen.get_id_type("demand"), "") + + @staticmethod def snake_to_camel(name): return "".join(word.title() for word in name.split("_")) diff --git a/cornflow-server/cornflow/tests/unit/test_health.py b/cornflow-server/cornflow/tests/unit/test_health.py index 2b5c556ae..222d77163 100644 --- a/cornflow-server/cornflow/tests/unit/test_health.py +++ b/cornflow-server/cornflow/tests/unit/test_health.py @@ -1,6 +1,6 @@ import os -from cornflow_core.shared import db +from cornflow.shared import db from cornflow.app import create_app from cornflow.commands import access_init_command diff --git a/cornflow-server/cornflow/tests/unit/test_log_in.py b/cornflow-server/cornflow/tests/unit/test_log_in.py index 74d619f46..1e9f253d3 100644 --- a/cornflow-server/cornflow/tests/unit/test_log_in.py +++ b/cornflow-server/cornflow/tests/unit/test_log_in.py @@ -7,7 +7,7 @@ # Import from internal modules from cornflow.models import UserModel -from cornflow_core.shared import db +from cornflow.shared import db from cornflow.tests.custom_test_case import LoginTestCases diff --git a/cornflow-server/cornflow/tests/unit/test_permissions.py b/cornflow-server/cornflow/tests/unit/test_permissions.py index 353f1553d..15994319c 100644 --- a/cornflow-server/cornflow/tests/unit/test_permissions.py +++ b/cornflow-server/cornflow/tests/unit/test_permissions.py @@ -5,11 +5,11 @@ # Import from libraries import json -from cornflow_core.models import ( - ActionBaseModel, - PermissionViewRoleBaseModel, - RoleBaseModel, - ViewBaseModel, +from cornflow.models import ( + ActionModel, + PermissionViewRoleModel, + RoleModel, + ViewModel, ) # Import from internal modules @@ -27,7 +27,7 @@ class TestPermissionsViewRoleEndpoint(CustomTestCase): def setUp(self): super().setUp() - self.model = PermissionViewRoleBaseModel + self.model = PermissionViewRoleModel self.roles_with_access = PermissionsViewRoleEndpoint.ROLES_WITH_ACCESS self.payload = {"role_id": 1, "permission_id": 1, "api_view_id": 1} @@ -88,7 +88,7 @@ def test_new_role_not_authorized(self): class TestPermissionViewRolesDetailEndpoint(CustomTestCase): def setUp(self): super().setUp() - self.model = PermissionViewRoleBaseModel + self.model = PermissionViewRoleModel self.roles_with_access = PermissionsViewRoleDetailEndpoint.ROLES_WITH_ACCESS self.payload = {"role_id": 1, "action_id": 3, "api_view_id": 1} self.items_to_check = [] @@ -223,22 +223,22 @@ def setUp(self): super().setUp() def test_permission_role_cascade_deletion(self): - before_permissions = PermissionViewRoleBaseModel.get_all_objects() - role = RoleBaseModel.get_one_object(VIEWER_ROLE) + before_permissions = PermissionViewRoleModel.get_all_objects() + role = RoleModel.get_one_object(VIEWER_ROLE) role.delete() - after_permissions = PermissionViewRoleBaseModel.get_all_objects() + after_permissions = PermissionViewRoleModel.get_all_objects() self.assertNotEqual(before_permissions, after_permissions) def test_permission_action_cascade_deletion(self): - before_permissions = PermissionViewRoleBaseModel.get_all_objects() - action = ActionBaseModel.get_one_object(GET_ACTION) + before_permissions = PermissionViewRoleModel.get_all_objects() + action = ActionModel.get_one_object(GET_ACTION) action.delete() - after_permissions = PermissionViewRoleBaseModel.get_all_objects() + after_permissions = PermissionViewRoleModel.get_all_objects() self.assertNotEqual(before_permissions, after_permissions) def test_permission_api_view_cascade_deletion(self): - before_permissions = PermissionViewRoleBaseModel.get_all_objects() - api_view = ViewBaseModel.get_one_by_name("instance") + before_permissions = PermissionViewRoleModel.get_all_objects() + api_view = ViewModel.get_one_by_name("instance") api_view.delete() - after_permissions = PermissionViewRoleBaseModel.get_all_objects() + after_permissions = PermissionViewRoleModel.get_all_objects() self.assertNotEqual(before_permissions, after_permissions) diff --git a/cornflow-server/cornflow/tests/unit/test_roles.py b/cornflow-server/cornflow/tests/unit/test_roles.py index ab426aa38..a0b0fe826 100644 --- a/cornflow-server/cornflow/tests/unit/test_roles.py +++ b/cornflow-server/cornflow/tests/unit/test_roles.py @@ -3,7 +3,7 @@ """ import json import logging as log -from cornflow_core.models import PermissionViewRoleBaseModel, RoleBaseModel +from cornflow.models import PermissionViewRoleModel, RoleModel # Import from internal modules from cornflow.endpoints import ( @@ -32,7 +32,7 @@ def setUp(self): self.payload = {"name": "new_role"} self.payloads = [{"id": key, "name": value} for key, value in ROLES_MAP.items()] self.url = ROLES_URL - self.model = RoleBaseModel + self.model = RoleModel self.items_to_check = ["name"] self.roles_with_access = RolesListEndpoint.ROLES_WITH_ACCESS @@ -115,7 +115,7 @@ class TestRolesDetailEndpoint(CustomTestCase): def setUp(self): super().setUp() self.url = ROLES_URL - self.model = RoleBaseModel + self.model = RoleModel self.items_to_check = ["id", "name"] self.roles_with_access = RoleDetailEndpoint.ROLES_WITH_ACCESS @@ -427,7 +427,7 @@ class TestRolesModelMethods(CustomTestCase): def setUp(self): super().setUp() self.url = ROLES_URL - self.model = RoleBaseModel + self.model = RoleModel self.payload = {"name": "test_role"} def test_user_role_delete_cascade(self): @@ -447,10 +447,10 @@ def test_permission_delete_cascade(self): self.token = self.create_user_with_role(ADMIN_ROLE) idx = self.create_new_row(self.url, self.model, self.payload) payload = {"action_id": 1, "api_view_id": 1, "role_id": idx} - PermissionViewRoleBaseModel(payload).save() + PermissionViewRoleModel(payload).save() role = self.model.query.get(idx) - permission = PermissionViewRoleBaseModel.query.filter_by(role_id=idx).first() + permission = PermissionViewRoleModel.query.filter_by(role_id=idx).first() self.assertIsNotNone(role) self.assertIsNotNone(permission) @@ -458,7 +458,7 @@ def test_permission_delete_cascade(self): role.delete() role = self.model.query.get(idx) - permission = PermissionViewRoleBaseModel.query.filter_by(role_id=idx).first() + permission = PermissionViewRoleModel.query.filter_by(role_id=idx).first() self.assertIsNone(role) self.assertIsNone(permission) diff --git a/libs/core/cornflow_core/tests/test_schema_from_models.py b/cornflow-server/cornflow/tests/unit/test_schema_from_models.py similarity index 84% rename from libs/core/cornflow_core/tests/test_schema_from_models.py rename to cornflow-server/cornflow/tests/unit/test_schema_from_models.py index d6d589961..8e39ab04c 100644 --- a/libs/core/cornflow_core/tests/test_schema_from_models.py +++ b/cornflow-server/cornflow/tests/unit/test_schema_from_models.py @@ -4,7 +4,7 @@ from click.testing import CliRunner -from cornflow_core.cli.schema_from_models import schema_from_models +from cornflow.cli import cli path_to_tests = os.path.dirname(os.path.abspath(__file__)) @@ -12,7 +12,7 @@ class SchemaFromModelsTests(unittest.TestCase): def setUp(self): super().setUp() - self.models_path = self._get_path("./data/models") + self.models_path = self._get_path("../data/models") self.output_path = self._get_path(os.path.join(os.getcwd(), "test_output.json")) @staticmethod @@ -32,11 +32,17 @@ def tearDown(self): def test_base(self): runner = CliRunner() result = runner.invoke( - schema_from_models, ["-p", self.models_path, "-o", self.output_path] + cli, + [ + "schemas", + "schema_from_models", + "-p", + self.models_path, + "-o", + self.output_path + ] ) - print(result.output) - self.assertEqual(result.exit_code, 0) schema = self.import_schema(self._get_path(self.output_path)) @@ -98,8 +104,17 @@ def test_base(self): def test_ignore(self): runner = CliRunner() result = runner.invoke( - schema_from_models, - ["-p", self.models_path, "-o", self.output_path, "-i", "instance.py"], + cli, + [ + "schemas", + "schema_from_models", + "-p", + self.models_path, + "-o", + self.output_path, + "-i", + "instance.py" + ] ) self.assertEqual(result.exit_code, 0) diff --git a/cornflow-server/cornflow/tests/unit/test_sign_up.py b/cornflow-server/cornflow/tests/unit/test_sign_up.py index 603878a6f..143cd5b2b 100644 --- a/cornflow-server/cornflow/tests/unit/test_sign_up.py +++ b/cornflow-server/cornflow/tests/unit/test_sign_up.py @@ -12,7 +12,7 @@ from cornflow.app import create_app from cornflow.models import UserModel, UserRoleModel from cornflow.shared.const import PLANNER_ROLE -from cornflow_core.shared import db +from cornflow.shared import db from cornflow.tests.const import SIGNUP_URL diff --git a/cornflow-server/cornflow/tests/unit/test_tables.py b/cornflow-server/cornflow/tests/unit/test_tables.py index c60d0b035..bd23046b9 100644 --- a/cornflow-server/cornflow/tests/unit/test_tables.py +++ b/cornflow-server/cornflow/tests/unit/test_tables.py @@ -9,9 +9,9 @@ # Import from internal modules from cornflow.app import create_app from cornflow.commands.access import access_init_command +from cornflow.models import UserRoleModel +from cornflow.shared import db from cornflow.shared.const import ADMIN_ROLE, SERVICE_ROLE -from cornflow.models import UserModel, UserRoleModel -from cornflow_core.shared import db from cornflow.tests.const import LOGIN_URL, SIGNUP_URL, TABLES_URL diff --git a/cornflow-server/cornflow/tests/unit/test_token.py b/cornflow-server/cornflow/tests/unit/test_token.py index f196305ce..63750a1e5 100644 --- a/cornflow-server/cornflow/tests/unit/test_token.py +++ b/cornflow-server/cornflow/tests/unit/test_token.py @@ -8,7 +8,7 @@ # Import from internal modules from cornflow.models import UserModel -from cornflow_core.shared import db +from cornflow.shared import db from cornflow.tests.custom_test_case import CheckTokenTestCase from cornflow.tests.const import LOGIN_URL diff --git a/cornflow-server/cornflow/tests/unit/test_users.py b/cornflow-server/cornflow/tests/unit/test_users.py index b47eaff36..c40fa1a72 100644 --- a/cornflow-server/cornflow/tests/unit/test_users.py +++ b/cornflow-server/cornflow/tests/unit/test_users.py @@ -15,7 +15,7 @@ ) from cornflow.shared.const import ADMIN_ROLE, PLANNER_ROLE, SERVICE_ROLE, VIEWER_ROLE -from cornflow_core.shared import db +from cornflow.shared import db from cornflow.tests.const import ( CASE_PATH, CASE_URL, diff --git a/cornflow-server/requirements.txt b/cornflow-server/requirements.txt index 991ac7819..04eee0a24 100644 --- a/cornflow-server/requirements.txt +++ b/cornflow-server/requirements.txt @@ -1,11 +1,13 @@ alembic==1.9.2 apispec<=6.2.0 click<=8.1.3 -cornflow-core<=0.1.10 -cornflow-client<=1.0.12 +cornflow-client<=1.0.13 cryptography<=39.0.2 -Flask==2.1.3 +disposable-email-domains>=0.0.86 +Flask==2.3.2 flask-apispec<=0.11.4 +Flask-Bcrypt<=1.0.1 +Flask-Compress<=1.13 flask-cors<=3.0.10 flask-inflate<=0.3 Flask-Migrate<=4.0.4 @@ -15,9 +17,13 @@ gevent<=22.10.2 greenlet<=2.0.2 gunicorn<=20.1.0 jsonpatch<=1.32 +ldap3<=2.9.1 marshmallow<=3.19.0 PuLP<=2.7.0 psycopg2<=2.95 PyJWT<=2.6.0 +pytups>=0.86.2 +requests<=2.29.0 SQLAlchemy==1.3.21 -Werkzeug<=2.2.3 +webargs<=8.2.0 +Werkzeug<=2.3.3 diff --git a/cornflow-server/setup.py b/cornflow-server/setup.py index 96eb7871f..61822bdba 100644 --- a/cornflow-server/setup.py +++ b/cornflow-server/setup.py @@ -9,7 +9,7 @@ setuptools.setup( name="cornflow", - version="1.0.4", + version="1.0.5", author="baobab soluciones", author_email="cornflow@baobabsoluciones.es", description="Cornflow is an open source multi-solver optimization server with a REST API built using flask.", diff --git a/libs/client/changelog.rst b/libs/client/changelog.rst index 68b4ddb55..202ccd337 100644 --- a/libs/client/changelog.rst +++ b/libs/client/changelog.rst @@ -1,3 +1,12 @@ +version 1.0.13 +--------------- + +- released: 2023-05-04 +- description: bugfix on error handling in dag solving workflow +- changelog: + - bugfix on error handling in dag solving workflow + - calls to cornflow now use the raw client. + version 1.0.12 --------------- diff --git a/libs/client/cornflow_client/airflow/dag_utilities.py b/libs/client/cornflow_client/airflow/dag_utilities.py index df37725a4..8b803dd4d 100644 --- a/libs/client/cornflow_client/airflow/dag_utilities.py +++ b/libs/client/cornflow_client/airflow/dag_utilities.py @@ -99,7 +99,7 @@ def try_to_save_error(client, exec_id, state=-1): Attempt at saving that the execution failed """ try: - client.put_api_for_id("dag/", id=exec_id, payload=dict(state=state)) + client.raw.put_api_for_id("dag/", id=exec_id, payload=dict(state=state)) except Exception as e: print(f"An exception trying to register the failed status: {e}") @@ -110,13 +110,15 @@ def try_to_save_airflow_log(client, exec_id, ti, base_log_folder): f"{ti.dag_id}", f"{ti.task_id}", f"{ti.run_id}".replace("manual__", "").replace("scheduled__", ""), - f"{ti.try_number}.log" + f"{ti.try_number}.log", ) if os.path.exists(log_file): - with open(log_file, 'r') as fd: + with open(log_file, "r") as fd: log_file_txt = fd.read() try: - client.put_api_for_id("dag/", id=exec_id, payload=dict(log_text=log_file_txt)) + client.raw.put_api_for_id( + "dag/", id=exec_id, payload=dict(log_text=log_file_txt) + ) except Exception as e: print(f"An exception occurred while trying to register airflow log: {e}") diff --git a/libs/client/cornflow_client/core/read_tools.py b/libs/client/cornflow_client/core/read_tools.py index bdd90062e..5e1c0bef5 100644 --- a/libs/client/cornflow_client/core/read_tools.py +++ b/libs/client/cornflow_client/core/read_tools.py @@ -9,12 +9,15 @@ import numbers -def read_excel(path: str, param_tables_names: list = None) -> dict: +def read_excel( + path: str, param_tables_names: list = None, preserve_types=False +) -> dict: """ - Read an entire excel file. + Read an entire Excel file. - :param path: path of the excel file - :param param_tables_names: names of the parameter tables + :param path: path of the Excel file. + :param param_tables_names: names of the parameter tables. + :param preserve_types: if true pandas will not interpret type. :return: a dict with a list of dict (records format) for each table. """ is_xl_type(path) @@ -29,7 +32,9 @@ def read_excel(path: str, param_tables_names: list = None) -> dict: except (ModuleNotFoundError, ImportError): raise Exception("You must install pandas package to use this method") - data = pd.read_excel(path, sheet_name=None) + dtype = "object" if preserve_types else None + + data = pd.read_excel(path, sheet_name=None, dtype=dtype) data_tables = { name: TupList(content.to_dict(orient="records")).vapply( diff --git a/libs/client/cornflow_client/schema/tools.py b/libs/client/cornflow_client/schema/tools.py index 3b34d7e4a..de4dc0024 100644 --- a/libs/client/cornflow_client/schema/tools.py +++ b/libs/client/cornflow_client/schema/tools.py @@ -34,31 +34,153 @@ def get_empty_schema(properties=None, solvers=None): return schema -def schema_from_excel(path_in, param_tables=None, path_out=None): +def clean_none(dic): """ - Create a jsonschema based on an excel data file. + Remove empty values from a dict - :param path_in: path of the excel file + :param dic: a dict + :return: the filtered dict + """ + remove = ["NaT", "NaN", None] + return {k: v for k, v in dic.items() if not v in remove} + + +def check_fk(fk_dic): + """ + Check the format of foreign keys + + :param fk_dic: a dict of foreign keys values + :return: None (raise an error if problems are detected) + """ + problems = [] + for table, fk in fk_dic.items(): + for k, v in fk.items(): + if "." not in v: + problems += [(table, k, v)] + if len(problems): + message = ( + f'Foreign key format should be "table.key". ' + f"Problem detected for the following table, keys and values: {problems}" + ) + raise ValueError(message) + + +def schema_from_excel( + path_in, + param_tables=None, + path_out=None, + fk=False, + date_format=False, + path_methods=None, + path_access=None, +): + """ + Create a jsonschema based on an Excel data file. + + :param path_in: path of the Excel file :param param_tables: array containing the names of the parameter tables :param path_out: path where to save the json schema as a json file. + :param fk: True if foreign key are described in the second row. + :param date_format: if format is true special format (like date, time or datetime) are specified in the third row. + :param path_methods: path where to save the methods dict as a json file + :param path_access: path where to save the access dict as a json file :return: the jsonschema """ if not param_tables: param_tables = [] - xl_data = read_excel(path_in, param_tables) - data = {k: str_columns(v) if isinstance(v, list) else v for k, v in xl_data.items()} - + xl_data = read_excel(path_in, param_tables, preserve_types=True) + + # process and remove special tables + if "endpoints_methods" in xl_data: + endpoints_methods = { + e["endpoint"]: [k for k, v in e.items() if v and k != "endpoint"] + for e in xl_data["endpoints_methods"] + } + del xl_data["endpoints_methods"] + else: + endpoints_methods = None + + if "endpoints_access" in xl_data: + endpoints_access = { + e["endpoint"]: [k for k, v in e.items() if v and k != "endpoint"] + for e in xl_data["endpoints_access"] + } + del xl_data["endpoints_access"] + else: + endpoints_access = None + + # process foreign keys + next_row = -1 + if fk: + next_row += 1 + fk_values = { + k: clean_none(v[next_row]) + for k, v in xl_data.items() + if isinstance(v, list) + } + check_fk(fk_values) + else: + fk_values = {} + + if date_format: + next_row += 1 + format_values = { + k: clean_none(v[next_row]) + for k, v in xl_data.items() + if isinstance(v, list) + } + else: + format_values = {} + next_row += 1 + data = { + k: str_columns(v[next_row:]) if isinstance(v, list) else v + for k, v in xl_data.items() + } + + # create the json schema class InstSol(InstanceSolutionCore): schema = {} instance = InstSol(data) schema = instance.generate_schema() + add_details("foreign_key", fk_values, schema) + add_details("format", format_values, schema) + fix_required(schema) + # Save json files if path_out is not None: with open(path_out, "w") as f: json.dump(schema, f, indent=4, sort_keys=False) + if path_methods is not None: + with open(path_methods, "w") as f: + json.dump(endpoints_methods, f, indent=4, sort_keys=False) + if path_access is not None: + with open(path_access, "w") as f: + json.dump(endpoints_access, f, indent=4, sort_keys=False) - return schema + return schema, endpoints_methods, endpoints_access + + +def add_details(name, details, schema): + """ + Add a detail attribute to a json schema property. + Example: + add_details("foreign_key", {first_table:{"name":"other_table.name"}}, schema) + # generate: + "name": { + "type": "string" + "foreign_key": "other_table.name" + } + + :param name: name of the attribute to add + :param details: dict of dict in format {table:{column_name:value}} + :param schema: schema to update + :return: None + """ + for table, val in details.items(): + for k, v in val.items(): + if v is not None: + schema["properties"][table]["items"]["properties"][k].update({name: v}) def str_key(dic): @@ -80,3 +202,18 @@ def str_columns(table): :return: the modified list of dict """ return [str_key(d) for d in table] + + +def fix_required(schema): + """ + Fix required property in schema: if a field is allowed null, it is not required + + :param schema: the json schema + :return: None + """ + for table_name, table in schema["properties"].items(): + required = [] + for field_name, field in table["items"]["properties"].items(): + if "null" not in field["type"]: + required += [field_name] + table["items"]["required"] = required diff --git a/libs/client/cornflow_client/tests/data/endpoints_access.json b/libs/client/cornflow_client/tests/data/endpoints_access.json new file mode 100644 index 000000000..eadeb1e19 --- /dev/null +++ b/libs/client/cornflow_client/tests/data/endpoints_access.json @@ -0,0 +1,15 @@ +{ + "table1": [ + "ADMIN_ROLE", + "SERVICE_ROLE" + ], + "table2": [ + "SERVICE_ROLE" + ], + "table3": [ + "VIEWER_ROLE", + "PLANNER_ROLE", + "ADMIN_ROLE", + "SERVICE_ROLE" + ] +} \ No newline at end of file diff --git a/libs/client/cornflow_client/tests/data/endpoints_methods.json b/libs/client/cornflow_client/tests/data/endpoints_methods.json new file mode 100644 index 000000000..0560cf35f --- /dev/null +++ b/libs/client/cornflow_client/tests/data/endpoints_methods.json @@ -0,0 +1,20 @@ +{ + "table1": [ + "get_list", + "post_list", + "get_detail", + "put_detail", + "patch_detail", + "delete_detail" + ], + "table2": [ + "get_list", + "get_detail" + ], + "table3": [ + "get_list", + "post_list", + "post_bulk", + "put_bulk" + ] +} \ No newline at end of file diff --git a/libs/client/cornflow_client/tests/data/schema_with_fk.json b/libs/client/cornflow_client/tests/data/schema_with_fk.json new file mode 100644 index 000000000..7bf19bbf2 --- /dev/null +++ b/libs/client/cornflow_client/tests/data/schema_with_fk.json @@ -0,0 +1,94 @@ +{ + "$schema": "http://json-schema.org/schema#", + "type": "object", + "properties": { + "table1": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "col1": { + "type": [ + "null", + "string" + ] + }, + "col2": { + "type": "number" + }, + "col3": { + "type": "boolean" + } + }, + "required": [ + "id", + "col2", + "col3" + ] + } + }, + "table2": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "col1": { + "type": [ + "null", + "string" + ] + }, + "col2": { + "type": [ + "integer", + "string" + ] + }, + "date": { + "type": "string" + } + }, + "required": [ + "id", + "col2", + "date" + ] + } + }, + "table3": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id_t1": { + "type": "integer", + "foreign_key": "table1.id" + }, + "id_t2": { + "type": "integer", + "foreign_key": "table2.id" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "id_t1", + "id_t2", + "value" + ] + } + } + }, + "required": [ + "table1", + "table2", + "table3" + ] +} \ No newline at end of file diff --git a/libs/client/cornflow_client/tests/data/schema_without_fk.json b/libs/client/cornflow_client/tests/data/schema_without_fk.json new file mode 100644 index 000000000..d4f93f66f --- /dev/null +++ b/libs/client/cornflow_client/tests/data/schema_without_fk.json @@ -0,0 +1,92 @@ +{ + "$schema": "http://json-schema.org/schema#", + "type": "object", + "properties": { + "table1": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "col1": { + "type": [ + "null", + "string" + ] + }, + "col2": { + "type": "integer" + }, + "col3": { + "type": "boolean" + } + }, + "required": [ + "id", + "col2", + "col3" + ] + } + }, + "table2": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "col1": { + "type": [ + "null", + "string" + ] + }, + "col2": { + "type": [ + "integer", + "string" + ] + }, + "date": { + "type": "string" + } + }, + "required": [ + "id", + "col2", + "date" + ] + } + }, + "table3": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id_t1": { + "type": "integer" + }, + "id_t2": { + "type": "integer" + }, + "value": { + "type": "integer" + } + }, + "required": [ + "id_t1", + "id_t2", + "value" + ] + } + } + }, + "required": [ + "table1", + "table2", + "table3" + ] +} \ No newline at end of file diff --git a/libs/client/cornflow_client/tests/data/xl_with_access.xlsx b/libs/client/cornflow_client/tests/data/xl_with_access.xlsx new file mode 100644 index 000000000..4ef65fb30 Binary files /dev/null and b/libs/client/cornflow_client/tests/data/xl_with_access.xlsx differ diff --git a/libs/client/cornflow_client/tests/data/xl_with_fk.xlsx b/libs/client/cornflow_client/tests/data/xl_with_fk.xlsx new file mode 100644 index 000000000..48b3b96f3 Binary files /dev/null and b/libs/client/cornflow_client/tests/data/xl_with_fk.xlsx differ diff --git a/libs/client/cornflow_client/tests/data/xl_with_methods.xlsx b/libs/client/cornflow_client/tests/data/xl_with_methods.xlsx new file mode 100644 index 000000000..cb9f65acb Binary files /dev/null and b/libs/client/cornflow_client/tests/data/xl_with_methods.xlsx differ diff --git a/libs/client/cornflow_client/tests/data/xl_without_fk.xlsx b/libs/client/cornflow_client/tests/data/xl_without_fk.xlsx new file mode 100644 index 000000000..cb1f1183d Binary files /dev/null and b/libs/client/cornflow_client/tests/data/xl_without_fk.xlsx differ diff --git a/libs/client/cornflow_client/tests/unit/test_schema_from_excel.py b/libs/client/cornflow_client/tests/unit/test_schema_from_excel.py new file mode 100644 index 000000000..93215a17f --- /dev/null +++ b/libs/client/cornflow_client/tests/unit/test_schema_from_excel.py @@ -0,0 +1,41 @@ +import json +import os +from unittest import TestCase +from cornflow_client.schema.tools import schema_from_excel +from cornflow_client.core.tools import load_json + + +class TestSchemaFromExcel(TestCase): + def setUp(self) -> None: + self.root_data = os.path.join(os.path.dirname(__file__), "../data") + self.xl_with_fk = self.get_data_file("xl_with_fk.xlsx") + self.xl_without_fk = self.get_data_file("xl_without_fk.xlsx") + self.xl_with_methods = self.get_data_file("xl_with_methods.xlsx") + self.xl_with_access = self.get_data_file("xl_with_access.xlsx") + self.schema_with_fk = self.get_data_file("schema_with_fk.json") + self.schema_without_fk = self.get_data_file("schema_without_fk.json") + self.endpoints_methods = self.get_data_file("endpoints_methods.json") + self.endpoints_access = self.get_data_file("endpoints_access.json") + + def get_data_file(self, filename): + return os.path.join(self.root_data, filename) + + def test_schema_with_fk(self): + schema, methods, access = schema_from_excel(self.xl_with_fk, fk=True) + expected = load_json(self.schema_with_fk) + self.assertEqual(schema, expected) + + def test_schema_without_fk(self): + schema, methods, access = schema_from_excel(self.xl_without_fk, fk=False) + expected = load_json(self.schema_without_fk) + self.assertEqual(schema, expected) + + def test_endpoints_methods(self): + schema, methods, access = schema_from_excel(self.xl_with_methods, fk=True) + expected = load_json(self.endpoints_methods) + self.assertEqual(methods, expected) + + def test_endpoints_access(self): + schema, methods, access = schema_from_excel(self.xl_with_access, fk=True) + expected = load_json(self.endpoints_access) + self.assertEqual(access, expected) \ No newline at end of file diff --git a/libs/client/setup.py b/libs/client/setup.py index e4b4eecdb..83c0568e4 100644 --- a/libs/client/setup.py +++ b/libs/client/setup.py @@ -12,7 +12,7 @@ setuptools.setup( name="cornflow-client", - version="1.0.12", + version="1.0.13", author="baobab soluciones", author_email="sistemas@baobabsoluciones.es", description="Client to connect to a cornflow server", diff --git a/libs/core/LICENSE b/libs/core/LICENSE deleted file mode 100644 index bb37f6690..000000000 --- a/libs/core/LICENSE +++ /dev/null @@ -1,190 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - Copyright 2020 baobab soluciones S.L. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. \ No newline at end of file diff --git a/libs/core/MANIFEST.in b/libs/core/MANIFEST.in deleted file mode 100644 index d9bebba87..000000000 --- a/libs/core/MANIFEST.in +++ /dev/null @@ -1,4 +0,0 @@ -include LICENSE -include MANIFEST.in -include README.rst -include setup.py diff --git a/libs/core/cornflow_core/__init__.py b/libs/core/cornflow_core/__init__.py deleted file mode 100644 index 5f126f4bf..000000000 --- a/libs/core/cornflow_core/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Main file""" diff --git a/libs/core/cornflow_core/cli/__init__.py b/libs/core/cornflow_core/cli/__init__.py deleted file mode 100644 index 60da36084..000000000 --- a/libs/core/cornflow_core/cli/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -""" -Init file for the cli module -""" diff --git a/libs/core/cornflow_core/cli/generate_from_schema.py b/libs/core/cornflow_core/cli/generate_from_schema.py deleted file mode 100644 index 06df06bfe..000000000 --- a/libs/core/cornflow_core/cli/generate_from_schema.py +++ /dev/null @@ -1,86 +0,0 @@ -""" -File that implements the generate from schema cli command -""" -import click - -from cornflow_core.cli.tools.api_generator import APIGenerator - -METHOD_OPTIONS = [ - "get_list", - "post_list", - "get_detail", - "put_detail", - "patch_detail", - "delete_detail", - "post_bulk", - "put_bulk" -] - - -@click.command(name="generate_from_schema") -@click.option( - "--path", "-p", type=str, help="The absolute path to the JSONSchema", required=True -) -@click.option("--app-name", "-a", type=str, help="The name of the application") -@click.option( - "--output-path", - "-o", - type=str, - default="output", - help="The output path", - required=False, -) -@click.option( - "--remove_methods", - "-r", - type=click.Choice(METHOD_OPTIONS, case_sensitive=False), - help="Methods that will NOT be added to the new endpoints", - multiple=True, - required=False, -) -@click.option( - "--one", - type=str, - help="If your schema describes only one table, use this option to indicate the name of the table", - required=False, -) -def generate_from_schema(path, app_name, output_path, remove_methods, one): - """ - This method is executed for the command and creates all the files for the REST API from the provided JSONSchema - - :param str path: the path to the JSONSchema file - :param str app_name: the name of the application - :param str output_path: the output path - :param tuple remove_methods: the methods that will not be added to the new endpoints - :param str one: if your schema describes only one table, use this option to indicate the name of the table - :return: a click status code - :rtype: int - """ - path = path.replace("\\", "/") - output = None - if output_path != "output": - output = output_path.replace("\\", "/") - - if remove_methods is not None: - methods_to_add = list(set(METHOD_OPTIONS) - set(remove_methods)) - else: - methods_to_add = [] - - name_table = None - if one: - name_table = one - - click.echo("Generating REST API components from the provided JSONSchema") - click.echo(f"The path to the JSONSchema is {path}") - click.echo(f"The app_name is {app_name}") - click.echo(f"The output_path is {output}") - click.echo(f"The methods to add is {methods_to_add}") - click.echo(f"The name_table is {name_table}") - - APIGenerator( - path, - app_name=app_name, - output_path=output_path, - options=methods_to_add, - name_table=name_table, - ).main() diff --git a/libs/core/cornflow_core/cli/schema_from_models.py b/libs/core/cornflow_core/cli/schema_from_models.py deleted file mode 100644 index f2b8b5cb5..000000000 --- a/libs/core/cornflow_core/cli/schema_from_models.py +++ /dev/null @@ -1,57 +0,0 @@ -import click - -from cornflow_core.cli.tools.schema_generator import SchemaGenerator - - -@click.command(name="schema_from_models") -@click.option( - "--path", - "-p", - type=str, - help="The absolute path to folder containing the models", - required=True, -) -@click.option("--output-path", "-o", type=str, help="The output path", required=False) -@click.option( - "--ignore-files", - "-i", - type=str, - help="Files that will be ignored (with the .py extension). " - "__init__.py files are automatically ignored. Ex: 'instance.py'", - multiple=True, - required=False, -) -@click.option( - "--leave-bases/--no-leave-bases", - "-l/-nl", - default=False, - help="Use this option to leave the bases classes BaseDataModel, " - "EmptyModel and TraceAttributes in the schema. By default, they will be deleted", -) -def schema_from_models(path, output_path, ignore_files, leave_bases): - """ - - :param str path: the path to the folder that contains the models - :param output_path: the output path where the JSONSchema should be placed - :param str ignore_files: files to be ignored. - :param str leave_bases: if the JSONSchema should have abstract classes used as the base for other clases. - :return: a click status code - :rtype: int - """ - path = path.replace("\\", "/") - output = None - if output_path: - output = output_path.replace("\\", "/") - - if ignore_files: - ignore_files = list(ignore_files) - - click.echo("Generating JSONSchema file from the REST API") - click.echo(f"The path to the JSONSchema is {path}") - click.echo(f"The output_path is {output}") - click.echo(f"The ignore_files is {ignore_files}") - click.echo(f"The leave_bases is {leave_bases}") - - SchemaGenerator( - path, output_path=output, ignore_files=ignore_files, leave_bases=leave_bases - ).main() diff --git a/libs/core/cornflow_core/cli/tools/api_generator.py b/libs/core/cornflow_core/cli/tools/api_generator.py deleted file mode 100644 index b6d6b4e3f..000000000 --- a/libs/core/cornflow_core/cli/tools/api_generator.py +++ /dev/null @@ -1,424 +0,0 @@ -""" -This file has the class that creates the new API -""" -import json -import os - -from .endpoint_tools import EndpointGenerator -from .models_tools import ModelGenerator, model_shared_imports -from .schemas_tools import SchemaGenerator, schemas_imports -from .tools import generate_class_def - - -class APIGenerator: - """ - This class is used to create the new API - """ - - def __init__( - self, - schema_path, - app_name, - output_path=None, - options=None, - name_table=None, - ): - self.path = schema_path - self.name = app_name - self.options = options - if not self.options: - self.options = ["all"] - self.schema = self.import_schema() - if self.schema["type"] == "array" and not name_table: - self.schema = {"properties": {"data": self.schema}} - elif self.schema["type"] == "array" and name_table: - self.schema = {"properties": {name_table: self.schema}} - elif self.schema["type"] != "array" and name_table: - print( - "The JSONSchema does not contain only one table. The --one option will be ignored" - ) - self.output_path = output_path or "output" - self.model_path = os.path.join(self.output_path, "models") - self.endpoint_path = os.path.join(self.output_path, "endpoints") - self.schema_path = os.path.join(self.output_path, "schemas") - - def import_schema(self) -> dict: - """ - This method imports the JSONSchema file - - :return: the read schema - :rtype: dict - """ - with open(self.path, "r") as fd: - schema = json.load(fd) - return schema - - def prepare_dirs(self): - """ - This method creates all the folders needed - - :return: None - :rtype: None - """ - if not os.path.isdir(self.output_path): - os.mkdir(self.output_path) - if not os.path.isdir(self.model_path): - os.mkdir(self.model_path) - - init_path = os.path.join(self.model_path, "__init__.py") - with open(init_path, "w") as file: - file.write(f'"""\nThis file exposes the models\n"""\n') - - if not os.path.isdir(self.endpoint_path): - os.mkdir(self.endpoint_path) - - init_path = os.path.join(self.endpoint_path, "__init__.py") - with open(init_path, "w") as file: - file.write(f'"""\nThis file exposes the endpoints\n"""\n') - - if not os.path.isdir(self.schema_path): - os.mkdir(self.schema_path) - - init_path = os.path.join(self.schema_path, "__init__.py") - with open(init_path, "w") as file: - file.write(f'"""\nThis file exposes the schemas\n"""\n') - - def main(self): - """ - This is the main method that gets executed - - :return: None - :rtype: None - """ - self.prepare_dirs() - tables = self.schema["properties"].keys() - for table in tables: - if self.schema["properties"][table]["type"] != "array": - print( - f'\nThe table "{table}" does not have the correct format. ' - f"The structures will not be generated for this table" - ) - continue - model_name = self.new_model(table) - schemas_names = self.new_schemas(table) - self.new_endpoint(table, model_name, schemas_names) - - init_file = os.path.join(self.endpoint_path, "__init__.py") - with open(init_file, "a") as file: - file.write("\nresources = []\n") - print( - f"The generated files will be stored in {os.path.join(os.getcwd(), self.output_path)}\n" - ) - return 0 - - def new_model(self, table_name): - """ - This method takes a table name and creates a flask database model with the fields os said table - - :param str table_name: the name of the table to create - :return: the name of the created model - :rtype: str - """ - if self.name is None: - filename = os.path.join(self.model_path, f"{table_name}.py") - class_name = self.snake_to_camel(table_name + "_model") - else: - filename = os.path.join(self.model_path, f"{self.name}_{table_name}.py") - class_name = self.snake_to_camel(self.name + "_" + table_name + "_model") - parents_class = ["TraceAttributesModel"] - mg = ModelGenerator( - class_name, self.schema, parents_class, table_name, self.name - ) - with open(filename, "w") as fd: - fd.write(model_shared_imports) - fd.write("\n") - fd.write(generate_class_def(class_name, parents_class)) - fd.write(mg.generate_model_description()) - fd.write("\n") - fd.write(mg.generate_table_name()) - fd.write("\n") - fd.write(mg.generate_model_fields()) - fd.write("\n") - fd.write(mg.generate_model_init()) - fd.write("\n") - fd.write(mg.generate_model_repr_str()) - fd.write("\n") - - init_file = os.path.join(self.model_path, "__init__.py") - - with open(init_file, "a") as file: - if self.name is None: - file.write(f"from .{table_name} import {class_name}\n") - else: - file.write(f"from .{self.name}_{table_name} import {class_name}\n") - - return class_name - - def new_schemas(self, table_name: str) -> dict: - """ - This method takes a table name and creates a flask database model with the fields os said table - - :param str table_name: the name of the table to create - :return: the dictionary with the names of the schemas created - :rtype: dict - """ - if self.name is None: - filename = os.path.join(self.schema_path, table_name + ".py") - class_name_one = self.snake_to_camel(table_name + "_response") - class_name_edit = self.snake_to_camel(table_name + "_edit_request") - class_name_post = self.snake_to_camel(table_name + "_post_request") - class_name_post_bulk = self.snake_to_camel(table_name + "_post_bulk_request") - class_name_put_bulk = self.snake_to_camel(table_name + "_put_bulk_request") - class_name_put_bulk_one = class_name_put_bulk + "One" - else: - filename = os.path.join( - self.schema_path, self.name + "_" + table_name + ".py" - ) - class_name_one = self.snake_to_camel( - self.name + "_" + table_name + "_response" - ) - class_name_edit = self.snake_to_camel( - self.name + "_" + table_name + "_edit_request" - ) - class_name_post = self.snake_to_camel( - self.name + "_" + table_name + "_post_request" - ) - class_name_post_bulk = self.snake_to_camel( - self.name + "_" + table_name + "_post_bulk_request" - ) - class_name_put_bulk = self.snake_to_camel( - self.name + "_" + table_name + "_put_bulk_request" - ) - class_name_put_bulk_one = class_name_put_bulk + "One" - - parents_class = ["Schema"] - partial_schema = self.schema["properties"][table_name]["items"] - sg = SchemaGenerator(partial_schema, table_name, self.name) - with open(filename, "w") as fd: - fd.write(sg.generate_schema_file_description()) - fd.write(schemas_imports) - fd.write("\n") - fd.write(generate_class_def(class_name_edit, parents_class)) - fd.write(sg.generate_edit_schema()) - fd.write("\n\n") - - fd.write(generate_class_def(class_name_post, parents_class)) - fd.write(sg.generate_post_schema()) - fd.write("\n\n") - - fd.write(generate_class_def(class_name_post_bulk, parents_class)) - fd.write(sg.generate_bulk_schema(class_name_post)) - fd.write("\n\n") - - parents_class = [class_name_edit] - fd.write(generate_class_def(class_name_put_bulk_one, parents_class)) - fd.write(sg.generate_put_bulk_schema_one()) - fd.write("\n\n") - - parents_class = ["Schema"] - fd.write(generate_class_def(class_name_put_bulk, parents_class)) - fd.write(sg.generate_bulk_schema(class_name_put_bulk_one)) - fd.write("\n\n") - - parents_class = [class_name_post] - fd.write(generate_class_def(class_name_one, parents_class)) - fd.write(sg.generate_schema()) - - init_file = os.path.join(self.schema_path, "__init__.py") - with open(init_file, "a") as file: - if self.name is None: - file.write( - f"from .{table_name} import {class_name_one}, " - f"{class_name_edit}, {class_name_post}, {class_name_post_bulk}, {class_name_put_bulk}\n" - ) - else: - file.write( - f"from .{self.name}_{table_name} import {class_name_one}, " - f"{class_name_edit}, {class_name_post}, {class_name_post_bulk}, {class_name_put_bulk}\n" - ) - - return { - "one": class_name_one, - "editRequest": class_name_edit, - "postRequest": class_name_post, - "postBulkRequest": class_name_post_bulk, - "putBulkRequest": class_name_put_bulk - } - - def new_endpoint( - self, table_name: str, model_name: str, schemas_names: dict - ) -> None: - """ - This method takes a table name, a model_name and the names of the marshmallow schemas and - creates a flask endpoint with the methods passed - - :param str table_name: the name of the table to create - :param str model_name: the name of the model that have been created - :param dict schemas_names: the names of the schemas that have been created - :return: None - :rtype: None - """ - if self.name is None: - filename = os.path.join(self.endpoint_path, table_name + ".py") - class_name_all = self.snake_to_camel(table_name + "_endpoint") - class_name_details = self.snake_to_camel(table_name + "_details_endpoint") - class_name_bulk = self.snake_to_camel(table_name + "_bulk_endpoint") - else: - filename = os.path.join( - self.endpoint_path, self.name + "_" + table_name + ".py" - ) - class_name_all = self.snake_to_camel( - self.name + "_" + table_name + "_endpoint" - ) - class_name_details = self.snake_to_camel( - self.name + "_" + table_name + "_details_endpoint" - ) - class_name_bulk = self.snake_to_camel( - self.name + "_" + table_name + "_bulk_endpoint" - ) - - parents_class = ["BaseMetaResource"] - roles_with_access = ["SERVICE_ROLE"] - eg = EndpointGenerator(table_name, self.name, model_name, schemas_names) - with open(filename, "w") as fd: - # Global - if any(m in self.options for m in ["get_list", "post_list", "all"]): - fd.write(eg.generate_endpoints_imports()) - fd.write("\n") - fd.write(generate_class_def(class_name_all, parents_class)) - fd.write(eg.generate_endpoint_description()) - fd.write("\n") - fd.write(f' ROLES_WITH_ACCESS = [{", ".join(roles_with_access)}]\n') - fd.write("\n") - fd.write(eg.generate_endpoint_init()) - fd.write("\n") - if "get_list" in self.options or "all" in self.options: - fd.write(eg.generate_endpoint_get_all()) - fd.write("\n") - if "post_list" in self.options or "all" in self.options: - fd.write(eg.generate_endpoint_post()) - fd.write("\n") - - fd.write("\n") - - if any( - m in self.options - for m in [ - "get_detail", - "put_detail", - "patch_detail", - "delete_detail", - "all", - ] - ): - # Details - fd.write(generate_class_def(class_name_details, parents_class)) - fd.write(eg.generate_endpoint_description()) - fd.write("\n") - fd.write(f' ROLES_WITH_ACCESS = [{", ".join(roles_with_access)}]\n') - fd.write("\n") - fd.write(eg.generate_endpoint_init()) - fd.write("\n") - if "get_detail" in self.options or "all" in self.options: - fd.write(eg.generate_endpoint_get_one()) - fd.write("\n") - if "put_detail" in self.options or "all" in self.options: - fd.write(eg.generate_endpoint_put()) - fd.write("\n") - if "patch_detail" in self.options or "all" in self.options: - fd.write(eg.generate_endpoint_patch()) - fd.write("\n") - if "delete_detail" in self.options or "all" in self.options: - fd.write(eg.generate_endpoint_delete_one()) - fd.write("\n") - - fd.write("\n") - - if any( - m in self.options - for m in [ - "post_bulk", - "put_bulk" - ] - ): - # Bulk post/put - fd.write(generate_class_def(class_name_bulk, parents_class)) - fd.write(eg.generate_endpoint_description()) - fd.write("\n") - fd.write(f' ROLES_WITH_ACCESS = [{", ".join(roles_with_access)}]\n') - fd.write("\n") - fd.write(eg.generate_endpoint_init()) - fd.write("\n") - if "post_bulk" in self.options or "all" in self.options: - fd.write(eg.generate_endpoint_post_bulk()) - fd.write("\n") - if "put_bulk" in self.options or "all" in self.options: - fd.write(eg.generate_endpoint_put_bulk()) - fd.write("\n") - - init_file = os.path.join(self.endpoint_path, "__init__.py") - with open(init_file, "a") as file: - if any(m in self.options for m in ["get_list", "post_list", "all"]): - if any( - m in self.options - for m in [ - "get_detail", - "put_detail", - "patch_detail", - "delete_detail", - "all", - ] - ): - if self.name is None: - file.write( - f"from .{table_name} import {class_name_all}, {class_name_details}\n" - ) - else: - file.write( - f"from .{self.name}_{table_name} import {class_name_all}, {class_name_details}\n" - ) - else: - if self.name is None: - file.write(f"from .{table_name} import {class_name_all}\n") - else: - file.write( - f"from .{self.name}_{table_name} import {class_name_all}\n" - ) - elif any( - m in self.options - for m in [ - "get_detail", - "put_detail", - "patch_detail", - "delete_detail", - "all", - ] - ): - if self.name is None: - file.write(f"from .{table_name} import {class_name_details}\n") - else: - file.write( - f"from .{self.name}_{table_name} import {class_name_details}\n" - ) - - elif any( - m in self.options - for m in ["post_bulk", "put_bulk"] - ): - if self.name is None: - file.write(f"from .{table_name} import {class_name_bulk}\n") - else: - file.write( - f"from .{self.name}_{table_name} import {class_name_bulk}\n" - ) - - @staticmethod - def snake_to_camel(name: str) -> str: - """ - This static method takes a name with underscores in it and changes it to camelCase - - :param str name: the name to mutate - :return: the mutated name - :rtype: str - """ - return "".join(word.title() for word in name.split("_")) diff --git a/libs/core/cornflow_core/cli/tools/tools.py b/libs/core/cornflow_core/cli/tools/tools.py deleted file mode 100644 index 1273787a5..000000000 --- a/libs/core/cornflow_core/cli/tools/tools.py +++ /dev/null @@ -1,3 +0,0 @@ -# Shared -def generate_class_def(class_name, parent_class): - return f'class {class_name}({", ".join(parent_class)}):\n' diff --git a/libs/core/cornflow_core/compress/__init__.py b/libs/core/cornflow_core/compress/__init__.py deleted file mode 100644 index 90c7445a9..000000000 --- a/libs/core/cornflow_core/compress/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -""" -Expose the decorator and method to register -""" -from .compress import compressed, init_compress diff --git a/libs/core/cornflow_core/constants/__init__.py b/libs/core/cornflow_core/constants/__init__.py deleted file mode 100644 index 959812f34..000000000 --- a/libs/core/cornflow_core/constants/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -""" -Expose the constants -""" -from .authentication import * -from .roles import * diff --git a/libs/core/cornflow_core/constants/authentication.py b/libs/core/cornflow_core/constants/authentication.py deleted file mode 100644 index f9e38d02e..000000000 --- a/libs/core/cornflow_core/constants/authentication.py +++ /dev/null @@ -1,23 +0,0 @@ -""" -THIS FILE CONTAINS THE CONSTANTS RELATED WITH THE AUTH PROCESS -""" - -# These codes and names are inherited from flask app-builder in order to have the same names and values -# as this library that is the base of airflow -AUTH_DB = 1 -AUTH_LDAP = 2 -AUTH_OAUTH = 4 -AUTH_OID = 0 - -# Providers of open ID: -OID_NONE = 0 -OID_AZURE = 1 -OID_GOOGLE = 2 - -# AZURE OPEN ID URLS -OID_AZURE_DISCOVERY_COMMON_URL = ( - "https://login.microsoftonline.com/common/.well-known/openid-configuration" -) -OID_AZURE_DISCOVERY_TENANT_URL = ( - "https://login.microsoftonline.com/{tenant_id}/.well-known/openid-configuration" -) diff --git a/libs/core/cornflow_core/constants/roles.py b/libs/core/cornflow_core/constants/roles.py deleted file mode 100644 index d1e4df9d1..000000000 --- a/libs/core/cornflow_core/constants/roles.py +++ /dev/null @@ -1,16 +0,0 @@ -""" -This file contains the constants related to the roles of the REST API. -""" -VIEWER_ROLE = 1 -PLANNER_ROLE = 2 -ADMIN_ROLE = 3 -SERVICE_ROLE = 4 - -ALL_DEFAULT_ROLES = [VIEWER_ROLE, PLANNER_ROLE, ADMIN_ROLE, SERVICE_ROLE] - -ROLES_MAP = { - PLANNER_ROLE: "planner", - VIEWER_ROLE: "viewer", - ADMIN_ROLE: "admin", - SERVICE_ROLE: "service", -} diff --git a/libs/core/cornflow_core/exceptions/__init__.py b/libs/core/cornflow_core/exceptions/__init__.py deleted file mode 100644 index 873e3d8c4..000000000 --- a/libs/core/cornflow_core/exceptions/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -""" -Exposes the exceptions and the method to register them -""" -from .exceptions import * diff --git a/libs/core/cornflow_core/messages/__init__.py b/libs/core/cornflow_core/messages/__init__.py deleted file mode 100644 index 88e2777ee..000000000 --- a/libs/core/cornflow_core/messages/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -""" -Expose methods -""" -from .email import get_email, get_password_recover_email, send_email_to diff --git a/libs/core/cornflow_core/models/__init__.py b/libs/core/cornflow_core/models/__init__.py deleted file mode 100644 index 4760061b4..000000000 --- a/libs/core/cornflow_core/models/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -""" -Exposes the different models defined -""" -# Import generic first -from .meta_models import EmptyBaseModel -from .meta_models import TraceAttributesModel - -# Import particular after -from .action import ActionBaseModel -from .permissions import PermissionViewRoleBaseModel -from .role import RoleBaseModel -from .user import UserBaseModel -from .user_role import UserRoleBaseModel -from .view import ViewBaseModel diff --git a/libs/core/cornflow_core/models/user.py b/libs/core/cornflow_core/models/user.py deleted file mode 100644 index 46a7a78c2..000000000 --- a/libs/core/cornflow_core/models/user.py +++ /dev/null @@ -1,236 +0,0 @@ -""" -This file contains the UserBaseModel -""" -import random -import string - -from cornflow_core.exceptions import InvalidCredentials -from cornflow_core.shared import ( - bcrypt, - db, - check_password_pattern, - check_email_pattern, -) -from .meta_models import TraceAttributesModel -from .user_role import UserRoleBaseModel - - -class UserBaseModel(TraceAttributesModel): - """ - Model class for the Users - It inherits from :class:`TraceAttributesModel` to have trace fields - - The :class:`UserBaseModel` has the following fields: - - - **id**: int, the primary key for the users, a integer value thar is auto incremented - - **first_name**: str, to store the first name of the user. Usually is an optional field. - - **last_name**: str, to store the last name of the user. Usually is and optional field. - - **username**: str, the username of the user, used for the log in. - - **password**: str, the hashed password stored on the database in the case that the authentication is done - with auth db. In other authentication methods this field is empty - - **email**: str, the email of the user, used to send emails in case of password recovery. - - **created_at**: datetime, the datetime when the user was created (in UTC). - This datetime is generated automatically, the user does not need to provide it. - - **updated_at**: datetime, the datetime when the user was last updated (in UTC). - This datetime is generated automatically, the user does not need to provide it. - - **deleted_at**: datetime, the datetime when the user was deleted (in UTC). - This field is used only if we deactivate instead of deleting the record. - This datetime is generated automatically, the user does not need to provide it. - """ - - __tablename__ = "users" - id = db.Column(db.Integer, primary_key=True, autoincrement=True) - first_name = db.Column(db.String(128), nullable=True) - last_name = db.Column(db.String(128), nullable=True) - username = db.Column(db.String(128), nullable=False, unique=True) - password = db.Column(db.String(128), nullable=True) - email = db.Column(db.String(128), nullable=False, unique=True) - - user_roles = db.relationship( - "UserRoleBaseModel", cascade="all,delete", backref="users" - ) - - @property - def roles(self): - """ - This property gives back the roles assigned to the user - """ - return {r.role.id: r.role.name for r in self.user_roles} - - def __init__(self, data): - super().__init__() - self.first_name = data.get("first_name") - self.last_name = data.get("last_name") - self.username = data.get("username") - # TODO: handle better None passwords that can be found when using ldap - check_pass, msg = check_password_pattern(data.get("password")) - if check_pass: - self.password = self.__generate_hash(data.get("password")) - else: - raise InvalidCredentials( - msg, - log_txt="Error while trying to create a new user. " + msg - ) - - check_email, msg = check_email_pattern(data.get("email")) - if check_email: - self.email = data.get("email") - else: - raise InvalidCredentials( - msg, - log_txt="Error while trying to create a new user. " + msg - ) - - def update(self, data): - """ - Updates the user information in the database - - :param dict data: the data to update the user - """ - # First we create the hash of the new password and then we update the object - new_password = data.get("password") - if new_password: - new_password = self.__generate_hash(new_password) - data["password"] = new_password - super().update(data) - - def comes_from_external_provider(self): - """ - Returns a boolean if the user comes from an external_provider or not - """ - return self.password is None - - @staticmethod - def __generate_hash(password): - """ - Method to generate the hash from the password. - - :param str password: the password given by the user . - :return: the hashed password. - :rtype: str - """ - if password is None: - return None - return bcrypt.generate_password_hash(password, rounds=10).decode("utf8") - - def check_hash(self, password): - """ - Method to check if the hash stored in the database is the same as the password given by the user - - :param str password: the password given by the user. - :return: if the password is the same or not. - :rtype: bool - """ - return bcrypt.check_password_hash(self.password, password) - - @classmethod - def get_all_users(cls): - """ - Query to get all users - - :return: a list with all the users. - :rtype: list(:class:`UserModel`) - """ - return cls.get_all_objects() - - @classmethod - def get_one_user(cls, idx): - """ - Query to get the information of one user - - :param int idx: ID of the user - :return: the user object - :rtype: :class:`UserModel` - """ - return cls.get_one_object(idx=idx) - - @classmethod - def get_one_user_by_email(cls, email): - """ - Query to get one user from the email - - :param str email: User email - :return: the user object - :rtype: :class:`UserModel` - """ - return cls.get_one_object(email=email) - - @classmethod - def get_one_user_by_username(cls, username): - """ - Returns one user (object) given a username - - :param str username: the user username that we want to query for - :return: the user object - :rtype: :class:`UserModel` - """ - return cls.get_one_object(username=username) - - def check_username_in_use(self): - """ - Checks if a username is already in use - - :return: a boolean if the username is in use - :rtype: bool - """ - return self.query.filter_by(username=self.username).first() is not None - - def check_email_in_use(self): - """ - Checks if a email is already in use - - :return: a boolean if the username is in use - :rtype: bool - """ - return self.query.filter_by(email=self.email).first() is not None - - @staticmethod - def generate_random_password() -> str: - """ - Method to generate a new random password for the user - - :return: the newly generated password - :rtype: str - """ - nb_lower = random.randint(1, 9) - nb_upper = random.randint(10 - nb_lower, 11) - nb_numbers = random.randint(1, 3) - nb_special_char = random.randint(1, 3) - upper_letters = random.sample(string.ascii_uppercase, nb_upper) - lower_letters = random.sample(string.ascii_lowercase, nb_lower) - numbers = random.sample(list(map(str, list(range(10)))), nb_numbers) - symbols = random.sample("!ยก?ยฟ#$%&'()*+-_./:;,<>=@[]^`{}|~\"\\", nb_special_char) - chars = upper_letters + lower_letters + numbers + symbols - random.shuffle(chars) - pwd = "".join(chars) - return pwd - - def is_admin(self) -> bool: - """ - This should return True or False if the user is an admin - - :return: if the user is an admin or not - :rtype: bool - """ - return UserRoleBaseModel.is_admin(self.id) - - def is_service_user(self) -> bool: - """ - This should return True or False if the user is a service user (type of user used for internal tasks) - - :return: if the user is a service_user or not - :rtype: bool - """ - return UserRoleBaseModel.is_service_user(self.id) - - def __repr__(self): - """ - Representation method of the class - - :return: the representation of the class - :rtype: str - """ - return "".format(self.username) - - def __str__(self): - return self.__repr__() diff --git a/libs/core/cornflow_core/models/user_role.py b/libs/core/cornflow_core/models/user_role.py deleted file mode 100644 index 18be90882..000000000 --- a/libs/core/cornflow_core/models/user_role.py +++ /dev/null @@ -1,139 +0,0 @@ -""" -Model for the relationship between users and roles -""" - -from cornflow_core.constants import ADMIN_ROLE, SERVICE_ROLE -from cornflow_core.models import TraceAttributesModel -from cornflow_core.shared import db - - -class UserRoleBaseModel(TraceAttributesModel): - """ - Model class for the relationship between user and roles. Which roles has a user assigned - It inherits from :class:`TraceAttributesModel` to have trace fields - - The :class:`UserRoleBaseModel` has the following fields: - - - **id**: int, the primary key of the assignation, an integer value that is auto incremented - - **user_id**: the id of the user. - - **role_id**: the id of the assigned role. - - **created_at**: datetime, the datetime when the user was created (in UTC). - This datetime is generated automatically, the user does not need to provide it. - - **updated_at**: datetime, the datetime when the user was last updated (in UTC). - This datetime is generated automatically, the user does not need to provide it. - - **deleted_at**: datetime, the datetime when the user was deleted (in UTC). - This field is used only if we deactivate instead of deleting the record. - This datetime is generated automatically, the user does not need to provide it. - """ - - __tablename__ = "user_role" - __table_args__ = (db.UniqueConstraint("user_id", "role_id"),) - - id = db.Column(db.Integer, primary_key=True, autoincrement=True) - - user_id = db.Column(db.Integer, db.ForeignKey("users.id"), nullable=False) - user = db.relationship("UserBaseModel", viewonly=True, lazy=False) - - role_id = db.Column(db.Integer, db.ForeignKey("roles.id"), nullable=False) - role = db.relationship("RoleBaseModel", viewonly=True, lazy=False) - - def __init__(self, data): - """ - Method to initialize th assignation of a role to a user that - - :param dict data: dict with the information needed to create a new assignation of a role to a user - """ - super().__init__() - self.user_id = data.get("user_id") - self.role_id = data.get("role_id") - - @classmethod - def is_admin(cls, user_id): - """ - Method that checks if a given user has the admin role assigned - - :param int user_id: the ID of the user - :return: a boolean indicating if the user has the admin role assigned or not - :rtype: boolean - """ - user_roles = cls.get_all_objects(user_id=user_id) - for role in user_roles: - if role.role_id == ADMIN_ROLE: - return True - - return False - - @classmethod - def is_service_user(cls, user_id): - """ - Method that checks if a given user has the service role assigned - - :param int user_id: the ID of the user - :return: a boolean indicating if the user has the service role assigned or not - :rtype: boolean - """ - user_roles = cls.get_all_objects(user_id=user_id) - for role in user_roles: - if role.role_id == SERVICE_ROLE: - return True - - return False - - @classmethod - def check_if_role_assigned(cls, user_id, role_id): - """ - Method to check if a user has a given role assigned - - :param int user_id: id of the specific user - :param int role_id: id of the specific role - :return: a boolean if the user has the role assigned - :rtype: bool - """ - return cls.get_one_object(user_id=user_id, role_id=role_id) is not None - - @classmethod - def check_if_role_assigned_disabled(cls, user_id, role_id): - """ - Method to check if a user has a given role assigned but disabled - - :param user_id: id of the specific user - :param role_id: id of the specific role - :return: a boolean if the user has the role assigned but disabled - :rtype: bool - """ - user_role = cls.query.filter( - cls.user_id == user_id, cls.role_id == role_id, cls.deleted_at != None - ).first() - return user_role is not None - - @classmethod - def del_one_user(cls, user_id): - """ - Method to delete all the assigned roles to one user - - :param int user_id: the ID of the user - :return: a list with all the deleted objects. - :rtype: list - """ - return cls.query.filter_by(user_id=user_id).delete(synchronize_session=False) - - def __repr__(self): - """ - Method for the representation of the assigned roles - - :return: the representation - :rtype: str - """ - try: - return f"{self.user.username} has role {self.role.name}" - except AttributeError: - return f"{self.user_id} has role {self.role_id}" - - def __str__(self): - """ - Method for the string representation of the assigned roles - - :return: the string representation - :rtype: str - """ - return self.__repr__() diff --git a/libs/core/cornflow_core/resources/__init__.py b/libs/core/cornflow_core/resources/__init__.py deleted file mode 100644 index b8686c6a6..000000000 --- a/libs/core/cornflow_core/resources/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -""" -Expose the resources -""" -# First the generic ones -from .meta_resource import BaseMetaResource - -# Then the specific -from .log_in import LoginBaseEndpoint -from .recover_password import RecoverPasswordBaseEndpoint -from .sign_up import SignupBaseEndpoint diff --git a/libs/core/cornflow_core/resources/log_in.py b/libs/core/cornflow_core/resources/log_in.py deleted file mode 100644 index ce4b9052f..000000000 --- a/libs/core/cornflow_core/resources/log_in.py +++ /dev/null @@ -1,175 +0,0 @@ -""" -This file contains the logic for the LoginBaseEndpoint -""" -from flask import current_app -from sqlalchemy.exc import IntegrityError, DBAPIError - -from cornflow_core.authentication import BaseAuth, LDAPBase -from cornflow_core.constants import ( - AUTH_DB, - AUTH_LDAP, - AUTH_OID, - OID_AZURE, - OID_GOOGLE, - OID_NONE, -) -from cornflow_core.exceptions import ( - ConfigurationError, - InvalidCredentials, - InvalidUsage, - EndpointNotImplemented, -) -from cornflow_core.models import UserBaseModel, UserRoleBaseModel -from .meta_resource import BaseMetaResource -from ..shared import db - - -class LoginBaseEndpoint(BaseMetaResource): - """ - Base endpoint to perform a login action from a user - """ - - def __init__(self): - super().__init__() - self.data_model = UserBaseModel - self.auth_class = BaseAuth - self.ldap_class = LDAPBase - self.user_role_association = UserRoleBaseModel - - def log_in(self, **kwargs): - """ - This method is in charge of performing the log in of the user - - :param kwargs: keyword arguments passed for the login, these can be username, password or a token - :return: the response of the login or it raises an error. The correct response is a dict - with the newly issued token and the user id, and a status code of 200 - :rtype: dict - """ - auth_type = current_app.config["AUTH_TYPE"] - - if auth_type == AUTH_DB: - user = self.auth_db_authenticate(**kwargs) - elif auth_type == AUTH_LDAP: - user = self.auth_ldap_authenticate(**kwargs) - elif auth_type == AUTH_OID: - user = self.auth_oid_authenticate(**kwargs) - else: - raise ConfigurationError() - - try: - token = self.auth_class.generate_token(user.id) - except Exception as e: - raise InvalidUsage(f"Error in generating user token: {str(e)}", 400) - - return {"token": token, "id": user.id}, 200 - - def auth_db_authenticate(self, username, password): - """ - Method in charge of performing the authentication against the database - - :param str username: the username of the user to log in - :param str password: the password of the user to log in - :return: the user object or it raises an error if it has not been possible to log in - :rtype: :class:`UserBaseModel` - """ - user = self.data_model.get_one_object(username=username) - - if not user: - raise InvalidCredentials() - - if not user.check_hash(password): - raise InvalidCredentials() - - return user - - def auth_ldap_authenticate(self, username, password): - """ - Method in charge of performing the authentication against the ldap server - - :param str username: the username of the user to log in - :param str password: the password of the user to log in - :return: the user object or it raises an error if it has not been possible to log in - :rtype: :class:`UserBaseModel` - """ - ldap_obj = self.ldap_class(current_app.config) - if not ldap_obj.authenticate(username, password): - raise InvalidCredentials() - user = self.data_model.get_one_object(username=username) - if not user: - current_app.logger.info(f"LDAP user {username} does not exist and is created") - email = ldap_obj.get_user_email(username) - if not email: - email = "" - data = {"username": username, "email": email} - user = self.data_model(data=data) - user.save() - - roles = ldap_obj.get_user_roles(username) - - try: - self.user_role_association.del_one_user(user.id) - for role in roles: - user_role = self.user_role_association( - data={"user_id": user.id, "role_id": role} - ) - user_role.save() - - except IntegrityError as e: - db.session.rollback() - current_app.logger.error(f"Integrity error on user role assignment on log in: {e}") - except DBAPIError as e: - db.session.rollback() - current_app.logger.error(f"Unknown error on user role assignment on log in: {e}") - - return user - - def auth_oid_authenticate(self, token): - """ - Method in charge of performing the log in with the token issued by an Open ID provider - - :param str token: the token that the user has obtained from the Open ID provider - :return: the user object or it raises an error if it has not been possible to log in - :rtype: :class:`UserBaseModel` - """ - oid_provider = int(current_app.config["OID_PROVIDER"]) - - client_id = current_app.config["OID_CLIENT_ID"] - tenant_id = current_app.config["OID_TENANT_ID"] - issuer = current_app.config["OID_ISSUER"] - - if client_id is None or tenant_id is None or issuer is None: - raise ConfigurationError("The OID provider configuration is not valid") - - if oid_provider == OID_AZURE: - decoded_token = self.auth_class().validate_oid_token( - token, client_id, tenant_id, issuer, oid_provider - ) - - elif oid_provider == OID_GOOGLE: - raise EndpointNotImplemented("The selected OID provider is not implemented") - elif oid_provider == OID_NONE: - raise EndpointNotImplemented("The OID provider configuration is not valid") - else: - raise EndpointNotImplemented("The OID provider configuration is not valid") - - username = decoded_token["preferred_username"] - - user = self.data_model.get_one_object(username=username) - - if not user: - current_app.logger.info(f"OpenID user {username} does not exist and is created") - - data = {"username": username, "email": username} - - user = self.data_model(data=data) - user.save() - - self.user_role_association(user.id) - - user_role = self.user_role_association( - {"user_id": user.id, "role_id": int(current_app.config["DEFAULT_ROLE"])} - ) - - user_role.save() - - return user diff --git a/libs/core/cornflow_core/resources/recover_password.py b/libs/core/cornflow_core/resources/recover_password.py deleted file mode 100644 index 51d1e2b6f..000000000 --- a/libs/core/cornflow_core/resources/recover_password.py +++ /dev/null @@ -1,69 +0,0 @@ -""" -This file contains the base resource to recover a password -""" - -from flask import current_app - -from cornflow_core.exceptions import ConfigurationError -from cornflow_core.messages import get_password_recover_email, send_email_to -from cornflow_core.models import UserBaseModel -from .meta_resource import BaseMetaResource - - -class RecoverPasswordBaseEndpoint(BaseMetaResource): - """ - Endpoint to recover the password - """ - - def __init__(self): - super().__init__() - self.data_model = UserBaseModel - - def recover_password(self, email): - """ - - :param str email: The email where the password needs to be sent to - :return: - :rtype: - """ - sender = current_app.config["SERVICE_EMAIL_ADDRESS"] - password = current_app.config["SERVICE_EMAIL_PASSWORD"] - smtp_server = current_app.config["SERVICE_EMAIL_SERVER"] - port = current_app.config["SERVICE_EMAIL_PORT"] - service_name = current_app.config["SERVICE_NAME"] - receiver = email - - if sender is None or password is None or smtp_server is None or port is None: - raise ConfigurationError( - "This functionality is not available. Check that cornflow's email is correctly configured" - ) - - message = "The password recovery process has started. Check the email inbox." - - user_obj = self.data_model({"email": receiver}) - if not user_obj.check_email_in_use(): - return {"message": message}, 200 - - new_password = self.data_model.generate_random_password() - - text_email = get_password_recover_email( - temp_password=new_password, - service_name=service_name, - sender=sender, - receiver=receiver, - ) - - send_email_to( - email=text_email, - smtp_server=smtp_server, - port=port, - sender=sender, - password=password, - receiver=receiver, - ) - - data = {"password": new_password} - user_obj = self.data_model.get_one_user_by_email(receiver) - user_obj.update(data) - - return {"message": message}, 200 diff --git a/libs/core/cornflow_core/resources/sign_up.py b/libs/core/cornflow_core/resources/sign_up.py deleted file mode 100644 index b7427d3be..000000000 --- a/libs/core/cornflow_core/resources/sign_up.py +++ /dev/null @@ -1,81 +0,0 @@ -""" -This file contains the base for a sign up endpoint -""" - -from flask import current_app - -from cornflow_core.authentication import BaseAuth -from cornflow_core.constants import AUTH_LDAP, AUTH_OID -from cornflow_core.exceptions import ( - EndpointNotImplemented, - InvalidCredentials, - InvalidUsage, -) -from cornflow_core.models import UserBaseModel, UserRoleBaseModel -from .meta_resource import BaseMetaResource - - -class SignupBaseEndpoint(BaseMetaResource): - """ - Ths base for the sign up endpoint - """ - - def __init__(self): - super().__init__() - self.data_model = UserBaseModel - self.auth_class = BaseAuth - self.user_role_association = UserRoleBaseModel - - def sign_up(self, **kwargs): - """ - The method in charge of performing the sign up of users - - :param kwargs: the keyword arguments needed to perform the sign up - :return: a dictionary with the newly issued token and the user id, and a status code - """ - auth_type = current_app.config["AUTH_TYPE"] - if auth_type == AUTH_LDAP: - err = "The user has to sign up on the active directory" - raise EndpointNotImplemented( - err, - log_txt="Error while user tries to sign up. " + err - ) - elif auth_type == AUTH_OID: - err = "The user has to sign up with the OpenID protocol" - raise EndpointNotImplemented( - err, - log_txt="Error while user tries to sign up. " + err - ) - - user = self.data_model(kwargs) - - if user.check_username_in_use(): - raise InvalidCredentials( - error="Username already in use, please supply another username", - log_txt="Error while user tries to sign up. Username already in use." - - ) - - if user.check_email_in_use(): - raise InvalidCredentials( - error="Email already in use, please supply another email address", - log_txt="Error while user tries to sign up. Email already in use." - ) - - user.save() - - user_role = self.user_role_association( - {"user_id": user.id, "role_id": current_app.config["DEFAULT_ROLE"]} - ) - - user_role.save() - - try: - token = self.auth_class.generate_token(user.id) - except Exception as e: - raise InvalidUsage( - error="Error in generating user token: " + str(e), status_code=400, - log_txt="Error while user tries to sign up. Unable to generate token." - ) - current_app.logger.info(f"New user created: {user}") - return {"token": token, "id": user.id}, 201 diff --git a/libs/core/cornflow_core/schemas/__init__.py b/libs/core/cornflow_core/schemas/__init__.py deleted file mode 100644 index c5ebbf7e4..000000000 --- a/libs/core/cornflow_core/schemas/__init__.py +++ /dev/null @@ -1,20 +0,0 @@ -""" -Import all schemas to make available -""" -from .action import ActionsResponse -from .patch import BasePatchOperation -from .permissions import ( - PermissionViewRoleBaseEditRequest, - PermissionViewRoleBaseRequest, - PermissionViewRoleBaseResponse, -) -from .query import BaseQueryFilters -from .role import RolesRequest, RolesResponse -from .user import ( - BaseUserSchema, - LoginEndpointRequest, - LoginOpenAuthRequest, - SignupRequest, -) -from .user_role import UserRoleRequest, UserRoleResponse -from .view import ViewResponse diff --git a/libs/core/cornflow_core/schemas/user.py b/libs/core/cornflow_core/schemas/user.py deleted file mode 100644 index 193a5ab0b..000000000 --- a/libs/core/cornflow_core/schemas/user.py +++ /dev/null @@ -1,48 +0,0 @@ -""" -This file contains the schemas used by the user model -""" -from marshmallow import fields, Schema - - -class BaseUserSchema(Schema): - """ - This is the base schema used for the users - """ - - id = fields.Int(dump_only=True) - first_name = fields.Str() - last_name = fields.Str() - username = fields.Str(required=True) - email = fields.Email(required=True) - password = fields.Str(required=True, load_only=True) - created_at = fields.DateTime(dump_only=True) - modified_at = fields.DateTime(dump_only=True) - - -class LoginEndpointRequest(Schema): - """ - This is the schema used by the login endpoint with auth db or ldap - """ - - username = fields.Str(required=True) - password = fields.Str(required=True) - - -class LoginOpenAuthRequest(Schema): - """ - This is the schema used by the login endpoint with Open ID protocol - """ - - token = fields.Str(required=True) - - -class SignupRequest(Schema): - """ - This is the schema used by the sign up - """ - - username = fields.Str(required=True) - email = fields.Email(required=True) - password = fields.Str(required=True, load_only=True) - first_name = fields.Str(required=False) - last_name = fields.Str(required=False) diff --git a/libs/core/cornflow_core/shared/__init__.py b/libs/core/cornflow_core/shared/__init__.py deleted file mode 100644 index 3793567ca..000000000 --- a/libs/core/cornflow_core/shared/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -""" -Exposes the objects and functions of the module -""" -from .utils import db, bcrypt -from .validators import ( - check_email_pattern, - check_password_pattern, - json_schema_validate, - json_schema_validate_as_string, -) diff --git a/libs/core/cornflow_core/shared/utils.py b/libs/core/cornflow_core/shared/utils.py deleted file mode 100644 index 63b7c2835..000000000 --- a/libs/core/cornflow_core/shared/utils.py +++ /dev/null @@ -1,24 +0,0 @@ -""" -This file defines the database session with SQLAlchemy and the password encryption with Bcrypt -Additionally we add the option to have our database models inherit ABCMeta class so that abstract methods can be defined -""" -from abc import ABCMeta - -from flask_bcrypt import Bcrypt -from flask_sqlalchemy import SQLAlchemy -from flask_sqlalchemy.model import Model, DefaultMeta -from sqlalchemy.ext.declarative import declarative_base - - -class CustomABCMeta(DefaultMeta, ABCMeta): - """ - Custom meta class so that the models inherit ABCMeta - """ - - pass - - -db = SQLAlchemy( - model_class=declarative_base(cls=Model, metaclass=CustomABCMeta, name="Model") -) -bcrypt = Bcrypt() diff --git a/libs/core/cornflow_core/tests/__init__.py b/libs/core/cornflow_core/tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/libs/core/requirements-dev.txt b/libs/core/requirements-dev.txt deleted file mode 100644 index 9d2dd8770..000000000 --- a/libs/core/requirements-dev.txt +++ /dev/null @@ -1,4 +0,0 @@ --r requirements.txt -docutils -coverage -pipreqs \ No newline at end of file diff --git a/libs/core/requirements.txt b/libs/core/requirements.txt deleted file mode 100644 index 1c5ab8094..000000000 --- a/libs/core/requirements.txt +++ /dev/null @@ -1,18 +0,0 @@ -click -cornflow-client<=1.0.12 -cryptography -disposable-email-domains -Flask==2.1.3 -flask-apispec -Flask-Bcrypt -Flask-Compress -Flask-RESTful -Flask-SQLAlchemy==2.5.1 -ldap3 -marshmallow -PyJWT -pytups -requests -SQLAlchemy==1.3.21 -webargs -Werkzeug diff --git a/libs/core/setup.py b/libs/core/setup.py deleted file mode 100644 index 269c6af3f..000000000 --- a/libs/core/setup.py +++ /dev/null @@ -1,35 +0,0 @@ -import setuptools - -with open("README.rst") as fh: - long_description = fh.read() - -required = [] -with open("requirements.txt", "r") as fh: - required.append(fh.read().splitlines()) - -setuptools.setup( - name="cornflow-core", - version="0.1.10", - author="baobab soluciones", - author_email="sistemas@baobabsoluciones.es", - description="REST API flask backend components used by cornflow and other REST APIs", - long_description=long_description, - long_description_content_type="text/x-rst", - url="https://github.com/baobabsoluciones/cornflow", - packages=setuptools.find_packages(), - classifiers=[ - "Programming Language :: Python :: 3", - "License :: OSI Approved :: MIT License", - "Operating System :: OS Independent", - "Development Status :: 3 - Alpha", - ], - python_requires=">=3.7", - include_package_data=True, - install_requires=required, - entry_points={ - "console_scripts": [ - "generate_from_schema = cornflow_core.cli.generate_from_schema:generate_from_schema", - "schema_from_models = cornflow_core.cli.schema_from_models:schema_from_models", - ] - }, -)