From a2229fea55b492c8c77dfd99ea432fa484b678d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20Gonz=C3=A1lez-Santander=20de=20la=20Cruz?= Date: Tue, 16 Apr 2024 18:58:54 +0200 Subject: [PATCH 1/8] Upgrade versions to address security issues (#514) * Upgrade versions to address security issues * Fixed gunicorn import statement * Fixed licenses endpoint and unit test --- cornflow-server/cornflow/shared/licenses.py | 1 + cornflow-server/cornflow/tests/unit/test_licenses.py | 4 ++-- cornflow-server/requirements.txt | 9 +++++---- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/cornflow-server/cornflow/shared/licenses.py b/cornflow-server/cornflow/shared/licenses.py index c009c4dd8..3186d7d44 100644 --- a/cornflow-server/cornflow/shared/licenses.py +++ b/cornflow-server/cornflow/shared/licenses.py @@ -65,6 +65,7 @@ def get_licenses_summary(): :return: a list of dicts with library, license, version, author, description, home page and license text. """ license_list = [] + # TODO: pkg_resources.working_set is deprecated, find a better way to get the list of packages for pkg in sorted(pkg_resources.working_set, key=lambda x: str(x).lower()): license_list += [ { diff --git a/cornflow-server/cornflow/tests/unit/test_licenses.py b/cornflow-server/cornflow/tests/unit/test_licenses.py index c68743643..abfea2ac6 100644 --- a/cornflow-server/cornflow/tests/unit/test_licenses.py +++ b/cornflow-server/cornflow/tests/unit/test_licenses.py @@ -11,9 +11,9 @@ def read_requirements(): requirements = content.split("\n") requirements = [ - r.split("=")[0].split(">")[0].split("<")[0].lower() + r.split("=")[0].split(">")[0].split("<")[0].split(" @")[0].lower() for r in requirements - if r != "" + if r != "" and not r.startswith("#") ] return requirements diff --git a/cornflow-server/requirements.txt b/cornflow-server/requirements.txt index 185d44355..b88e983dd 100644 --- a/cornflow-server/requirements.txt +++ b/cornflow-server/requirements.txt @@ -2,7 +2,7 @@ alembic==1.9.2 apispec<=6.2.0 click<=8.1.3 cornflow-client<=1.0.16 -cryptography<=39.0.2 +cryptography<=42.0.5 disposable-email-domains>=0.0.86 Flask==2.3.2 flask-apispec<=0.11.4 @@ -16,7 +16,8 @@ Flask-SQLAlchemy==2.5.1 gevent==23.9.1 greenlet<=2.0.2;python_version<"3.11" greenlet==3.0.0;python_version>="3.11" -gunicorn<=20.1.0 +# this should be changed once the correct version (higher than 21.2.0) is released +gunicorn @ git+http://github.com/benoitc/gunicorn@88fc4a43152039c28096c8ba3eeadb3fbaa4aff9 jsonpatch<=1.32 ldap3<=2.9.1 marshmallow<=3.19.0 @@ -24,7 +25,7 @@ PuLP<=2.7.0 psycopg2<=2.95 PyJWT<=2.6.0 pytups>=0.86.2 -requests<=2.29.0 +requests<=2.31.0 SQLAlchemy==1.3.21 webargs<=8.2.0 -Werkzeug<=2.3.3 +Werkzeug<=2.3.8 From f40727e7b30221c17319e7270ec27bb69fb0624c Mon Sep 17 00:00:00 2001 From: Guillermo Gonzalez-Santander Date: Tue, 16 Apr 2024 19:18:05 +0200 Subject: [PATCH 2/8] Bumped versions of airflow and cornflow to create new pypi versions and dockerhub images --- cornflow-server/Dockerfile | 4 ++-- cornflow-server/airflow_config/Dockerfile | 10 ++++++---- cornflow-server/setup.py | 2 +- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/cornflow-server/Dockerfile b/cornflow-server/Dockerfile index 567b8245b..1a87f66e9 100644 --- a/cornflow-server/Dockerfile +++ b/cornflow-server/Dockerfile @@ -1,4 +1,4 @@ -# VERSION 1.0.8 +# VERSION 1.0.10 # AUTHOR: sistemas@baobabsoluciones.es FROM python:3.10-slim-buster @@ -9,7 +9,7 @@ ENV DEBIAN_FRONTEND noninteractive ENV TERM linux # CORNFLOW vars -ARG CORNFLOW_VERSION=1.0.9 +ARG CORNFLOW_VERSION=1.0.10a1 # install linux pkg RUN apt update -y && apt-get install -y --no-install-recommends \ diff --git a/cornflow-server/airflow_config/Dockerfile b/cornflow-server/airflow_config/Dockerfile index f0ae2ca64..f56e32be2 100644 --- a/cornflow-server/airflow_config/Dockerfile +++ b/cornflow-server/airflow_config/Dockerfile @@ -1,7 +1,7 @@ -# VERSION 2.7.1 +# VERSION 2.9.0 # AUTHOR: cornflow@baobabsoluciones.es -# DESCRIPTION: Airflow 2.7.1 image personalized for use with Cornflow (from baobabsoluciones/pysolver image) -# baobab code version is 1.0.8 +# DESCRIPTION: Airflow 2.9.0 image personalized for use with Cornflow (from baobabsoluciones/pysolver image) +# baobab code version is 1.0.10 FROM baobabsoluciones/pysolver:1.0 LABEL maintainer="cornflow@baobabsoluciones" @@ -11,7 +11,7 @@ ENV DEBIAN_FRONTEND noninteractive ENV TERM linux # Airflow vars -ARG AIRFLOW_VERSION=2.7.1 +ARG AIRFLOW_VERSION=2.9.0 ARG AIRFLOW_USER_HOME=/usr/local/airflow ARG CONSTRAINT_URL="https://raw.githubusercontent.com/apache/airflow/constraints-${AIRFLOW_VERSION}/constraints-3.10.txt" ARG AIRFLOW__CORE__LOAD_EXAMPLES=False @@ -19,6 +19,8 @@ ENV AIRFLOW_HOME=${AIRFLOW_USER_HOME} # install Airflow and extras: celery,postgres and redis RUN pip install "apache-airflow[celery,google,postgres,redis,sendgrid]==${AIRFLOW_VERSION}" --constraint "${CONSTRAINT_URL}" +# We add these overruns due to security reasons as suggested here: https://airflow.apache.org/docs/apache-airflow/stable/installation/installing-from-pypi.html#upgrading-and-installing-dependencies-including-providers +RUN pip install "apache-airflow[celery,google,postgres,redis,sendgrid]==${AIRFLOW_VERSION} cryptography<=42.0.5 gunicorn @ git+http://github.com/benoitc/gunicorn@88fc4a43152039c28096c8ba3eeadb3fbaa4aff9 requests<=2.31.0 Werkzeug<=2.3.8" # copy init script and config to container COPY scripts ${AIRFLOW_HOME}/scripts diff --git a/cornflow-server/setup.py b/cornflow-server/setup.py index dbb93c5d4..1d81a9ed8 100644 --- a/cornflow-server/setup.py +++ b/cornflow-server/setup.py @@ -9,7 +9,7 @@ setuptools.setup( name="cornflow", - version="1.0.9", + version="1.0.10a1", author="baobab soluciones", author_email="cornflow@baobabsoluciones.es", description="Cornflow is an open source multi-solver optimization server with a REST API built using flask.", From 8736ef121b885e3d1ada03ce5ed25e8620f04604 Mon Sep 17 00:00:00 2001 From: Guillermo Gonzalez-Santander Date: Tue, 16 Apr 2024 19:29:04 +0200 Subject: [PATCH 3/8] Fix dependecy in requirements.txt --- cornflow-server/cornflow/tests/unit/test_licenses.py | 2 +- cornflow-server/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cornflow-server/cornflow/tests/unit/test_licenses.py b/cornflow-server/cornflow/tests/unit/test_licenses.py index abfea2ac6..710538112 100644 --- a/cornflow-server/cornflow/tests/unit/test_licenses.py +++ b/cornflow-server/cornflow/tests/unit/test_licenses.py @@ -11,7 +11,7 @@ def read_requirements(): requirements = content.split("\n") requirements = [ - r.split("=")[0].split(">")[0].split("<")[0].split(" @")[0].lower() + r.split("=")[0].split(">")[0].split("<")[0].split("@")[0].lower() for r in requirements if r != "" and not r.startswith("#") ] diff --git a/cornflow-server/requirements.txt b/cornflow-server/requirements.txt index b88e983dd..938bd7541 100644 --- a/cornflow-server/requirements.txt +++ b/cornflow-server/requirements.txt @@ -17,7 +17,7 @@ gevent==23.9.1 greenlet<=2.0.2;python_version<"3.11" greenlet==3.0.0;python_version>="3.11" # this should be changed once the correct version (higher than 21.2.0) is released -gunicorn @ git+http://github.com/benoitc/gunicorn@88fc4a43152039c28096c8ba3eeadb3fbaa4aff9 +gunicorn@git+http://github.com/benoitc/gunicorn@88fc4a43152039c28096c8ba3eeadb3fbaa4aff9 jsonpatch<=1.32 ldap3<=2.9.1 marshmallow<=3.19.0 From 09bb87df49ba1615422b27a8203d0cfccb36ed75 Mon Sep 17 00:00:00 2001 From: Guillermo Gonzalez-Santander Date: Wed, 17 Apr 2024 09:34:23 +0200 Subject: [PATCH 4/8] Changed to final dependency --- cornflow-server/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cornflow-server/requirements.txt b/cornflow-server/requirements.txt index 938bd7541..5f9c5b9bb 100644 --- a/cornflow-server/requirements.txt +++ b/cornflow-server/requirements.txt @@ -17,7 +17,7 @@ gevent==23.9.1 greenlet<=2.0.2;python_version<"3.11" greenlet==3.0.0;python_version>="3.11" # this should be changed once the correct version (higher than 21.2.0) is released -gunicorn@git+http://github.com/benoitc/gunicorn@88fc4a43152039c28096c8ba3eeadb3fbaa4aff9 +gunicorn<=22.0.0 jsonpatch<=1.32 ldap3<=2.9.1 marshmallow<=3.19.0 From 137bcfb1fc9c6d58a2f1de15657406e86bf29327 Mon Sep 17 00:00:00 2001 From: Guillermo Gonzalez-Santander Date: Wed, 17 Apr 2024 09:57:06 +0200 Subject: [PATCH 5/8] Changed airflow Dockerfile --- cornflow-server/airflow_config/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cornflow-server/airflow_config/Dockerfile b/cornflow-server/airflow_config/Dockerfile index f56e32be2..fe1263120 100644 --- a/cornflow-server/airflow_config/Dockerfile +++ b/cornflow-server/airflow_config/Dockerfile @@ -20,7 +20,7 @@ ENV AIRFLOW_HOME=${AIRFLOW_USER_HOME} # install Airflow and extras: celery,postgres and redis RUN pip install "apache-airflow[celery,google,postgres,redis,sendgrid]==${AIRFLOW_VERSION}" --constraint "${CONSTRAINT_URL}" # We add these overruns due to security reasons as suggested here: https://airflow.apache.org/docs/apache-airflow/stable/installation/installing-from-pypi.html#upgrading-and-installing-dependencies-including-providers -RUN pip install "apache-airflow[celery,google,postgres,redis,sendgrid]==${AIRFLOW_VERSION} cryptography<=42.0.5 gunicorn @ git+http://github.com/benoitc/gunicorn@88fc4a43152039c28096c8ba3eeadb3fbaa4aff9 requests<=2.31.0 Werkzeug<=2.3.8" +RUN pip install "apache-airflow[celery,google,postgres,redis,sendgrid]==${AIRFLOW_VERSION} cryptography<=42.0.5 gunicorn<=22.0.0 requests<=2.31.0 Werkzeug<=2.3.8" # copy init script and config to container COPY scripts ${AIRFLOW_HOME}/scripts From 319bb42cfccc7335bcc9945f65d00181ff06c65d Mon Sep 17 00:00:00 2001 From: Guillermo Gonzalez-Santander Date: Wed, 17 Apr 2024 10:47:35 +0200 Subject: [PATCH 6/8] Changed airflow dockerfile --- cornflow-server/airflow_config/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cornflow-server/airflow_config/Dockerfile b/cornflow-server/airflow_config/Dockerfile index fe1263120..bb8f71abf 100644 --- a/cornflow-server/airflow_config/Dockerfile +++ b/cornflow-server/airflow_config/Dockerfile @@ -20,7 +20,7 @@ ENV AIRFLOW_HOME=${AIRFLOW_USER_HOME} # install Airflow and extras: celery,postgres and redis RUN pip install "apache-airflow[celery,google,postgres,redis,sendgrid]==${AIRFLOW_VERSION}" --constraint "${CONSTRAINT_URL}" # We add these overruns due to security reasons as suggested here: https://airflow.apache.org/docs/apache-airflow/stable/installation/installing-from-pypi.html#upgrading-and-installing-dependencies-including-providers -RUN pip install "apache-airflow[celery,google,postgres,redis,sendgrid]==${AIRFLOW_VERSION} cryptography<=42.0.5 gunicorn<=22.0.0 requests<=2.31.0 Werkzeug<=2.3.8" +RUN pip install "apache-airflow[celery,google,postgres,redis,sendgrid]==${AIRFLOW_VERSION}" "cryptography<=42.0.5" "gunicorn<=22.0.0" "requests<=2.31.0" "Werkzeug<=2.3.8" # copy init script and config to container COPY scripts ${AIRFLOW_HOME}/scripts From fcbb007668b67f82e710b957981700e85d63d3d4 Mon Sep 17 00:00:00 2001 From: Guillermo Gonzalez-Santander Date: Wed, 17 Apr 2024 11:28:06 +0200 Subject: [PATCH 7/8] Modified dockerfile for airflow again --- cornflow-server/airflow_config/Dockerfile | 2 +- cornflow-server/requirements.txt | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/cornflow-server/airflow_config/Dockerfile b/cornflow-server/airflow_config/Dockerfile index bb8f71abf..b15e0451c 100644 --- a/cornflow-server/airflow_config/Dockerfile +++ b/cornflow-server/airflow_config/Dockerfile @@ -20,7 +20,7 @@ ENV AIRFLOW_HOME=${AIRFLOW_USER_HOME} # install Airflow and extras: celery,postgres and redis RUN pip install "apache-airflow[celery,google,postgres,redis,sendgrid]==${AIRFLOW_VERSION}" --constraint "${CONSTRAINT_URL}" # We add these overruns due to security reasons as suggested here: https://airflow.apache.org/docs/apache-airflow/stable/installation/installing-from-pypi.html#upgrading-and-installing-dependencies-including-providers -RUN pip install "apache-airflow[celery,google,postgres,redis,sendgrid]==${AIRFLOW_VERSION}" "cryptography<=42.0.5" "gunicorn<=22.0.0" "requests<=2.31.0" "Werkzeug<=2.3.8" +RUN pip install "apache-airflow[celery,google,postgres,redis,sendgrid]==${AIRFLOW_VERSION}" "cryptography==42.0.5" "gunicorn==22.0.0" "requests==2.31.0" "Werkzeug==2.3.8" # copy init script and config to container COPY scripts ${AIRFLOW_HOME}/scripts diff --git a/cornflow-server/requirements.txt b/cornflow-server/requirements.txt index 5f9c5b9bb..02c6db8a2 100644 --- a/cornflow-server/requirements.txt +++ b/cornflow-server/requirements.txt @@ -16,7 +16,6 @@ Flask-SQLAlchemy==2.5.1 gevent==23.9.1 greenlet<=2.0.2;python_version<"3.11" greenlet==3.0.0;python_version>="3.11" -# this should be changed once the correct version (higher than 21.2.0) is released gunicorn<=22.0.0 jsonpatch<=1.32 ldap3<=2.9.1 From 649c58305728b2d31fce85bc5bd71c3771e43426 Mon Sep 17 00:00:00 2001 From: Guillermo Gonzalez-Santander Date: Wed, 17 Apr 2024 11:56:35 +0200 Subject: [PATCH 8/8] Updated changelog file and version codes. --- cornflow-server/Dockerfile | 2 +- cornflow-server/airflow_config/Dockerfile | 2 +- cornflow-server/changelog.rst | 11 +++++++++++ cornflow-server/setup.py | 2 +- 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/cornflow-server/Dockerfile b/cornflow-server/Dockerfile index 1a87f66e9..194621bb4 100644 --- a/cornflow-server/Dockerfile +++ b/cornflow-server/Dockerfile @@ -9,7 +9,7 @@ ENV DEBIAN_FRONTEND noninteractive ENV TERM linux # CORNFLOW vars -ARG CORNFLOW_VERSION=1.0.10a1 +ARG CORNFLOW_VERSION=1.0.10 # install linux pkg RUN apt update -y && apt-get install -y --no-install-recommends \ diff --git a/cornflow-server/airflow_config/Dockerfile b/cornflow-server/airflow_config/Dockerfile index b15e0451c..e2a21369d 100644 --- a/cornflow-server/airflow_config/Dockerfile +++ b/cornflow-server/airflow_config/Dockerfile @@ -1,4 +1,4 @@ -# VERSION 2.9.0 +# AIRFLOW VERSION 2.9.0 # AUTHOR: cornflow@baobabsoluciones.es # DESCRIPTION: Airflow 2.9.0 image personalized for use with Cornflow (from baobabsoluciones/pysolver image) # baobab code version is 1.0.10 diff --git a/cornflow-server/changelog.rst b/cornflow-server/changelog.rst index e82f112c9..ef817fc03 100644 --- a/cornflow-server/changelog.rst +++ b/cornflow-server/changelog.rst @@ -1,3 +1,14 @@ +version 1.0.10 +--------------- + +- released: 2024-04-17 +- description: changed libraries versions due to discovered vulnerabilities +- changelog: + - Upgraded cryptography version to 42.0.5 + - Upgraded gunicorn version to 22.0.0 + - Upgraded requests version to 2.31.0 + - Upgraded Werkzeug version to 2.3.8 + version 1.0.9 -------------- diff --git a/cornflow-server/setup.py b/cornflow-server/setup.py index 1d81a9ed8..dbb3bf68d 100644 --- a/cornflow-server/setup.py +++ b/cornflow-server/setup.py @@ -9,7 +9,7 @@ setuptools.setup( name="cornflow", - version="1.0.10a1", + version="1.0.10", author="baobab soluciones", author_email="cornflow@baobabsoluciones.es", description="Cornflow is an open source multi-solver optimization server with a REST API built using flask.",