Merge pull request #566 from baobabsoluciones/release/v1.1.3

Release/v1.1.3

cornflow-dags/DAG/bar_cutting/core/instance.py

@@ -38,6 +38,9 @@ def to_dict(self) -> dict:
 
         return pickle.loads(pickle.dumps(data_p, -1))
 
+    def check(self):
+        return dict()
+
     def get_bars(self) -> TupList:
         """
         Returns a TupList with the ids of the bars.

cornflow-dags/DAG/dag_timer.py

@@ -9,6 +9,9 @@ class Instance(InstanceCore):
     schema = get_empty_schema()
     schema_checks = get_empty_schema()
 
+    def check(self):
+        return dict()
+
 
 class Solution(SolutionCore):
     schema = get_empty_schema()
@@ -30,7 +33,7 @@ def solve(self, options):
     def get_objective(self) -> float:
         return 0
 
-    def check_solution(self, *args, **kwargs):
+    def check_solution(self):
         return dict()
 
 

cornflow-dags/DAG/facility_location/core/instance.py

@@ -56,6 +56,9 @@ def to_dict(self) -> dict:
         data_p["parameters"] = self.data["parameters"]
         return pickle.loads(pickle.dumps(data_p, -1))
 
+    def check(self):
+        return dict()
+
     def get_suppliers(self):
         return self.data["suppliers"].keys_tl()
 

cornflow-dags/DAG/graph_coloring/core/instance.py

@@ -8,5 +8,8 @@ class Instance(InstanceCore):
     schema = load_json(os.path.join(os.path.dirname(__file__), "../schemas/input.json"))
     schema_checks = get_empty_schema()
 
+    def check(self):
+        return dict()
+
     def get_pairs(self):
         return pt.TupList((el["n1"], el["n2"]) for el in self.data["pairs"])

cornflow-dags/DAG/knapsack/core/instance.py

@@ -61,6 +61,9 @@ def to_dict(self):
 
         return data_dict
 
+    def check(self):
+        return dict()
+
     def get_objects_values(self):
         return {
             self.data["ids"][i]: self.data["values"][i]

cornflow-dags/DAG/roadef/core/instance.py

@@ -116,6 +116,9 @@ def to_dict(self):
             ._update(sources_tables)
         )
 
+    def check(self):
+        return dict()
+
     @staticmethod
     def dict_to_int_or_float(data_dict):
         """

cornflow-dags/DAG/solve_model_dag/__init__.py

@@ -29,6 +29,9 @@ class Instance(InstanceCore):
     schema = get_pulp_jsonschema()
     schema_checks = get_empty_schema()
 
+    def check(self):
+        return dict()
+
 
 class Solution(SolutionCore):
     schema = get_pulp_jsonschema()

cornflow-dags/DAG/tsp/core/instance.py

@@ -59,5 +59,8 @@ def to_tsplib95(self):
         )
         return tsp.models.StandardProblem(**dict_data)
 
+    def check(self):
+        return dict()
+
     def get_arcs(self) -> TupList:
         return self.data["arcs"]

cornflow-dags/DAG/two_dimension_bin_packing/core/instance.py

@@ -32,6 +32,9 @@ def to_dict(self):
         data_p["parameters"] = self.data["parameters"]
         return pickle.loads(pickle.dumps(data_p, -1))
 
+    def check(self):
+        return dict()
+
     def _get_property(self, key, prop) -> SuperDict:
         return self.data[key].get_property(prop)
 

cornflow-dags/DAG/vrp/core/instance.py

@@ -33,6 +33,9 @@ def to_dict(self):
 
         return data
 
+    def check(self):
+        return dict()
+
     def get_nodes(self):
         return [i["n"] for i in self.data["demand"].values()]
 

cornflow-dags/tests/test_dags.py

@@ -36,6 +36,7 @@ def test_schema_load(self):
             self.assertIsInstance(self.app.instance.schema, dict)
             self.assertIsInstance(self.app.solution.schema, dict)
             self.assertIsInstance(self.app.schema, dict)
+            self.assertIsInstance(self.app.instance.schema_checks, dict)
 
         def test_config_requirements(self):
             keys = {"solver", "timeLimit"}
@@ -85,8 +86,9 @@ def test_try_solving_testcase(self, config=None):
                 instance = self.app.instance.from_dict(instance_data)
                 solution = self.app.solution.from_dict(solution_test)
                 s = self.app.get_default_solver_name()
-                experim = self.app.get_solver(s)(instance, solution)
-                checks = experim.check_solution()
+                experiment = self.app.get_solver(s)(instance, solution)
+                self.assertIsInstance(experiment.schema_checks, dict)
+                checks = experiment.check_solution()
                 failed_checks = [k for k, v in checks.items() if len(v) > 0]
                 if len(failed_checks) > 0:
                     print(
@@ -97,9 +99,9 @@ def test_try_solving_testcase(self, config=None):
                         if len(values) > 0:
                             print(f"{check}: {values}")
 
-                experim.get_objective()
+                experiment.get_objective()
 
-                validator = Draft7Validator(experim.schema_checks)
+                validator = Draft7Validator(experiment.schema_checks)
                 if not validator.is_valid(solution_check):
                     raise Exception("The solution checks have invalid format")
 

cornflow-server/changelog.rst

@@ -1,3 +1,13 @@
+version 1.1.3
+--------------
+
+- released: 2024-12-05
+- description: small changes
+- changelog:
+    - changed the json schema validation on airflow so that solution, instance checks and solution checks are correctly reviewed.
+    - added some small changes to make sure that future compatibility with new version of libraries is ready.
+    - added a reconnect from airflow to cornflow to make sure that if the model fails we can get back as much as possible.
+
 version 1.1.2
 --------------
 
@@ -9,7 +19,7 @@ version 1.1.2
 version 1.1.1
 --------------
 
-- released: 2024-08-29
+- released: 2024-09-18
 - description: small security fixes
 - changelog:
     - bump PuLP to version 2.9.0

cornflow-server/cornflow/endpoints/login.py

@@ -34,6 +34,7 @@ class LoginBaseEndpoint(BaseMetaResource):
     """
     Base endpoint to perform a login action from a user
     """
+
     def __init__(self):
         super().__init__()
         self.ldap_class = LDAPBase
@@ -102,7 +103,9 @@ def auth_ldap_authenticate(self, username, password):
             raise InvalidCredentials()
         user = self.data_model.get_one_object(username=username)
         if not user:
-            current_app.logger.info(f"LDAP user {username} does not exist and is created")
+            current_app.logger.info(
+                f"LDAP user {username} does not exist and is created"
+            )
             email = ldap_obj.get_user_email(username)
             if not email:
                 email = ""
@@ -122,10 +125,14 @@ def auth_ldap_authenticate(self, username, password):
 
         except IntegrityError as e:
             db.session.rollback()
-            current_app.logger.error(f"Integrity error on user role assignment on log in: {e}")
+            current_app.logger.error(
+                f"Integrity error on user role assignment on log in: {e}"
+            )
         except DBAPIError as e:
             db.session.rollback()
-            current_app.logger.error(f"Unknown error on user role assignment on log in: {e}")
+            current_app.logger.error(
+                f"Unknown error on user role assignment on log in: {e}"
+            )
 
         return user
 
@@ -163,7 +170,9 @@ def auth_oid_authenticate(self, token):
         user = self.data_model.get_one_object(username=username)
 
         if not user:
-            current_app.logger.info(f"OpenID user {username} does not exist and is created")
+            current_app.logger.info(
+                f"OpenID user {username} does not exist and is created"
+            )
 
             data = {"username": username, "email": username}
 
@@ -183,7 +192,11 @@ def auth_oid_authenticate(self, token):
 
 def check_last_password_change(user):
     if user.pwd_last_change:
-        if user.pwd_last_change + timedelta(days=int(current_app.config["PWD_ROTATION_TIME"])) < datetime.utcnow():
+        if (
+            user.pwd_last_change
+            + timedelta(days=int(current_app.config["PWD_ROTATION_TIME"]))
+            < datetime.utcnow()
+        ):
             return True
     return False
 

cornflow-server/setup.py

@@ -9,7 +9,7 @@
 
 setuptools.setup(
     name="cornflow",
-    version="1.1.2",
+    version="1.1.3",
     author="baobab soluciones",
     author_email="[email protected]",
     description="Cornflow is an open source multi-solver optimization server with a REST API built using flask.",

libs/client/changelog.rst

@@ -1,7 +1,18 @@
+version 1.1.3
+------------
+
+- released: 2024-12-05
+- description: changes to json schemas validation on airflow.
+- changelog:
+    - changed the json schema validation on airflow so that solution, instance checks and solution checks are correctly reviewed.
+    - added some small changes to make sure that future compatibility with new version of libraries is ready.
+    - added a reconnect from airflow to cornflow to make sure that if the model fails we can get back as much as possible.
+
+
 version 1.1.1
 --------------
 
-- released: 2024-08-29
+- released: 2024-09-18
 - description: small security fixes
 - changelog:
     - bump PuLP to version 2.9.0

libs/client/cornflow_client/airflow/dag_utilities.py

@@ -226,13 +226,17 @@ def cf_solve(fun, dag_name, secrets, **kwargs):
     except NoSolverException as e:
         if config.get("msg", True):
             print("No solver found !")
+        # We reconnect in case the solver has been more than 24 hours solving before the error is raised
+        client = connect_to_cornflow(secrets)
         try_to_save_error(client, exec_id, -1)
         client.update_status(exec_id, {"status": -1})
         try_to_save_airflow_log(client, exec_id, ti, base_log_folder)
         raise AirflowDagException(e)
     except Exception as e:
         if config.get("msg", True):
             print("Some unknown error happened")
+        # We reconnect in case the solver has been more than 24 hours solving before the error is raised
+        client = connect_to_cornflow(secrets)
         try_to_save_error(client, exec_id, -1)
         client.update_status(exec_id, {"status": -1})
         try_to_save_airflow_log(client, exec_id, ti, base_log_folder)

libs/client/cornflow_client/constants.py

@@ -1,6 +1,7 @@
 """
 Constants values used in schemas functions.
 """
+
 import pulp as pl
 from ortools.sat.python import cp_model
 
@@ -119,7 +120,6 @@
     ("timeLimit", "pyomo", "cbc"): "sec",
     ("pump_passes", "pyomo", "cbc"): "pumpC",
     ("heuristics", "pyomo", "cbc"): "heur",
-
     ("is_mip", "pulp", "cbc"): "mip",
     ("abs_gap", "pulp", "cbc"): "gapAbs",
     ("rel_gap", "pulp", "cbc"): "gapRel",
@@ -129,7 +129,6 @@
     ("threads", "pulp", "cbc"): "threads",
     ("presolve", "pulp", "cbc"): "presolve",
     ("msg", "pulp", "cbc"): "msg",
-
     ("abs_gap", "pyomo", "gurobi"): "MIPGapAbs",
     ("rel_gap", "pyomo", "gurobi"): "MIPGap",
     ("time_limit", "pyomo", "gurobi"): "TimeLimit",
@@ -145,7 +144,6 @@
     ("pump_passes", "pyomo", "gurobi"): "PumpPasses",
     ("heuristics", "pyomo", "gurobi"): "Heuristics",
     ("threads", "pyomo", "gurobi"): "threads",
-
     ("is_mip", "pulp", "gurobi"): "mip",
     ("abs_gap", "pulp", "gurobi"): "gapAbs",
     ("rel_gap", "pulp", "gurobi"): "gapRel",
@@ -155,7 +153,6 @@
     ("feasibility_tol", "pulp", "gurobi"): "FeasibilityTol",
     ("iteration_limit", "pulp", "gurobi"): "IterationLimit",
     ("msg", "pulp", "gurobi"): "msg",
-
     ("abs_gap", "pyomo", "scip"): "limits/absgap",
     ("rel_gap", "pyomo", "scip"): "limits/gap",
     ("time_limit", "pyomo", "scip"): "limits/time",
@@ -171,7 +168,6 @@
     ("nlp_tol", "pyomo", "scip"): "heuristics/subnlp/opttol",
     ("cutoff", "pyomo", "scip"): "heuristics/subnlp/setcutoff",
     ("nlp_iteration_limit", "pyomo", "scip"): "heuristics/subnlp/itermin",
-
     ("is_mip", "pulp", "scip"): "mip",
     ("abs_gap", "pulp", "scip"): "gapAbs",
     ("rel_gap", "pulp", "scip"): "gapRel",
@@ -186,7 +182,6 @@
     ("solution_limit", "pulp", "scip"): "limits/maxsol",
     ("nlp_iteration_limit", "pulp", "scip"): "heuristics/subnlp/itermin",
     ("msg", "pulp", "scip"): "msg",
-
     ("rel_gap", "pyomo", "highs"): "mip_rel_gap",
     ("abs_gap", "pyomo", "highs"): "mip_abs_gap",
     ("time_limit", "pyomo", "highs"): "time_limit",
@@ -195,7 +190,6 @@
     ("parallel", "pyomo", "highs"): "parallel",
     ("crossover", "pyomo", "highs"): "run_crossover",
     ("heuristics", "pyomo", "highs"): "mip_heuristic_effort",
-
     ("is_mip", "pulp", "highs"): "mip",
     ("abs_gap", "pulp", "highs"): "gapAbs",
     ("rel_gap", "pulp", "highs"): "gapRel",
@@ -213,7 +207,6 @@
     "cbc": "cbc",
     "scip": "scip",
     "highs": "highs",
-
     # PULP
     "PULP_CBC_CMD": "cbc",
     "GUROBI_CMD": "gurobi",
@@ -262,3 +255,11 @@ class BadInstance(Exception):
 
 class BadSolution(Exception):
     pass
+
+
+class BadInstanceChecks(Exception):
+    pass
+
+
+class BadSolutionChecks(Exception):
+    pass