From d0b6dbdbf6eb7e7440d7cb32bd7f747f81ec0340 Mon Sep 17 00:00:00 2001 From: Bochuan Lyu Date: Fri, 19 Jan 2024 12:17:42 -0500 Subject: [PATCH 01/11] Add SAS API interface --- pulp/apis/__init__.py | 10 + pulp/apis/sas_api.py | 720 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 730 insertions(+) create mode 100755 pulp/apis/sas_api.py diff --git a/pulp/apis/__init__.py b/pulp/apis/__init__.py index 41c3240f..c2e62979 100644 --- a/pulp/apis/__init__.py +++ b/pulp/apis/__init__.py @@ -9,6 +9,7 @@ from .xpress_api import * from .highs_api import * from .copt_api import * +from .sas_api import * from .core import * _all_solvers = [ @@ -35,6 +36,8 @@ COPT, COPT_DLL, COPT_CMD, + SAS94, + SASCAS ] import json @@ -156,3 +159,10 @@ def listSolvers(onlyAvailable=False): result.append(solver.name) del solver return result + + +# DEPRECATED aliases: +get_solver = getSolver +get_solver_from_json = getSolverFromJson +get_solver_from_dict = getSolverFromDict +list_solvers = listSolvers diff --git a/pulp/apis/sas_api.py b/pulp/apis/sas_api.py new file mode 100755 index 00000000..329d79ab --- /dev/null +++ b/pulp/apis/sas_api.py @@ -0,0 +1,720 @@ +# PuLP : Python LP Modeler +# Version 1.4.2 + +# Copyright (c) 2002-2005, Jean-Sebastien Roy (js@jeannot.org) +# Modifications Copyright (c) 2007- Stuart Anthony Mitchell (s.mitchell@auckland.ac.nz) +# $Id:solvers.py 1791 2008-04-23 22:54:34Z smit023 $ + +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: + +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.""" + +from .core import LpSolver_CMD, LpSolver, subprocess, PulpSolverError, clock, log +from .core import gurobi_path +from io import StringIO +from contextlib import redirect_stdout +import os +import sys +from .. import constants +import warnings +from typing import Union + +from uuid import uuid4 + +# STATUS_TO_SOLVERSTATUS = { +# "OK": SolverStatus.ok, +# "SYNTAX_ERROR": SolverStatus.error, +# "DATA_ERROR": SolverStatus.error, +# "OUT_OF_MEMORY": SolverStatus.aborted, +# "IO_ERROR": SolverStatus.error, +# "ERROR": SolverStatus.error, +# } + +# The maximum length of the names of variables and constraints. +MAX_NAME_LENGTH=256 + +# This combines all status codes from OPTLP/solvelp and OPTMILP/solvemilp +SOLSTATUS_TO_STATUS = { + "OPTIMAL": constants.LpStatusOptimal, + "OPTIMAL_AGAP": constants.LpStatusOptimal, + "OPTIMAL_RGAP": constants.LpStatusOptimal, + "OPTIMAL_COND": constants.LpStatusOptimal, + "TARGET": constants.LpStatusOptimal, + "CONDITIONAL_OPTIMAL": constants.LpStatusOptimal, + "FEASIBLE": constants.LpStatusNotSolved, + "INFEASIBLE": constants.LpStatusInfeasible, + "UNBOUNDED": constants.LpStatusUnbounded, + "INFEASIBLE_OR_UNBOUNDED": constants.LpStatusUnbounded, + "SOLUTION_LIM": constants.LpStatusNotSolved, + "NODE_LIM_SOL": constants.LpStatusNotSolved, + "NODE_LIM_NOSOL": constants.LpStatusNotSolved, + "ITERATION_LIMIT_REACHED": constants.LpStatusNotSolved, + "TIME_LIM_SOL": constants.LpStatusNotSolved, + "TIME_LIM_NOSOL": constants.LpStatusNotSolved, + "TIME_LIMIT_REACHED": constants.LpStatusNotSolved, + "ABORTED": constants.LpStatusNotSolved, + "ABORT_SOL": constants.LpStatusNotSolved, + "ABORT_NOSOL": constants.LpStatusNotSolved, + "OUTMEM_SOL": constants.LpStatusNotSolved, + "OUTMEM_NOSOL": constants.LpStatusNotSolved, + "FAILED": constants.LpStatusNotSolved, + "FAIL_SOL": constants.LpStatusNotSolved, + "FAIL_NOSOL": constants.LpStatusNotSolved, + "ERROR": constants.LpStatusNotSolved, +} + + +# SOLSTATUS_TO_MESSAGE = { +# "OPTIMAL": "The solution is optimal.", +# "OPTIMAL_AGAP": "The solution is optimal within the absolute gap specified by the ABSOBJGAP= option.", +# "OPTIMAL_RGAP": "The solution is optimal within the relative gap specified by the RELOBJGAP= option.", +# "OPTIMAL_COND": "The solution is optimal, but some infeasibilities (primal, bound, or integer) exceed tolerances due to scaling or choice of a small INTTOL= value.", +# "TARGET": "The solution is not worse than the target specified by the TARGET= option.", +# "CONDITIONAL_OPTIMAL": "The solution is optimal, but some infeasibilities (primal, dual or bound) exceed tolerances due to scaling or preprocessing.", +# "FEASIBLE": "The problem is feasible. This status is displayed when the IIS=TRUE option is specified and the problem is feasible.", +# "INFEASIBLE": "The problem is infeasible.", +# "UNBOUNDED": "The problem is unbounded.", +# "INFEASIBLE_OR_UNBOUNDED": "The problem is infeasible or unbounded.", +# "SOLUTION_LIM": "The solver reached the maximum number of solutions specified by the MAXSOLS= option.", +# "NODE_LIM_SOL": "The solver reached the maximum number of nodes specified by the MAXNODES= option and found a solution.", +# "NODE_LIM_NOSOL": "The solver reached the maximum number of nodes specified by the MAXNODES= option and did not find a solution.", +# "ITERATION_LIMIT_REACHED": "The maximum allowable number of iterations was reached.", +# "TIME_LIM_SOL": "The solver reached the execution time limit specified by the MAXTIME= option and found a solution.", +# "TIME_LIM_NOSOL": "The solver reached the execution time limit specified by the MAXTIME= option and did not find a solution.", +# "TIME_LIMIT_REACHED": "The solver reached its execution time limit.", +# "ABORTED": "The solver was interrupted externally.", +# "ABORT_SOL": "The solver was stopped by the user but still found a solution.", +# "ABORT_NOSOL": "The solver was stopped by the user and did not find a solution.", +# "OUTMEM_SOL": "The solver ran out of memory but still found a solution.", +# "OUTMEM_NOSOL": "The solver ran out of memory and either did not find a solution or failed to output the solution due to insufficient memory.", +# "FAILED": "The solver failed to converge, possibly due to numerical issues.", +# "FAIL_SOL": "The solver stopped due to errors but still found a solution.", +# "FAIL_NOSOL": "The solver stopped due to errors and did not find a solution.", +# } + + +CAS_OPTION_NAMES = [ + "hostname", + "port", + "username", + "password", + "session", + "locale", + "name", + "nworkers", + "authinfo", + "protocol", + "path", + "ssl_ca_list", + "authcode", +] + + +class SAS94(LpSolver): + name = "SAS94" + + try: + global saspy + import saspy + + except ImportError: + + def available(self): + """True if SAS94 is available.""" + return False + + def actualSolve(self, lp, callback=None): + """Solves a well-formulated lp problem.""" + raise PulpSolverError("SAS94 : Not Available") + + else: + saspy.logger.setLevel(log.level) + def __init__( + self, + mip=True, + msg=True, + warmStart=False, + **solverParams, + ): + """ + :param bool mip: if False, assume LP even if integer variables + :param bool msg: if False, no log is shown + :param bool warmStart: if False, no warmstart or initial primal solution provided. + """ + LpSolver.__init__( + self, + mip=mip, + msg=msg, + ) + self._sas = saspy.SASsession() + self.warmStart=warmStart + # Only named options are allowed in SAS solvers. + self.solverOptions = {key:value for key, value in solverParams.items()} + def available(self): + """True if SAS94 is available.""" + return True + + # def __del__(self): + # self._sas.endsas() + + def _create_statement_str(self, statement): + """Helper function to create the strings for the statements of the proc OPTLP/OPTMILP code.""" + stmt = self.solverOptions.pop(statement, None) + if stmt: + return ( + statement.strip() + + " " + + " ".join(option + "=" + str(value) for option, value in stmt.items()) + + ";" + ) + else: + return "" + + def toDict(self): + data = dict(solver=self.name) + for k in ["mip", "msg", "warmStart"]: + try: + data[k] = getattr(self, k) + except AttributeError: + pass + data.update(self.solverOptions) + return data + + def _create_tmp_files(self, name, *args): + + return (f"{name}-pulp.{n}" for n in args) + + def _delete_tmp_files(self, *args): + for file in args: + try: + os.remove(file) + except FileNotFoundError: + pass + + def _write_sol(self, filename, vs): + """Writes a SAS solution file""" + values = [(v.name, v.value()) for v in vs if v.value() is not None] + with open(filename, "w") as f: + f.write('_VAR_,_VALUE_\n') + for name, value in values: + f.write(f"{name},{value}\n") + return True + + def _get_max_upload_len(self, fileName): + maxLen = 0 + with open(fileName, 'r') as f: + for line in f.readlines(): + maxLen = max(maxLen, max([len(word) for word in line.split(" ")])) + return maxLen + 1 + + def actualSolve(self, lp): + """Solve a well formulated lp problem""" + log.debug("Running SAS") + + if len(lp.sos1) or len(lp.sos2): + raise PulpSolverError("SAS94: Currently SAS doesn't support SOS1 and SOS2.") + + postfix = uuid4().hex[:16] + tmpMps, tmpMst = self._create_tmp_files(postfix, "mps", "mst") + vs = lp.writeMPS(tmpMps, with_objsense=False) + + nameLen = self._get_max_upload_len(tmpMps) + if nameLen > MAX_NAME_LENGTH: + raise PulpSolverError(f"SAS94: The lengths of the variable or constraint names \ + (including indices) should not exceed {MAX_NAME_LENGTH}.") + + proc = self.proc = "OPTMILP" if self.mip else "OPTLP" + + # Get Obj Sense + if lp.sense == constants.LpMaximize: + self.solverOptions["objsense"] = "max" + elif lp.sense == constants.LpMinimize: + self.solverOptions["objsense"] = "min" + else: + raise PulpSolverError("SAS94 : Objective sense should be min or max.") + # Get the rootnode options + decomp_str = self._create_statement_str("decomp") + decompmaster_str = self._create_statement_str("decompmaster") + decompmasterip_str = self._create_statement_str("decompmasterip") + decompsubprob_str = self._create_statement_str("decompsubprob") + rootnode_str = self._create_statement_str("rootnode") + + if lp.isMIP(): + if not self.mip: + warnings.warn("SAS94 will solve the relaxed problem of the MILP instance.") + # Handle warmstart + warmstart_str = "" + if self.warmStart: + self._write_sol(filename=tmpMst, vs=vs) + + # Set the warmstart basis option + if proc == "OPTMILP": + warmstart_str = """ + proc import datafile='{primalin}' + out=primalin{postfix} + dbms=csv + replace; + getnames=yes; + run; + """.format( + primalin=tmpMst, + postfix=postfix, + ) + self.solverOptions["primalin"] = f"primalin{postfix}" + elif proc == "OPTLP": + pass + + # Convert options to string + opt_str = " ".join( + option + "=" + str(value) for option, value in self.solverOptions.items() + ) + sas = self._sas + # Start a SAS session, submit the code and return the results`` + # with saspy.SASsession() as sas: + # self.solveTime = -clock() + # Find the version of 9.4 we are using + if sas.sasver.startswith("9.04.01M5"): + # In 9.4M5 we have to create an MPS data set from an MPS file first + # Earlier versions will not work because the MPS format in incompatible' + res = sas.submit( + """ + option notes nonumber nodate nosource pagesize=max; + {warmstart} + %MPS2SASD(MPSFILE="{mpsfile}", OUTDATA=mpsdata{postfix}, MAXLEN={maxLen}, FORMAT=FREE); + proc {proc} data=mpsdata{postfix} {options} primalout=primalout{postfix} dualout=dualout{postfix}; + {decomp} + {decompmaster} + {decompmasterip} + {decompsubprob} + {rootnode} + proc delete data=mpsdata{postfix}; + run; + """.format( + warmstart=warmstart_str, + postfix=postfix, + mpsfile=tmpMps, + proc=proc, + maxLen=min(nameLen, MAX_NAME_LENGTH), + options=opt_str, + decomp=decomp_str, + decompmaster=decompmaster_str, + decompmasterip=decompmasterip_str, + decompsubprob=decompsubprob_str, + rootnode=rootnode_str, + ), + results="TEXT", + ) + else: + # Since 9.4M6+ optlp/optmilp can read mps files directly + # TODO: Check whether there are limits for length of variable and constraint names + res = sas.submit( + """ + option notes nonumber nodate nosource pagesize=max; + {warmstart} + proc {proc} mpsfile=\"{mpsfile}\" {options} primalout=primalout{postfix} dualout=dualout{postfix}; + {decomp} + {decompmaster} + {decompmasterip} + {decompsubprob} + {rootnode} + run; + """.format( + warmstart=warmstart_str, + postfix=postfix, + proc=proc, + mpsfile=tmpMps, + options=opt_str, + decomp=decomp_str, + decompmaster=decompmaster_str, + decompmasterip=decompmasterip_str, + decompsubprob=decompsubprob_str, + rootnode=rootnode_str, + ), + results="TEXT", + ) + # TODO: make sure all the files handled correctly. + # breakpoint() + + + self._delete_tmp_files(tmpMps, tmpMst) + + # self.solveTime += clock() + + # Store log and ODS output + self._log = res["LOG"] + if self.msg: + print(self._log) + # if "ERROR 22-322: Syntax error" in self._log: + # raise PulpSolverError( + # "An option passed to the SAS solver caused a syntax error: {_log}".format( + # _log=self._log + # ) + # ) + self._macro = dict( + (key.strip(), value.strip()) + for key, value in ( + pair.split("=") for pair in sas.symget("_OR" + proc + "_").split() + ) + ) + + # self._lst = res["LST"] + + + + primal_out = sas.sd2df(f"primalout{postfix}") + dual_out = sas.sd2df(f"dualout{postfix}") + # t = time.time() + # # sas.endsas() + # print("Actual solve time: ", time.time() - t) + # # Remove primalout and dualout in SAS. + # rm_text = sas.submit( + # """ + # proc delete data=primalout{postfix}; + # proc delete data=dualout{postfix}; + # run; + # """.format(postfix=postfix), + # result="TEXT", + # ) + + + # print(rm_text["LOG"]) + # t = -clock() + # print(clock()+t) + + + if self._macro.get("STATUS", "ERROR") != "OK": + raise PulpSolverError("PuLP: Error ({err_name}) \ + while trying to solve the instance: {name}".format( + err_name=self._macro.get("STATUS", "ERROR"), name=lp.name)) + # print(self._macro) + status = self._read_solution(lp, primal_out, dual_out) + + return status + + def _read_solution(self, lp, primal_out, dual_out): + # print("The status: ", self._macro.get("SOLUTION_STATUS", "ERROR")) + status = SOLSTATUS_TO_STATUS[self._macro.get("SOLUTION_STATUS", "ERROR")] + + if self.proc == "OPTLP": + # TODO: Check whether there is better implementation than zip(). + values = dict(zip(primal_out['_VAR_'], primal_out['_VALUE_'])) + rc = dict(zip(primal_out['_VAR_'], primal_out['_R_COST_'])) + lp.assignVarsVals(values) + lp.assignVarsDj(rc) + + prices = dict(zip(dual_out['_ROW_'], dual_out['_VALUE_'])) + slacks = dict(zip(dual_out['_ROW_'], dual_out['_ACTIVITY_'])) + lp.assignConsPi(prices) + lp.assignConsSlack(slacks, activity=True) + else: + # Convert primal out data set to variable dictionary + # Use pandas functions for efficiency + values = dict(zip(primal_out['_VAR_'], primal_out['_VALUE_'])) + lp.assignVarsVals(values) + lp.assignStatus(status) + return status + + +class SASCAS(LpSolver): + name = "SASCAS" + + try: + global swat + import swat + + except ImportError: + + def available(self): + """True if SASCAS is available.""" + return False + + def actualSolve(self, lp, callback=None): + """Solves a well-formulated lp problem.""" + raise PulpSolverError("SASCAS : Not Available") + + else: + def __init__( + self, + mip=True, + msg=True, + warmStart=False, + cas_options={}, + **solverParams, + ): + """ + :param bool mip: if False, assume LP even if integer variables + :param bool msg: if False, no log is shown + :param bool warmStart: if False, no warmstart or initial primal solution provided. + """ + LpSolver.__init__( + self, + mip=mip, + msg=msg, + ) + self._sas = swat + self.warmStart=warmStart + self.cas_options=cas_options + # Only named options are allowed in SAS solvers. + self.solverOptions = {key:value for key, value in solverParams.items()} + def available(self): + """True if SAS94 is available.""" + return True + + + def toDict(self): + data = dict(solver=self.name) + for k in ["mip", "msg", "warmStart", "cas_options"]: + try: + data[k] = getattr(self, k) + except AttributeError: + pass + data.update(self.solverOptions) + return data + + def _create_tmp_files(self, name, *args): + + return (f"{name}-pulp.{n}" for n in args) + + def _delete_tmp_files(self, *args): + for file in args: + try: + os.remove(file) + except FileNotFoundError: + pass + + def _write_sol(self, filename, vs): + """Writes a SAS solution file""" + values = [(v.name, v.value()) for v in vs if v.value() is not None] + with open(filename, "w") as f: + f.write('_VAR_,_VALUE_\n') + for name, value in values: + f.write(f"{name},{value}\n") + return True + + def _read_solution(self, lp, primal_out, dual_out): + # print("The status: ", self._macro.get("SOLUTION_STATUS", "ERROR")) + status = SOLSTATUS_TO_STATUS[self._macro.get("SOLUTION_STATUS", "ERROR")] + + if self.proc == "OPTLP": + # TODO: Check whether there is better implementation than zip(). + values = dict(zip(primal_out['_VAR_'], primal_out['_VALUE_'])) + rc = dict(zip(primal_out['_VAR_'], primal_out['_R_COST_'])) + lp.assignVarsVals(values) + lp.assignVarsDj(rc) + + prices = dict(zip(dual_out['_ROW_'], dual_out['_VALUE_'])) + slacks = dict(zip(dual_out['_ROW_'], dual_out['_ACTIVITY_'])) + lp.assignConsPi(prices) + lp.assignConsSlack(slacks, activity=True) + else: + # Convert primal out data set to variable dictionary + # Use pandas functions for efficiency + values = dict(zip(primal_out['_VAR_'], primal_out['_VALUE_'])) + lp.assignVarsVals(values) + lp.assignStatus(status) + return status + + # TODO: Improve it without reading the mps file. + def _get_max_upload_len(self, fileName): + maxLen = 0 + with open(fileName, 'r') as f: + for line in f.readlines(): + maxLen = max(maxLen, max([len(word) for word in line.split(" ")])) + return maxLen + 1 + # varLen = max(len(varName) for varName in lp.variablesDict().keys()) + # conLen = max(len(varName) for varName in lp.constraints.keys()) + # return max(varLen, conLen, 2) + 1 + + def actualSolve(self, lp): + """Solve a well formulated lp problem""" + log.debug("Running SAS") + + if (self.cas_options == {}): + # print("cas_options:", CAS_OPTION_NAMES) + raise PulpSolverError("""SASCAS: Provide cas_options with + {port: , host: , authinfo: } + or {port: , host: , username: , password: }.""") + + if len(lp.sos1) or len(lp.sos2): + raise PulpSolverError("SASCAS: Currently SAS doesn't support SOS1 and SOS2.") + + + # breakpoint() + proc = self.proc = "OPTMILP" if self.mip else "OPTLP" + # Get Obj Sense + if lp.sense == constants.LpMaximize: + self.solverOptions["objsense"] = "max" + elif lp.sense == constants.LpMinimize: + self.solverOptions["objsense"] = "min" + else: + raise PulpSolverError("SASCAS : Objective sense should be min or max.") + + status = None + with redirect_stdout(SASLogWriter(self.msg)) as self._log_writer: + + s = self._sas.CAS(**self.cas_options) + postfix = uuid4().hex[:16] + tmpMps, tmpMpsCsv, tmpMstCsv = self._create_tmp_files(postfix, "mps", "mps.csv", "mst.csv") + vs = lp.writeMPS(tmpMps, with_objsense=False) + + nameLen = self._get_max_upload_len(tmpMps) + if nameLen > MAX_NAME_LENGTH: + raise PulpSolverError(f"SASCAS: The lengths of the variable or constraint names \ + (including indices) should not exceed {MAX_NAME_LENGTH}.") + try: + # Load the optimization action set + s.loadactionset('optimization') + if lp.isMIP(): + if not self.mip: + warnings.warn("SASCAS will solve the relaxed problem of the MILP instance.") + + if os.stat(tmpMps).st_size >= 2 * 1024 **3: + # if False: + # For large files, use convertMPS, first create file for upload + with open(tmpMpsCsv, 'w') as mpsWithId: + mpsWithId.write('_ID_\tText\n') + with open(tmpMps, 'r') as f: + id = 0 + for line in f: + id += 1 + mpsWithId.write(str(id) + '\t' + line.rstrip() + '\n') + + # Upload .mps.csv file + s.upload_file( + tmpMpsCsv, + casout={"name": f"mpscsv{postfix}", "replace": True}, + importoptions={"filetype": "CSV", "delimiter": "\t"}, + ) + + # Convert .mps.csv file to .mps + s.optimization.convertMps( + data=f"mpscsv{postfix}", + casOut={"name": f"mpsdata{postfix}", "replace": True}, + format="FREE", + maxLength=min(nameLen, MAX_NAME_LENGTH), + ) + else: + # For small files, use loadMPS + with open(tmpMps, 'r') as mps_file: + s.optimization.loadMps( + mpsFileString=mps_file.read(), + casout={"name": f"mpsdata{postfix}", "replace": True}, + format="FREE", + maxLength=min(nameLen, MAX_NAME_LENGTH), + ) + + if self.warmStart and (proc == "OPTMILP"): + self._write_sol(filename=tmpMstCsv, vs=vs) + # Upload warmstart file to CAS + s.upload_file( + tmpMstCsv, + casout={"name": f"primalin{postfix}", "replace": True}, + importoptions={"filetype": "CSV"}, + ) + self.solverOptions["primalin"] = f"primalin{postfix}" + # breakpoint() + # Delete the temp files. + self._delete_tmp_files(tmpMps, tmpMstCsv, tmpMpsCsv) + + # Solve the problem in CAS + if proc == "OPTMILP": + r = s.optimization.solveMilp( + data={"name": f"mpsdata{postfix}"}, + primalOut={"name": f"primalout{postfix}", "replace": True}, + **self.solverOptions + ) + else: + r = s.optimization.solveLp( + data={"name": f"mpsdata{postfix}"}, + primalOut={"name": f"primalout{postfix}", "replace": True}, + dualOut={"name": f"dualout{postfix}", "replace": True}, + **self.solverOptions + ) + if r: + self._macro = { + "STATUS": r.get("status", "ERROR").upper(), + "SOLUTION_STATUS": r.get("solutionStatus", "ERROR").upper() + } + if self._macro.get("STATUS", "ERROR") != "OK": + raise PulpSolverError("PuLP: Error ({err_name}) while trying to solve the instance: {name}".format( + err_name=self._macro.get("STATUS", "ERROR"), name=lp.name)) + # print("OBJ solution:", r["objective"]) + # If we get solution successfully. + if proc == "OPTMILP": + primal_out = s.CASTable(name=f"primalout{postfix}") + primal_out = primal_out[ + ['_VAR_', '_VALUE_', '_STATUS_', '_R_COST_'] + ] + dual_out = None + else: + primal_out = s.CASTable(name=f"primalout{postfix}") + primal_out = primal_out[ + ['_VAR_', '_VALUE_', '_STATUS_', '_R_COST_'] + ] + dual_out = s.CASTable(name=f"dualout{postfix}") + dual_out = dual_out[ + ['_ROW_', '_VALUE_', '_STATUS_', '_ACTIVITY_'] + ] + status = self._read_solution(lp, primal_out, dual_out) + finally: + self._delete_tmp_files(tmpMps, tmpMstCsv, tmpMpsCsv) + s.close() + + if self.msg: + print(self._log_writer.log()) + if status: + return status + else: + raise PulpSolverError(f"PuLP: Error while trying to solve the instance: \ + {lp.name} via SASCAS.") + + +class SASLogWriter: + """Helper class to take the log from stdout and put it also in a StringIO.""" + + def __init__(self, tee): + """Set up the two outputs.""" + self.tee = tee + self._log = StringIO() + self.stdout = sys.stdout + + def write(self, message): + """If the tee options is specified, write to both outputs.""" + if self.tee: + self.stdout.write(message) + self._log.write(message) + + def flush(self): + """Nothing to do, just here for compatibility reasons.""" + # Do nothing since we flush right away + pass + + def log(self): + """ "Get the log as a string.""" + return self._log.getvalue() + + + + + + + + + + + From 6f06de51d522a76348769d374c1a81fe9d4a16d2 Mon Sep 17 00:00:00 2001 From: Bochuan Lyu Date: Fri, 19 Jan 2024 12:42:00 -0500 Subject: [PATCH 02/11] Update the version --- pulp/apis/__init__.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/pulp/apis/__init__.py b/pulp/apis/__init__.py index c2e62979..bd333b84 100644 --- a/pulp/apis/__init__.py +++ b/pulp/apis/__init__.py @@ -159,10 +159,3 @@ def listSolvers(onlyAvailable=False): result.append(solver.name) del solver return result - - -# DEPRECATED aliases: -get_solver = getSolver -get_solver_from_json = getSolverFromJson -get_solver_from_dict = getSolverFromDict -list_solvers = listSolvers From a164338717b243539a9ec85a0a9d676a3ce273e6 Mon Sep 17 00:00:00 2001 From: Bochuan Lyu Date: Mon, 22 Jan 2024 17:50:28 -0500 Subject: [PATCH 03/11] Remove unused comments --- pulp/apis/sas_api.py | 107 ++----------------------------------------- 1 file changed, 5 insertions(+), 102 deletions(-) diff --git a/pulp/apis/sas_api.py b/pulp/apis/sas_api.py index 329d79ab..ca45e2d7 100755 --- a/pulp/apis/sas_api.py +++ b/pulp/apis/sas_api.py @@ -36,15 +36,6 @@ from uuid import uuid4 -# STATUS_TO_SOLVERSTATUS = { -# "OK": SolverStatus.ok, -# "SYNTAX_ERROR": SolverStatus.error, -# "DATA_ERROR": SolverStatus.error, -# "OUT_OF_MEMORY": SolverStatus.aborted, -# "IO_ERROR": SolverStatus.error, -# "ERROR": SolverStatus.error, -# } - # The maximum length of the names of variables and constraints. MAX_NAME_LENGTH=256 @@ -78,36 +69,6 @@ "ERROR": constants.LpStatusNotSolved, } - -# SOLSTATUS_TO_MESSAGE = { -# "OPTIMAL": "The solution is optimal.", -# "OPTIMAL_AGAP": "The solution is optimal within the absolute gap specified by the ABSOBJGAP= option.", -# "OPTIMAL_RGAP": "The solution is optimal within the relative gap specified by the RELOBJGAP= option.", -# "OPTIMAL_COND": "The solution is optimal, but some infeasibilities (primal, bound, or integer) exceed tolerances due to scaling or choice of a small INTTOL= value.", -# "TARGET": "The solution is not worse than the target specified by the TARGET= option.", -# "CONDITIONAL_OPTIMAL": "The solution is optimal, but some infeasibilities (primal, dual or bound) exceed tolerances due to scaling or preprocessing.", -# "FEASIBLE": "The problem is feasible. This status is displayed when the IIS=TRUE option is specified and the problem is feasible.", -# "INFEASIBLE": "The problem is infeasible.", -# "UNBOUNDED": "The problem is unbounded.", -# "INFEASIBLE_OR_UNBOUNDED": "The problem is infeasible or unbounded.", -# "SOLUTION_LIM": "The solver reached the maximum number of solutions specified by the MAXSOLS= option.", -# "NODE_LIM_SOL": "The solver reached the maximum number of nodes specified by the MAXNODES= option and found a solution.", -# "NODE_LIM_NOSOL": "The solver reached the maximum number of nodes specified by the MAXNODES= option and did not find a solution.", -# "ITERATION_LIMIT_REACHED": "The maximum allowable number of iterations was reached.", -# "TIME_LIM_SOL": "The solver reached the execution time limit specified by the MAXTIME= option and found a solution.", -# "TIME_LIM_NOSOL": "The solver reached the execution time limit specified by the MAXTIME= option and did not find a solution.", -# "TIME_LIMIT_REACHED": "The solver reached its execution time limit.", -# "ABORTED": "The solver was interrupted externally.", -# "ABORT_SOL": "The solver was stopped by the user but still found a solution.", -# "ABORT_NOSOL": "The solver was stopped by the user and did not find a solution.", -# "OUTMEM_SOL": "The solver ran out of memory but still found a solution.", -# "OUTMEM_NOSOL": "The solver ran out of memory and either did not find a solution or failed to output the solution due to insufficient memory.", -# "FAILED": "The solver failed to converge, possibly due to numerical issues.", -# "FAIL_SOL": "The solver stopped due to errors but still found a solution.", -# "FAIL_NOSOL": "The solver stopped due to errors and did not find a solution.", -# } - - CAS_OPTION_NAMES = [ "hostname", "port", @@ -169,9 +130,6 @@ def available(self): """True if SAS94 is available.""" return True - # def __del__(self): - # self._sas.endsas() - def _create_statement_str(self, statement): """Helper function to create the strings for the statements of the proc OPTLP/OPTMILP code.""" stmt = self.solverOptions.pop(statement, None) @@ -347,24 +305,13 @@ def actualSolve(self, lp): ), results="TEXT", ) - # TODO: make sure all the files handled correctly. - # breakpoint() - self._delete_tmp_files(tmpMps, tmpMst) - # self.solveTime += clock() - # Store log and ODS output self._log = res["LOG"] if self.msg: print(self._log) - # if "ERROR 22-322: Syntax error" in self._log: - # raise PulpSolverError( - # "An option passed to the SAS solver caused a syntax error: {_log}".format( - # _log=self._log - # ) - # ) self._macro = dict( (key.strip(), value.strip()) for key, value in ( @@ -372,42 +319,19 @@ def actualSolve(self, lp): ) ) - # self._lst = res["LST"] - - - primal_out = sas.sd2df(f"primalout{postfix}") dual_out = sas.sd2df(f"dualout{postfix}") - # t = time.time() - # # sas.endsas() - # print("Actual solve time: ", time.time() - t) - # # Remove primalout and dualout in SAS. - # rm_text = sas.submit( - # """ - # proc delete data=primalout{postfix}; - # proc delete data=dualout{postfix}; - # run; - # """.format(postfix=postfix), - # result="TEXT", - # ) - - - # print(rm_text["LOG"]) - # t = -clock() - # print(clock()+t) if self._macro.get("STATUS", "ERROR") != "OK": raise PulpSolverError("PuLP: Error ({err_name}) \ while trying to solve the instance: {name}".format( err_name=self._macro.get("STATUS", "ERROR"), name=lp.name)) - # print(self._macro) status = self._read_solution(lp, primal_out, dual_out) return status def _read_solution(self, lp, primal_out, dual_out): - # print("The status: ", self._macro.get("SOLUTION_STATUS", "ERROR")) status = SOLSTATUS_TO_STATUS[self._macro.get("SOLUTION_STATUS", "ERROR")] if self.proc == "OPTLP": @@ -507,7 +431,6 @@ def _write_sol(self, filename, vs): return True def _read_solution(self, lp, primal_out, dual_out): - # print("The status: ", self._macro.get("SOLUTION_STATUS", "ERROR")) status = SOLSTATUS_TO_STATUS[self._macro.get("SOLUTION_STATUS", "ERROR")] if self.proc == "OPTLP": @@ -536,16 +459,12 @@ def _get_max_upload_len(self, fileName): for line in f.readlines(): maxLen = max(maxLen, max([len(word) for word in line.split(" ")])) return maxLen + 1 - # varLen = max(len(varName) for varName in lp.variablesDict().keys()) - # conLen = max(len(varName) for varName in lp.constraints.keys()) - # return max(varLen, conLen, 2) + 1 def actualSolve(self, lp): """Solve a well formulated lp problem""" log.debug("Running SAS") if (self.cas_options == {}): - # print("cas_options:", CAS_OPTION_NAMES) raise PulpSolverError("""SASCAS: Provide cas_options with {port: , host: , authinfo: } or {port: , host: , username: , password: }.""") @@ -554,7 +473,6 @@ def actualSolve(self, lp): raise PulpSolverError("SASCAS: Currently SAS doesn't support SOS1 and SOS2.") - # breakpoint() proc = self.proc = "OPTMILP" if self.mip else "OPTLP" # Get Obj Sense if lp.sense == constants.LpMaximize: @@ -627,7 +545,6 @@ def actualSolve(self, lp): importoptions={"filetype": "CSV"}, ) self.solverOptions["primalin"] = f"primalin{postfix}" - # breakpoint() # Delete the temp files. self._delete_tmp_files(tmpMps, tmpMstCsv, tmpMpsCsv) @@ -653,7 +570,6 @@ def actualSolve(self, lp): if self._macro.get("STATUS", "ERROR") != "OK": raise PulpSolverError("PuLP: Error ({err_name}) while trying to solve the instance: {name}".format( err_name=self._macro.get("STATUS", "ERROR"), name=lp.name)) - # print("OBJ solution:", r["objective"]) # If we get solution successfully. if proc == "OPTMILP": primal_out = s.CASTable(name=f"primalout{postfix}") @@ -685,36 +601,23 @@ def actualSolve(self, lp): class SASLogWriter: - """Helper class to take the log from stdout and put it also in a StringIO.""" - + # Helper class to take the log from stdout and put it also in a StringIO. def __init__(self, tee): - """Set up the two outputs.""" + # Set up the two outputs. self.tee = tee self._log = StringIO() self.stdout = sys.stdout def write(self, message): - """If the tee options is specified, write to both outputs.""" + # If the tee options is specified, write to both outputs. if self.tee: self.stdout.write(message) self._log.write(message) def flush(self): - """Nothing to do, just here for compatibility reasons.""" # Do nothing since we flush right away pass def log(self): - """ "Get the log as a string.""" - return self._log.getvalue() - - - - - - - - - - - + # Get the log as a string. + return self._log.getvalue() \ No newline at end of file From ef96a526f604cd58a03ca18fb5ea91d2e384dd64 Mon Sep 17 00:00:00 2001 From: Bochuan Lyu Date: Fri, 2 Feb 2024 14:36:42 -0500 Subject: [PATCH 04/11] Add an empty line to the end of the file --- pulp/apis/sas_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulp/apis/sas_api.py b/pulp/apis/sas_api.py index ca45e2d7..c6184439 100755 --- a/pulp/apis/sas_api.py +++ b/pulp/apis/sas_api.py @@ -620,4 +620,4 @@ def flush(self): def log(self): # Get the log as a string. - return self._log.getvalue() \ No newline at end of file + return self._log.getvalue() From f22dfe48224e27a1bbcbe4e8dbfaf4168b98ca7a Mon Sep 17 00:00:00 2001 From: Bochuan Lyu Date: Tue, 5 Mar 2024 08:53:48 -0500 Subject: [PATCH 05/11] Update SAS API interface --- .github/workflows/pythonpackage.yml | 6 + pulp/apis/__init__.py | 2 +- pulp/apis/sas_api.py | 529 +++++++++++++++------------- pulp/tests/test_pulp.py | 24 ++ 4 files changed, 324 insertions(+), 237 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 29025de5..c6b39815 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -39,6 +39,12 @@ jobs: - name: Install coptpy run: | pip install coptpy + - name: Install saspy + run: | + pip install saspy + - name: Install swat + run: | + pip install swat - name: Test with pulptest run: pulptest - name: Code Quality diff --git a/pulp/apis/__init__.py b/pulp/apis/__init__.py index bd333b84..cfa6150b 100644 --- a/pulp/apis/__init__.py +++ b/pulp/apis/__init__.py @@ -37,7 +37,7 @@ COPT_DLL, COPT_CMD, SAS94, - SASCAS + SASCAS, ] import json diff --git a/pulp/apis/sas_api.py b/pulp/apis/sas_api.py index c6184439..892ca091 100755 --- a/pulp/apis/sas_api.py +++ b/pulp/apis/sas_api.py @@ -24,8 +24,7 @@ # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.""" -from .core import LpSolver_CMD, LpSolver, subprocess, PulpSolverError, clock, log -from .core import gurobi_path +from .core import LpSolver_CMD, LpSolver, PulpSolverError, log from io import StringIO from contextlib import redirect_stdout import os @@ -37,7 +36,7 @@ from uuid import uuid4 # The maximum length of the names of variables and constraints. -MAX_NAME_LENGTH=256 +MAX_NAME_LENGTH = 256 # This combines all status codes from OPTLP/solvelp and OPTMILP/solvemilp SOLSTATUS_TO_STATUS = { @@ -86,115 +85,194 @@ ] -class SAS94(LpSolver): +class SASsolver(LpSolver_CMD): + name = "SASsolver" + + def __init__( + self, + mip=True, + msg=True, + warmStart=False, + keepFiles=False, + timeLimit=None, + **solverParams, + ): + """ + :param bool mip: if False, assume LP even if integer variables + :param bool msg: if False, no log is shown + :param bool warmStart: if False, no warm start + :param bool keepFiles: if False, the generated mps mst files will be removed + :param solverParams: SAS proc OPTMILP or OPTLP parameters + """ + LpSolver_CMD.__init__( + self, mip=mip, msg=msg, warmStart=warmStart, keepFiles=keepFiles + ) + self.timeLimit = timeLimit + # Only named options are allowed in SAS solvers. + self.solverOptions = {key: value for key, value in solverParams.items()} + + def _create_statement_str(self, statement): + """Helper function to create the strings for the statements of the proc OPTLP/OPTMILP code.""" + stmt = self.solverOptions.pop(statement, None) + if stmt: + return ( + statement.strip() + + " " + + " ".join(option + "=" + str(value) for option, value in stmt.items()) + + ";" + ) + else: + return "" + + def defaultPath(self): + return os.path.abspath(os.getcwd()) + + def _write_sol(self, filename, vs): + """Writes a SAS solution file""" + values = [(v.name, v.value()) for v in vs if v.value() is not None] + with open(filename, "w") as f: + f.write("_VAR_,_VALUE_\n") + for name, value in values: + f.write(f"{name},{value}\n") + return True + + def _get_max_upload_len(self, fileName): + maxLen = 0 + with open(fileName, "r") as f: + for line in f.readlines(): + maxLen = max(maxLen, max([len(word) for word in line.split(" ")])) + return maxLen + 1 + + def _read_solution(self, lp, primal_out, dual_out): + status = SOLSTATUS_TO_STATUS[self._macro.get("SOLUTION_STATUS", "ERROR")] + + if self.proc == "OPTLP": + # TODO: Check whether there is better implementation than zip(). + values = dict(zip(primal_out["_VAR_"], primal_out["_VALUE_"])) + rc = dict(zip(primal_out["_VAR_"], primal_out["_R_COST_"])) + lp.assignVarsVals(values) + lp.assignVarsDj(rc) + + prices = dict(zip(dual_out["_ROW_"], dual_out["_VALUE_"])) + slacks = dict(zip(dual_out["_ROW_"], dual_out["_ACTIVITY_"])) + lp.assignConsPi(prices) + lp.assignConsSlack(slacks, activity=True) + else: + # Convert primal out data set to variable dictionary + # Use pandas functions for efficiency + values = dict(zip(primal_out["_VAR_"], primal_out["_VALUE_"])) + lp.assignVarsVals(values) + lp.assignStatus(status) + return status + + +class SAS94(SASsolver): name = "SAS94" try: global saspy import saspy - except ImportError: + except: def available(self): """True if SAS94 is available.""" return False + def sasAvailable(self): + return False + def actualSolve(self, lp, callback=None): """Solves a well-formulated lp problem.""" raise PulpSolverError("SAS94 : Not Available") else: saspy.logger.setLevel(log.level) + def __init__( self, mip=True, msg=True, + keepFiles=False, warmStart=False, + timeLimit=None, + sas=None, **solverParams, ): """ :param bool mip: if False, assume LP even if integer variables :param bool msg: if False, no log is shown - :param bool warmStart: if False, no warmstart or initial primal solution provided. + :param bool keepFiles: if False, mps and mst files will not be saved + :param bool warmStart: if False, no warmstart or initial primal solution provided + :param object sas: sas session. It must be provided by the user. + :param solverParams: SAS proc OPTMILP or OPTLP parameters """ - LpSolver.__init__( + SASsolver.__init__( self, mip=mip, msg=msg, + keepFiles=keepFiles, + warmStart=warmStart, + timeLimit=timeLimit, + **solverParams, ) - self._sas = saspy.SASsession() - self.warmStart=warmStart - # Only named options are allowed in SAS solvers. - self.solverOptions = {key:value for key, value in solverParams.items()} + + self.sas = sas + def available(self): """True if SAS94 is available.""" return True - def _create_statement_str(self, statement): - """Helper function to create the strings for the statements of the proc OPTLP/OPTMILP code.""" - stmt = self.solverOptions.pop(statement, None) - if stmt: - return ( - statement.strip() - + " " - + " ".join(option + "=" + str(value) for option, value in stmt.items()) - + ";" - ) - else: - return "" + def sasAvailable(self): + if not self.sas: + return False + try: + self.sas.sasver + return True + except: + return False def toDict(self): data = dict(solver=self.name) - for k in ["mip", "msg", "warmStart"]: + for k in ["mip", "msg", "keepFiles", "warmStart"]: try: data[k] = getattr(self, k) except AttributeError: pass - data.update(self.solverOptions) - return data - - def _create_tmp_files(self, name, *args): - - return (f"{name}-pulp.{n}" for n in args) - - def _delete_tmp_files(self, *args): - for file in args: + for k in ["sas", "timeLimit"]: + # with these ones, we only export if it has some content: try: - os.remove(file) - except FileNotFoundError: + value = getattr(self, k) + if value: + data[k] = value + except AttributeError: pass - - def _write_sol(self, filename, vs): - """Writes a SAS solution file""" - values = [(v.name, v.value()) for v in vs if v.value() is not None] - with open(filename, "w") as f: - f.write('_VAR_,_VALUE_\n') - for name, value in values: - f.write(f"{name},{value}\n") - return True - - def _get_max_upload_len(self, fileName): - maxLen = 0 - with open(fileName, 'r') as f: - for line in f.readlines(): - maxLen = max(maxLen, max([len(word) for word in line.split(" ")])) - return maxLen + 1 + data.update(self.solverOptions) + return data def actualSolve(self, lp): """Solve a well formulated lp problem""" log.debug("Running SAS") - + if not self.sasAvailable(): + raise PulpSolverError( + "SAS94: Provide a valid SAS session by parameter sas=." + ) if len(lp.sos1) or len(lp.sos2): - raise PulpSolverError("SAS94: Currently SAS doesn't support SOS1 and SOS2.") + raise PulpSolverError( + "SAS94: Currently SAS doesn't support SOS1 and SOS2." + ) postfix = uuid4().hex[:16] - tmpMps, tmpMst = self._create_tmp_files(postfix, "mps", "mst") + tmpMps, tmpMst = self.create_tmp_files(lp.name, "mps", "mst") + vs = lp.writeMPS(tmpMps, with_objsense=False) nameLen = self._get_max_upload_len(tmpMps) if nameLen > MAX_NAME_LENGTH: - raise PulpSolverError(f"SAS94: The lengths of the variable or constraint names \ - (including indices) should not exceed {MAX_NAME_LENGTH}.") + raise PulpSolverError( + f"SAS94: The lengths of the variable or constraint names \ + (including indices) should not exceed {MAX_NAME_LENGTH}." + ) proc = self.proc = "OPTMILP" if self.mip else "OPTLP" @@ -205,6 +283,10 @@ def actualSolve(self, lp): self.solverOptions["objsense"] = "min" else: raise PulpSolverError("SAS94 : Objective sense should be min or max.") + + # Get timeLimit. SAS solvers use MAXTIME instead of timeLimit as a parameter. + if self.timeLimit: + self.solverOptions["MAXTIME"] = self.timeLimit # Get the rootnode options decomp_str = self._create_statement_str("decomp") decompmaster_str = self._create_statement_str("decompmaster") @@ -212,12 +294,13 @@ def actualSolve(self, lp): decompsubprob_str = self._create_statement_str("decompsubprob") rootnode_str = self._create_statement_str("rootnode") - if lp.isMIP(): - if not self.mip: - warnings.warn("SAS94 will solve the relaxed problem of the MILP instance.") + if lp.isMIP() and not self.mip: + warnings.warn( + "SAS94 will solve the relaxed problem of the MILP instance." + ) # Handle warmstart warmstart_str = "" - if self.warmStart: + if self.optionsDict.get("warmStart", False): self._write_sol(filename=tmpMst, vs=vs) # Set the warmstart basis option @@ -233,18 +316,16 @@ def actualSolve(self, lp): primalin=tmpMst, postfix=postfix, ) - self.solverOptions["primalin"] = f"primalin{postfix}" elif proc == "OPTLP": pass # Convert options to string opt_str = " ".join( - option + "=" + str(value) for option, value in self.solverOptions.items() + option + "=" + str(value) + for option, value in self.solverOptions.items() ) - sas = self._sas - # Start a SAS session, submit the code and return the results`` - # with saspy.SASsession() as sas: - # self.solveTime = -clock() + sas = self.sas + # Find the version of 9.4 we are using if sas.sasver.startswith("9.04.01M5"): # In 9.4M5 we have to create an MPS data set from an MPS file first @@ -306,12 +387,11 @@ def actualSolve(self, lp): results="TEXT", ) - self._delete_tmp_files(tmpMps, tmpMst) + self.delete_tmp_files(tmpMps, tmpMst) - # Store log and ODS output - self._log = res["LOG"] + # Store SAS output if self.msg: - print(self._log) + print(res["LOG"]) self._macro = dict( (key.strip(), value.strip()) for key, value in ( @@ -322,39 +402,19 @@ def actualSolve(self, lp): primal_out = sas.sd2df(f"primalout{postfix}") dual_out = sas.sd2df(f"dualout{postfix}") - if self._macro.get("STATUS", "ERROR") != "OK": - raise PulpSolverError("PuLP: Error ({err_name}) \ + raise PulpSolverError( + "PuLP: Error ({err_name}) \ while trying to solve the instance: {name}".format( - err_name=self._macro.get("STATUS", "ERROR"), name=lp.name)) + err_name=self._macro.get("STATUS", "ERROR"), name=lp.name + ) + ) status = self._read_solution(lp, primal_out, dual_out) return status - def _read_solution(self, lp, primal_out, dual_out): - status = SOLSTATUS_TO_STATUS[self._macro.get("SOLUTION_STATUS", "ERROR")] - - if self.proc == "OPTLP": - # TODO: Check whether there is better implementation than zip(). - values = dict(zip(primal_out['_VAR_'], primal_out['_VALUE_'])) - rc = dict(zip(primal_out['_VAR_'], primal_out['_R_COST_'])) - lp.assignVarsVals(values) - lp.assignVarsDj(rc) - - prices = dict(zip(dual_out['_ROW_'], dual_out['_VALUE_'])) - slacks = dict(zip(dual_out['_ROW_'], dual_out['_ACTIVITY_'])) - lp.assignConsPi(prices) - lp.assignConsSlack(slacks, activity=True) - else: - # Convert primal out data set to variable dictionary - # Use pandas functions for efficiency - values = dict(zip(primal_out['_VAR_'], primal_out['_VALUE_'])) - lp.assignVarsVals(values) - lp.assignStatus(status) - return status - -class SASCAS(LpSolver): +class SASCAS(SASsolver): name = "SASCAS" try: @@ -367,112 +427,96 @@ def available(self): """True if SASCAS is available.""" return False + def sasAvailable(self): + return False + def actualSolve(self, lp, callback=None): """Solves a well-formulated lp problem.""" raise PulpSolverError("SASCAS : Not Available") else: + def __init__( self, mip=True, msg=True, + keepFiles=False, warmStart=False, - cas_options={}, + timeLimit=None, + cas=None, **solverParams, ): """ :param bool mip: if False, assume LP even if integer variables :param bool msg: if False, no log is shown - :param bool warmStart: if False, no warmstart or initial primal solution provided. + :param bool keepFiles: if False, mps and mst files will not be saved + :param bool warmStart: if False, no warmstart or initial primal solution provided + :param cas: CAS object. See swat.CAS + :param solverParams: SAS proc OPTMILP or OPTLP parameters """ - LpSolver.__init__( + SASsolver.__init__( self, mip=mip, msg=msg, + keepFiles=keepFiles, + warmStart=warmStart, + timeLimit=timeLimit, + **solverParams, ) - self._sas = swat - self.warmStart=warmStart - self.cas_options=cas_options - # Only named options are allowed in SAS solvers. - self.solverOptions = {key:value for key, value in solverParams.items()} + self.cas = cas + def available(self): - """True if SAS94 is available.""" return True + def sasAvailable(self): + if self.cas == None: + return False + try: + with redirect_stdout(SASLogWriter(self.msg)) as self._log_writer: + # Load the optimization action set + self.cas.loadactionset("optimization") + return True + except: + return False def toDict(self): data = dict(solver=self.name) - for k in ["mip", "msg", "warmStart", "cas_options"]: + for k in ["mip", "msg", "warmStart", "keepFiles"]: try: data[k] = getattr(self, k) except AttributeError: pass - data.update(self.solverOptions) - return data - - def _create_tmp_files(self, name, *args): - - return (f"{name}-pulp.{n}" for n in args) - - def _delete_tmp_files(self, *args): - for file in args: + for k in ["cas", "timeLimit"]: + # with these ones, we only export if it has some content: try: - os.remove(file) - except FileNotFoundError: + value = getattr(self, k) + if value: + data[k] = value + except AttributeError: pass - - def _write_sol(self, filename, vs): - """Writes a SAS solution file""" - values = [(v.name, v.value()) for v in vs if v.value() is not None] - with open(filename, "w") as f: - f.write('_VAR_,_VALUE_\n') - for name, value in values: - f.write(f"{name},{value}\n") - return True - - def _read_solution(self, lp, primal_out, dual_out): - status = SOLSTATUS_TO_STATUS[self._macro.get("SOLUTION_STATUS", "ERROR")] - - if self.proc == "OPTLP": - # TODO: Check whether there is better implementation than zip(). - values = dict(zip(primal_out['_VAR_'], primal_out['_VALUE_'])) - rc = dict(zip(primal_out['_VAR_'], primal_out['_R_COST_'])) - lp.assignVarsVals(values) - lp.assignVarsDj(rc) - - prices = dict(zip(dual_out['_ROW_'], dual_out['_VALUE_'])) - slacks = dict(zip(dual_out['_ROW_'], dual_out['_ACTIVITY_'])) - lp.assignConsPi(prices) - lp.assignConsSlack(slacks, activity=True) - else: - # Convert primal out data set to variable dictionary - # Use pandas functions for efficiency - values = dict(zip(primal_out['_VAR_'], primal_out['_VALUE_'])) - lp.assignVarsVals(values) - lp.assignStatus(status) - return status - - # TODO: Improve it without reading the mps file. - def _get_max_upload_len(self, fileName): - maxLen = 0 - with open(fileName, 'r') as f: - for line in f.readlines(): - maxLen = max(maxLen, max([len(word) for word in line.split(" ")])) - return maxLen + 1 + data.update(self.solverOptions) + return data def actualSolve(self, lp): """Solve a well formulated lp problem""" log.debug("Running SAS") - if (self.cas_options == {}): - raise PulpSolverError("""SASCAS: Provide cas_options with - {port: , host: , authinfo: } - or {port: , host: , username: , password: }.""") + if not self.sasAvailable(): + raise PulpSolverError( + """SASCAS: Provide a valid CAS session by parameter cas=.""" + ) - if len(lp.sos1) or len(lp.sos2): - raise PulpSolverError("SASCAS: Currently SAS doesn't support SOS1 and SOS2.") + # if (self.cas_options == {}): + # raise PulpSolverError("""SASCAS: Provide cas_options with + # {port: , host: , authinfo: } + # or {port: , host: , username: , password: }.""") + if len(lp.sos1) or len(lp.sos2): + raise PulpSolverError( + "SASCAS: Currently SAS doesn't support SOS1 and SOS2." + ) + s = self.cas proc = self.proc = "OPTMILP" if self.mip else "OPTLP" # Get Obj Sense if lp.sense == constants.LpMaximize: @@ -482,61 +526,37 @@ def actualSolve(self, lp): else: raise PulpSolverError("SASCAS : Objective sense should be min or max.") + # Get timeLimit. SAS solvers use MAXTIME instead of timeLimit as a parameter. + if self.timeLimit: + self.solverOptions["MAXTIME"] = self.timeLimit + status = None with redirect_stdout(SASLogWriter(self.msg)) as self._log_writer: - s = self._sas.CAS(**self.cas_options) + # Used for naming the data structure in SAS. postfix = uuid4().hex[:16] - tmpMps, tmpMpsCsv, tmpMstCsv = self._create_tmp_files(postfix, "mps", "mps.csv", "mst.csv") + tmpMps, tmpMpsCsv, tmpMstCsv = self.create_tmp_files( + lp.name, "mps", "mps.csv", "mst.csv" + ) vs = lp.writeMPS(tmpMps, with_objsense=False) nameLen = self._get_max_upload_len(tmpMps) if nameLen > MAX_NAME_LENGTH: - raise PulpSolverError(f"SASCAS: The lengths of the variable or constraint names \ - (including indices) should not exceed {MAX_NAME_LENGTH}.") + raise PulpSolverError( + f"SASCAS: The lengths of the variable or constraint names \ + (including indices) should not exceed {MAX_NAME_LENGTH}." + ) try: - # Load the optimization action set - s.loadactionset('optimization') - if lp.isMIP(): - if not self.mip: - warnings.warn("SASCAS will solve the relaxed problem of the MILP instance.") - - if os.stat(tmpMps).st_size >= 2 * 1024 **3: - # if False: - # For large files, use convertMPS, first create file for upload - with open(tmpMpsCsv, 'w') as mpsWithId: - mpsWithId.write('_ID_\tText\n') - with open(tmpMps, 'r') as f: - id = 0 - for line in f: - id += 1 - mpsWithId.write(str(id) + '\t' + line.rstrip() + '\n') - - # Upload .mps.csv file - s.upload_file( - tmpMpsCsv, - casout={"name": f"mpscsv{postfix}", "replace": True}, - importoptions={"filetype": "CSV", "delimiter": "\t"}, + # # Load the optimization action set + # s.loadactionset('optimization') + if lp.isMIP() and not self.mip: + warnings.warn( + "SASCAS will solve the relaxed problem of the MILP instance." ) + # load_mps + self._load_mps(s, tmpMps, tmpMpsCsv, postfix, nameLen) - # Convert .mps.csv file to .mps - s.optimization.convertMps( - data=f"mpscsv{postfix}", - casOut={"name": f"mpsdata{postfix}", "replace": True}, - format="FREE", - maxLength=min(nameLen, MAX_NAME_LENGTH), - ) - else: - # For small files, use loadMPS - with open(tmpMps, 'r') as mps_file: - s.optimization.loadMps( - mpsFileString=mps_file.read(), - casout={"name": f"mpsdata{postfix}", "replace": True}, - format="FREE", - maxLength=min(nameLen, MAX_NAME_LENGTH), - ) - - if self.warmStart and (proc == "OPTMILP"): + if self.optionsDict.get("warmStart", False) and (proc == "OPTMILP"): self._write_sol(filename=tmpMstCsv, vs=vs) # Upload warmstart file to CAS s.upload_file( @@ -546,58 +566,95 @@ def actualSolve(self, lp): ) self.solverOptions["primalin"] = f"primalin{postfix}" # Delete the temp files. - self._delete_tmp_files(tmpMps, tmpMstCsv, tmpMpsCsv) + self.delete_tmp_files(tmpMps, tmpMstCsv, tmpMpsCsv) # Solve the problem in CAS if proc == "OPTMILP": r = s.optimization.solveMilp( data={"name": f"mpsdata{postfix}"}, primalOut={"name": f"primalout{postfix}", "replace": True}, - **self.solverOptions + **self.solverOptions, ) else: r = s.optimization.solveLp( data={"name": f"mpsdata{postfix}"}, primalOut={"name": f"primalout{postfix}", "replace": True}, dualOut={"name": f"dualout{postfix}", "replace": True}, - **self.solverOptions + **self.solverOptions, ) if r: - self._macro = { - "STATUS": r.get("status", "ERROR").upper(), - "SOLUTION_STATUS": r.get("solutionStatus", "ERROR").upper() - } - if self._macro.get("STATUS", "ERROR") != "OK": - raise PulpSolverError("PuLP: Error ({err_name}) while trying to solve the instance: {name}".format( - err_name=self._macro.get("STATUS", "ERROR"), name=lp.name)) - # If we get solution successfully. - if proc == "OPTMILP": - primal_out = s.CASTable(name=f"primalout{postfix}") - primal_out = primal_out[ - ['_VAR_', '_VALUE_', '_STATUS_', '_R_COST_'] - ] - dual_out = None - else: - primal_out = s.CASTable(name=f"primalout{postfix}") - primal_out = primal_out[ - ['_VAR_', '_VALUE_', '_STATUS_', '_R_COST_'] - ] - dual_out = s.CASTable(name=f"dualout{postfix}") - dual_out = dual_out[ - ['_ROW_', '_VALUE_', '_STATUS_', '_ACTIVITY_'] - ] + primal_out, dual_out = self._get_output(lp, s, r, proc, postfix) status = self._read_solution(lp, primal_out, dual_out) finally: - self._delete_tmp_files(tmpMps, tmpMstCsv, tmpMpsCsv) - s.close() + self.delete_tmp_files(tmpMps, tmpMstCsv, tmpMpsCsv) if self.msg: print(self._log_writer.log()) if status: return status else: - raise PulpSolverError(f"PuLP: Error while trying to solve the instance: \ - {lp.name} via SASCAS.") + raise PulpSolverError( + f"PuLP: Error while trying to solve the instance: \ + {lp.name} via SASCAS." + ) + + def _get_output(self, lp, s, r, proc, postfix): + self._macro = { + "STATUS": r.get("status", "ERROR").upper(), + "SOLUTION_STATUS": r.get("solutionStatus", "ERROR").upper(), + } + if self._macro.get("STATUS", "ERROR") != "OK": + raise PulpSolverError( + "PuLP: Error ({err_name}) while trying to solve the instance: {name}".format( + err_name=self._macro.get("STATUS", "ERROR"), name=lp.name + ) + ) + # If we get solution successfully. + if proc == "OPTMILP": + primal_out = s.CASTable(name=f"primalout{postfix}") + primal_out = primal_out[["_VAR_", "_VALUE_", "_STATUS_", "_R_COST_"]] + dual_out = None + else: + primal_out = s.CASTable(name=f"primalout{postfix}") + primal_out = primal_out[["_VAR_", "_VALUE_", "_STATUS_", "_R_COST_"]] + dual_out = s.CASTable(name=f"dualout{postfix}") + dual_out = dual_out[["_ROW_", "_VALUE_", "_STATUS_", "_ACTIVITY_"]] + return primal_out, dual_out + + def _load_mps(self, s, tmpMps, tmpMpsCsv, postfix, nameLen): + if os.stat(tmpMps).st_size >= 2 * 1024**3: + # For large files, use convertMPS, first create file for upload + with open(tmpMpsCsv, "w") as mpsWithId: + mpsWithId.write("_ID_\tText\n") + with open(tmpMps, "r") as f: + id = 0 + for line in f: + id += 1 + mpsWithId.write(str(id) + "\t" + line.rstrip() + "\n") + + # Upload .mps.csv file + s.upload_file( + tmpMpsCsv, + casout={"name": f"mpscsv{postfix}", "replace": True}, + importoptions={"filetype": "CSV", "delimiter": "\t"}, + ) + + # Convert .mps.csv file to .mps + s.optimization.convertMps( + data=f"mpscsv{postfix}", + casOut={"name": f"mpsdata{postfix}", "replace": True}, + format="FREE", + maxLength=min(nameLen, MAX_NAME_LENGTH), + ) + else: + # For small files, use loadMPS + with open(tmpMps, "r") as mps_file: + s.optimization.loadMps( + mpsFileString=mps_file.read(), + casout={"name": f"mpsdata{postfix}", "replace": True}, + format="FREE", + maxLength=min(nameLen, MAX_NAME_LENGTH), + ) class SASLogWriter: diff --git a/pulp/tests/test_pulp.py b/pulp/tests/test_pulp.py index d36fcae5..2156285a 100644 --- a/pulp/tests/test_pulp.py +++ b/pulp/tests/test_pulp.py @@ -1,6 +1,7 @@ """ Tests for pulp """ + import os import tempfile @@ -78,6 +79,11 @@ def setUp(self): self.solver = self.solveInst(msg=False) if not self.solver.available(): self.skipTest(f"solver {self.solveInst} not available") + elif ( + self.solver.name in ["SASCAS", "SAS94"] + and not self.solver.sasAvailable() + ): + self.skipTest(f"solver {self.solveInst} valid solver not provided") def tearDown(self): for ext in ["mst", "log", "lp", "mps", "sol"]: @@ -236,6 +242,8 @@ def test_pulp_013(self): HiGHS_CMD, XPRESS, XPRESS_CMD, + SAS94, + SASCAS, ]: try: pulpTestCheck( @@ -287,6 +295,8 @@ def test_pulp_014(self): XPRESS, XPRESS_CMD, XPRESS_PY, + SAS94, + SASCAS, ]: try: pulpTestCheck( @@ -905,6 +915,9 @@ def test_export_dict_LP_no_obj(self): ) def test_export_json_LP(self): + if self.solver.name in ["SAS94", "SASCAS"]: + # SASCAS does not support this functionality + return name = self._testMethodName prob = LpProblem(name, const.LpMinimize) x = LpVariable("x", 0, 4) @@ -989,6 +1002,9 @@ def test_export_solver_dict_LP(self): ) def test_export_solver_json(self): + if self.solver.name in ["SAS94", "SASCAS"]: + # SASCAS does not support this functionality + return name = self._testMethodName prob = LpProblem(name, const.LpMinimize) x = LpVariable("x", 0, 4) @@ -1598,6 +1614,14 @@ class COPTTest(BaseSolverTest.PuLPTest): solveInst = COPT +class SAS94Test(BaseSolverTest.PuLPTest): + solveInst = SAS94 + + +class SASCASTest(BaseSolverTest.PuLPTest): + solveInst = SASCAS + + def pulpTestCheck( prob, solver, From 83218cc45d534eb48d262ffb6bfc5928f02b19e9 Mon Sep 17 00:00:00 2001 From: Bochuan Lyu Date: Fri, 8 Mar 2024 14:52:26 -0500 Subject: [PATCH 06/11] Update documentation --- pulp/apis/sas_api.py | 23 ----------------------- pulp/tests/test_pulp.py | 4 ++-- 2 files changed, 2 insertions(+), 25 deletions(-) diff --git a/pulp/apis/sas_api.py b/pulp/apis/sas_api.py index 892ca091..401af2a6 100755 --- a/pulp/apis/sas_api.py +++ b/pulp/apis/sas_api.py @@ -68,22 +68,6 @@ "ERROR": constants.LpStatusNotSolved, } -CAS_OPTION_NAMES = [ - "hostname", - "port", - "username", - "password", - "session", - "locale", - "name", - "nworkers", - "authinfo", - "protocol", - "path", - "ssl_ca_list", - "authcode", -] - class SASsolver(LpSolver_CMD): name = "SASsolver" @@ -506,11 +490,6 @@ def actualSolve(self, lp): """SASCAS: Provide a valid CAS session by parameter cas=.""" ) - # if (self.cas_options == {}): - # raise PulpSolverError("""SASCAS: Provide cas_options with - # {port: , host: , authinfo: } - # or {port: , host: , username: , password: }.""") - if len(lp.sos1) or len(lp.sos2): raise PulpSolverError( "SASCAS: Currently SAS doesn't support SOS1 and SOS2." @@ -547,8 +526,6 @@ def actualSolve(self, lp): (including indices) should not exceed {MAX_NAME_LENGTH}." ) try: - # # Load the optimization action set - # s.loadactionset('optimization') if lp.isMIP() and not self.mip: warnings.warn( "SASCAS will solve the relaxed problem of the MILP instance." diff --git a/pulp/tests/test_pulp.py b/pulp/tests/test_pulp.py index 2156285a..1fc018be 100644 --- a/pulp/tests/test_pulp.py +++ b/pulp/tests/test_pulp.py @@ -916,7 +916,7 @@ def test_export_dict_LP_no_obj(self): def test_export_json_LP(self): if self.solver.name in ["SAS94", "SASCAS"]: - # SASCAS does not support this functionality + # SAS API does not support this functionality return name = self._testMethodName prob = LpProblem(name, const.LpMinimize) @@ -1003,7 +1003,7 @@ def test_export_solver_dict_LP(self): def test_export_solver_json(self): if self.solver.name in ["SAS94", "SASCAS"]: - # SASCAS does not support this functionality + # SAS API does not support this functionality return name = self._testMethodName prob = LpProblem(name, const.LpMinimize) From c597f1d78ded2515fa01e69a46bf28f8e14f8951 Mon Sep 17 00:00:00 2001 From: Bochuan Lyu Date: Wed, 10 Apr 2024 12:44:55 -0400 Subject: [PATCH 07/11] Update SAS API --- pulp/apis/sas_api.py | 123 ++++++++++++++++++------------------------- 1 file changed, 52 insertions(+), 71 deletions(-) diff --git a/pulp/apis/sas_api.py b/pulp/apis/sas_api.py index 401af2a6..f0a3eb85 100755 --- a/pulp/apis/sas_api.py +++ b/pulp/apis/sas_api.py @@ -89,15 +89,18 @@ def __init__( :param solverParams: SAS proc OPTMILP or OPTLP parameters """ LpSolver_CMD.__init__( - self, mip=mip, msg=msg, warmStart=warmStart, keepFiles=keepFiles + self, + mip=mip, + msg=msg, + keepFiles=keepFiles, + timeLimit=timeLimit, + warmStart=warmStart, + **solverParams, ) - self.timeLimit = timeLimit - # Only named options are allowed in SAS solvers. - self.solverOptions = {key: value for key, value in solverParams.items()} def _create_statement_str(self, statement): """Helper function to create the strings for the statements of the proc OPTLP/OPTMILP code.""" - stmt = self.solverOptions.pop(statement, None) + stmt = self.optionsDict.get(statement, None) if stmt: return ( statement.strip() @@ -127,10 +130,10 @@ def _get_max_upload_len(self, fileName): maxLen = max(maxLen, max([len(word) for word in line.split(" ")])) return maxLen + 1 - def _read_solution(self, lp, primal_out, dual_out): + def _read_solution(self, lp, primal_out, dual_out, proc): status = SOLSTATUS_TO_STATUS[self._macro.get("SOLUTION_STATUS", "ERROR")] - if self.proc == "OPTLP": + if proc == "OPTLP": # TODO: Check whether there is better implementation than zip(). values = dict(zip(primal_out["_VAR_"], primal_out["_VALUE_"])) rc = dict(zip(primal_out["_VAR_"], primal_out["_R_COST_"])) @@ -198,42 +201,24 @@ def __init__( keepFiles=keepFiles, warmStart=warmStart, timeLimit=timeLimit, + sas=sas, **solverParams, ) - self.sas = sas - def available(self): """True if SAS94 is available.""" return True def sasAvailable(self): - if not self.sas: - return False try: - self.sas.sasver + if self.optionsDict.get("sas", None) is None: + return False + sas = self.optionsDict.get("sas", None) + sas.sasver return True except: return False - def toDict(self): - data = dict(solver=self.name) - for k in ["mip", "msg", "keepFiles", "warmStart"]: - try: - data[k] = getattr(self, k) - except AttributeError: - pass - for k in ["sas", "timeLimit"]: - # with these ones, we only export if it has some content: - try: - value = getattr(self, k) - if value: - data[k] = value - except AttributeError: - pass - data.update(self.solverOptions) - return data - def actualSolve(self, lp): """Solve a well formulated lp problem""" log.debug("Running SAS") @@ -258,19 +243,27 @@ def actualSolve(self, lp): (including indices) should not exceed {MAX_NAME_LENGTH}." ) - proc = self.proc = "OPTMILP" if self.mip else "OPTLP" + proc = "OPTMILP" if self.mip else "OPTLP" + + solverOptions = { + key: self.optionsDict[key] + for key in self.optionsDict.keys() + if key not in ["warmStart", "sas", "cas"] + } + warmStart = self.optionsDict.get("warmStart") + sas = self.optionsDict.get("sas") # Get Obj Sense if lp.sense == constants.LpMaximize: - self.solverOptions["objsense"] = "max" + solverOptions["objsense"] = "max" elif lp.sense == constants.LpMinimize: - self.solverOptions["objsense"] = "min" + solverOptions["objsense"] = "min" else: raise PulpSolverError("SAS94 : Objective sense should be min or max.") # Get timeLimit. SAS solvers use MAXTIME instead of timeLimit as a parameter. if self.timeLimit: - self.solverOptions["MAXTIME"] = self.timeLimit + solverOptions["MAXTIME"] = self.timeLimit # Get the rootnode options decomp_str = self._create_statement_str("decomp") decompmaster_str = self._create_statement_str("decompmaster") @@ -284,7 +277,7 @@ def actualSolve(self, lp): ) # Handle warmstart warmstart_str = "" - if self.optionsDict.get("warmStart", False): + if warmStart: self._write_sol(filename=tmpMst, vs=vs) # Set the warmstart basis option @@ -305,10 +298,8 @@ def actualSolve(self, lp): # Convert options to string opt_str = " ".join( - option + "=" + str(value) - for option, value in self.solverOptions.items() + option + "=" + str(value) for option, value in solverOptions.items() ) - sas = self.sas # Find the version of 9.4 we are using if sas.sasver.startswith("9.04.01M5"): @@ -393,7 +384,7 @@ def actualSolve(self, lp): err_name=self._macro.get("STATUS", "ERROR"), name=lp.name ) ) - status = self._read_solution(lp, primal_out, dual_out) + status = self._read_solution(lp, primal_out, dual_out, proc) return status @@ -445,42 +436,25 @@ def __init__( keepFiles=keepFiles, warmStart=warmStart, timeLimit=timeLimit, + cas=cas, **solverParams, ) - self.cas = cas def available(self): return True def sasAvailable(self): - if self.cas == None: - return False try: + if self.optionsDict.get("cas", None) is None: + return False with redirect_stdout(SASLogWriter(self.msg)) as self._log_writer: # Load the optimization action set - self.cas.loadactionset("optimization") + cas = self.optionsDict.get("cas", None) + cas.loadactionset("optimization") return True except: return False - def toDict(self): - data = dict(solver=self.name) - for k in ["mip", "msg", "warmStart", "keepFiles"]: - try: - data[k] = getattr(self, k) - except AttributeError: - pass - for k in ["cas", "timeLimit"]: - # with these ones, we only export if it has some content: - try: - value = getattr(self, k) - if value: - data[k] = value - except AttributeError: - pass - data.update(self.solverOptions) - return data - def actualSolve(self, lp): """Solve a well formulated lp problem""" log.debug("Running SAS") @@ -495,19 +469,26 @@ def actualSolve(self, lp): "SASCAS: Currently SAS doesn't support SOS1 and SOS2." ) - s = self.cas - proc = self.proc = "OPTMILP" if self.mip else "OPTLP" + solverOptions = { + key: self.optionsDict[key] + for key in self.optionsDict.keys() + if key not in ["warmStart", "sas", "cas"] + } + warmStart = self.optionsDict.get("warmStart") + s = self.optionsDict.get("cas") + + proc = "OPTMILP" if self.mip else "OPTLP" # Get Obj Sense if lp.sense == constants.LpMaximize: - self.solverOptions["objsense"] = "max" + solverOptions["objsense"] = "max" elif lp.sense == constants.LpMinimize: - self.solverOptions["objsense"] = "min" + solverOptions["objsense"] = "min" else: raise PulpSolverError("SASCAS : Objective sense should be min or max.") # Get timeLimit. SAS solvers use MAXTIME instead of timeLimit as a parameter. if self.timeLimit: - self.solverOptions["MAXTIME"] = self.timeLimit + solverOptions["MAXTIME"] = self.timeLimit status = None with redirect_stdout(SASLogWriter(self.msg)) as self._log_writer: @@ -533,7 +514,7 @@ def actualSolve(self, lp): # load_mps self._load_mps(s, tmpMps, tmpMpsCsv, postfix, nameLen) - if self.optionsDict.get("warmStart", False) and (proc == "OPTMILP"): + if warmStart and (proc == "OPTMILP"): self._write_sol(filename=tmpMstCsv, vs=vs) # Upload warmstart file to CAS s.upload_file( @@ -541,7 +522,7 @@ def actualSolve(self, lp): casout={"name": f"primalin{postfix}", "replace": True}, importoptions={"filetype": "CSV"}, ) - self.solverOptions["primalin"] = f"primalin{postfix}" + solverOptions["primalin"] = f"primalin{postfix}" # Delete the temp files. self.delete_tmp_files(tmpMps, tmpMstCsv, tmpMpsCsv) @@ -550,18 +531,18 @@ def actualSolve(self, lp): r = s.optimization.solveMilp( data={"name": f"mpsdata{postfix}"}, primalOut={"name": f"primalout{postfix}", "replace": True}, - **self.solverOptions, + **solverOptions, ) else: r = s.optimization.solveLp( data={"name": f"mpsdata{postfix}"}, primalOut={"name": f"primalout{postfix}", "replace": True}, dualOut={"name": f"dualout{postfix}", "replace": True}, - **self.solverOptions, + **solverOptions, ) if r: primal_out, dual_out = self._get_output(lp, s, r, proc, postfix) - status = self._read_solution(lp, primal_out, dual_out) + status = self._read_solution(lp, primal_out, dual_out, proc) finally: self.delete_tmp_files(tmpMps, tmpMstCsv, tmpMpsCsv) From a0287434913fa4be643169d5a5c4fcce3fc0affe Mon Sep 17 00:00:00 2001 From: Bochuan Lyu Date: Thu, 20 Jun 2024 10:17:52 -0400 Subject: [PATCH 08/11] Update SAS API interface (need more changes) --- pulp/apis/sas_api.py | 162 +++++++++++++++++++++------------------- pulp/tests/test_pulp.py | 6 -- 2 files changed, 86 insertions(+), 82 deletions(-) diff --git a/pulp/apis/sas_api.py b/pulp/apis/sas_api.py index f0a3eb85..1ca11ce6 100755 --- a/pulp/apis/sas_api.py +++ b/pulp/apis/sas_api.py @@ -183,7 +183,6 @@ def __init__( keepFiles=False, warmStart=False, timeLimit=None, - sas=None, **solverParams, ): """ @@ -191,7 +190,6 @@ def __init__( :param bool msg: if False, no log is shown :param bool keepFiles: if False, mps and mst files will not be saved :param bool warmStart: if False, no warmstart or initial primal solution provided - :param object sas: sas session. It must be provided by the user. :param solverParams: SAS proc OPTMILP or OPTLP parameters """ SASsolver.__init__( @@ -201,9 +199,14 @@ def __init__( keepFiles=keepFiles, warmStart=warmStart, timeLimit=timeLimit, - sas=sas, **solverParams, ) + self.sas = None + + + def __del__(self): + if self.sas: + self.sas.endsas() def available(self): """True if SAS94 is available.""" @@ -211,10 +214,9 @@ def available(self): def sasAvailable(self): try: - if self.optionsDict.get("sas", None) is None: - return False - sas = self.optionsDict.get("sas", None) - sas.sasver + if not self.sas: + self.sas = saspy.SASsession() + self.sas.sasver return True except: return False @@ -222,10 +224,12 @@ def sasAvailable(self): def actualSolve(self, lp): """Solve a well formulated lp problem""" log.debug("Running SAS") + if not self.sasAvailable(): raise PulpSolverError( - "SAS94: Provide a valid SAS session by parameter sas=." + "SAS94: Cannot connect to a SAS session." ) + sas = self.sas if len(lp.sos1) or len(lp.sos2): raise PulpSolverError( "SAS94: Currently SAS doesn't support SOS1 and SOS2." @@ -245,13 +249,15 @@ def actualSolve(self, lp): proc = "OPTMILP" if self.mip else "OPTLP" + optionList = ["warmStart", "decomp", "decompmaster", + "decompsubprob", "rootnode"] + solverOptions = { key: self.optionsDict[key] for key in self.optionsDict.keys() - if key not in ["warmStart", "sas", "cas"] + if key not in optionList } - warmStart = self.optionsDict.get("warmStart") - sas = self.optionsDict.get("sas") + warmStart = self.optionsDict.get("warmStart", None) # Get Obj Sense if lp.sense == constants.LpMaximize: @@ -418,7 +424,7 @@ def __init__( keepFiles=False, warmStart=False, timeLimit=None, - cas=None, + casOptions={}, **solverParams, ): """ @@ -426,7 +432,7 @@ def __init__( :param bool msg: if False, no log is shown :param bool keepFiles: if False, mps and mst files will not be saved :param bool warmStart: if False, no warmstart or initial primal solution provided - :param cas: CAS object. See swat.CAS + :param dict casOptions: options for cas connection. :param solverParams: SAS proc OPTMILP or OPTLP parameters """ SASsolver.__init__( @@ -436,21 +442,27 @@ def __init__( keepFiles=keepFiles, warmStart=warmStart, timeLimit=timeLimit, - cas=cas, + casOptions=casOptions, **solverParams, ) + self.cas = None + + def __del__(self): + if self.cas: + self.cas.close() def available(self): return True def sasAvailable(self): try: - if self.optionsDict.get("cas", None) is None: - return False + if not self.cas: + casOptions = self.optionsDict.get("casOptions", None) + self.cas = swat.CAS(**casOptions) + with redirect_stdout(SASLogWriter(self.msg)) as self._log_writer: # Load the optimization action set - cas = self.optionsDict.get("cas", None) - cas.loadactionset("optimization") + self.cas.loadactionset("optimization") return True except: return False @@ -461,9 +473,9 @@ def actualSolve(self, lp): if not self.sasAvailable(): raise PulpSolverError( - """SASCAS: Provide a valid CAS session by parameter cas=.""" + """SASCAS: Cannot connect to a CAS session.""" ) - + s = self.cas if len(lp.sos1) or len(lp.sos2): raise PulpSolverError( "SASCAS: Currently SAS doesn't support SOS1 and SOS2." @@ -472,10 +484,10 @@ def actualSolve(self, lp): solverOptions = { key: self.optionsDict[key] for key in self.optionsDict.keys() - if key not in ["warmStart", "sas", "cas"] + if key not in ["warmStart", "casOptions"] } warmStart = self.optionsDict.get("warmStart") - s = self.optionsDict.get("cas") + proc = "OPTMILP" if self.mip else "OPTLP" # Get Obj Sense @@ -546,8 +558,6 @@ def actualSolve(self, lp): finally: self.delete_tmp_files(tmpMps, tmpMstCsv, tmpMpsCsv) - if self.msg: - print(self._log_writer.log()) if status: return status else: @@ -556,63 +566,63 @@ def actualSolve(self, lp): {lp.name} via SASCAS." ) - def _get_output(self, lp, s, r, proc, postfix): - self._macro = { - "STATUS": r.get("status", "ERROR").upper(), - "SOLUTION_STATUS": r.get("solutionStatus", "ERROR").upper(), - } - if self._macro.get("STATUS", "ERROR") != "OK": - raise PulpSolverError( - "PuLP: Error ({err_name}) while trying to solve the instance: {name}".format( - err_name=self._macro.get("STATUS", "ERROR"), name=lp.name + def _get_output(self, lp, s, r, proc, postfix): + self._macro = { + "STATUS": r.get("status", "ERROR").upper(), + "SOLUTION_STATUS": r.get("solutionStatus", "ERROR").upper(), + } + if self._macro.get("STATUS", "ERROR") != "OK": + raise PulpSolverError( + "PuLP: Error ({err_name}) while trying to solve the instance: {name}".format( + err_name=self._macro.get("STATUS", "ERROR"), name=lp.name + ) + ) + # If we get solution successfully. + if proc == "OPTMILP": + primal_out = s.CASTable(name=f"primalout{postfix}") + primal_out = primal_out[["_VAR_", "_VALUE_", "_STATUS_", "_R_COST_"]] + dual_out = None + else: + primal_out = s.CASTable(name=f"primalout{postfix}") + primal_out = primal_out[["_VAR_", "_VALUE_", "_STATUS_", "_R_COST_"]] + dual_out = s.CASTable(name=f"dualout{postfix}") + dual_out = dual_out[["_ROW_", "_VALUE_", "_STATUS_", "_ACTIVITY_"]] + return primal_out, dual_out + + def _load_mps(self, s, tmpMps, tmpMpsCsv, postfix, nameLen): + if os.stat(tmpMps).st_size >= 2 * 1024**3: + # For large files, use convertMPS, first create file for upload + with open(tmpMpsCsv, "w") as mpsWithId: + mpsWithId.write("_ID_\tText\n") + with open(tmpMps, "r") as f: + id = 0 + for line in f: + id += 1 + mpsWithId.write(str(id) + "\t" + line.rstrip() + "\n") + + # Upload .mps.csv file + s.upload_file( + tmpMpsCsv, + casout={"name": f"mpscsv{postfix}", "replace": True}, + importoptions={"filetype": "CSV", "delimiter": "\t"}, ) - ) - # If we get solution successfully. - if proc == "OPTMILP": - primal_out = s.CASTable(name=f"primalout{postfix}") - primal_out = primal_out[["_VAR_", "_VALUE_", "_STATUS_", "_R_COST_"]] - dual_out = None - else: - primal_out = s.CASTable(name=f"primalout{postfix}") - primal_out = primal_out[["_VAR_", "_VALUE_", "_STATUS_", "_R_COST_"]] - dual_out = s.CASTable(name=f"dualout{postfix}") - dual_out = dual_out[["_ROW_", "_VALUE_", "_STATUS_", "_ACTIVITY_"]] - return primal_out, dual_out - - def _load_mps(self, s, tmpMps, tmpMpsCsv, postfix, nameLen): - if os.stat(tmpMps).st_size >= 2 * 1024**3: - # For large files, use convertMPS, first create file for upload - with open(tmpMpsCsv, "w") as mpsWithId: - mpsWithId.write("_ID_\tText\n") - with open(tmpMps, "r") as f: - id = 0 - for line in f: - id += 1 - mpsWithId.write(str(id) + "\t" + line.rstrip() + "\n") - - # Upload .mps.csv file - s.upload_file( - tmpMpsCsv, - casout={"name": f"mpscsv{postfix}", "replace": True}, - importoptions={"filetype": "CSV", "delimiter": "\t"}, - ) - # Convert .mps.csv file to .mps - s.optimization.convertMps( - data=f"mpscsv{postfix}", - casOut={"name": f"mpsdata{postfix}", "replace": True}, - format="FREE", - maxLength=min(nameLen, MAX_NAME_LENGTH), - ) - else: - # For small files, use loadMPS - with open(tmpMps, "r") as mps_file: - s.optimization.loadMps( - mpsFileString=mps_file.read(), - casout={"name": f"mpsdata{postfix}", "replace": True}, + # Convert .mps.csv file to .mps + s.optimization.convertMps( + data=f"mpscsv{postfix}", + casOut={"name": f"mpsdata{postfix}", "replace": True}, format="FREE", maxLength=min(nameLen, MAX_NAME_LENGTH), ) + else: + # For small files, use loadMPS + with open(tmpMps, "r") as mps_file: + s.optimization.loadMps( + mpsFileString=mps_file.read(), + casout={"name": f"mpsdata{postfix}", "replace": True}, + format="FREE", + maxLength=min(nameLen, MAX_NAME_LENGTH), + ) class SASLogWriter: diff --git a/pulp/tests/test_pulp.py b/pulp/tests/test_pulp.py index 89e1a097..5af77060 100644 --- a/pulp/tests/test_pulp.py +++ b/pulp/tests/test_pulp.py @@ -899,9 +899,6 @@ def test_export_dict_LP_no_obj(self): ) def test_export_json_LP(self): - if self.solver.name in ["SAS94", "SASCAS"]: - # SAS API does not support this functionality - return name = self._testMethodName prob = LpProblem(self._testMethodName, const.LpMinimize) x = LpVariable("x", 0, 4) @@ -986,9 +983,6 @@ def test_export_solver_dict_LP(self): ) def test_export_solver_json(self): - if self.solver.name in ["SAS94", "SASCAS"]: - # SAS API does not support this functionality - return name = self._testMethodName prob = LpProblem(name, const.LpMinimize) x = LpVariable("x", 0, 4) From 087b8ef1b97a2fa6b50326683ca8f1ad910f4938 Mon Sep 17 00:00:00 2001 From: phchri Date: Mon, 16 Sep 2024 11:51:39 +0200 Subject: [PATCH 09/11] Various improvements to the SAS interfaces --- pulp/apis/sas_api.py | 217 +++++++++++++++++++++++++++++----------- pulp/tests/test_pulp.py | 26 ++++- 2 files changed, 182 insertions(+), 61 deletions(-) diff --git a/pulp/apis/sas_api.py b/pulp/apis/sas_api.py index 1ca11ce6..39c035bd 100755 --- a/pulp/apis/sas_api.py +++ b/pulp/apis/sas_api.py @@ -68,6 +68,25 @@ "ERROR": constants.LpStatusNotSolved, } +SASPY_OPTIONS = ["cfgname", "cfgfile"] + +SWAT_OPTIONS = [ + "hostname", + "port", + "username", + "password", + "session", + "locale", + "name", + "nworkers", + "authinfo", + "protocol", + "path", + "ssl_ca_list", + "authcode", + "pkce", +] + class SASsolver(LpSolver_CMD): name = "SASsolver" @@ -132,23 +151,22 @@ def _get_max_upload_len(self, fileName): def _read_solution(self, lp, primal_out, dual_out, proc): status = SOLSTATUS_TO_STATUS[self._macro.get("SOLUTION_STATUS", "ERROR")] + primal_out = primal_out.set_index("_VAR_", drop=True) + values = primal_out["_VALUE_"].to_dict() + lp.assignVarsVals(values) if proc == "OPTLP": - # TODO: Check whether there is better implementation than zip(). - values = dict(zip(primal_out["_VAR_"], primal_out["_VALUE_"])) - rc = dict(zip(primal_out["_VAR_"], primal_out["_R_COST_"])) - lp.assignVarsVals(values) + rc = primal_out["_R_COST_"].to_dict() lp.assignVarsDj(rc) - prices = dict(zip(dual_out["_ROW_"], dual_out["_VALUE_"])) - slacks = dict(zip(dual_out["_ROW_"], dual_out["_ACTIVITY_"])) + dual_out = dual_out.set_index("_ROW_", drop=True) + + prices = dual_out["_VALUE_"].to_dict() lp.assignConsPi(prices) + + slacks = dual_out["_ACTIVITY_"].to_dict() lp.assignConsSlack(slacks, activity=True) - else: - # Convert primal out data set to variable dictionary - # Use pandas functions for efficiency - values = dict(zip(primal_out["_VAR_"], primal_out["_VALUE_"])) - lp.assignVarsVals(values) + lp.assignStatus(status) return status @@ -192,6 +210,13 @@ def __init__( :param bool warmStart: if False, no warmstart or initial primal solution provided :param solverParams: SAS proc OPTMILP or OPTLP parameters """ + # Extract saspy connection options + self._saspy_options = {} + for option in SASPY_OPTIONS: + value = solverParams.pop(option, None) + if value: + self._saspy_options[option] = value + SASsolver.__init__( self, mip=mip, @@ -201,8 +226,15 @@ def __init__( timeLimit=timeLimit, **solverParams, ) - self.sas = None + # Connect to saspy + self.sas = None + try: + self.sas = saspy.SASsession(**self._saspy_options) + except: + raise PulpSolverError( + "SAS94: Cannot connect to a SAS session. Try using using the cfgfile option." + ) def __del__(self): if self.sas: @@ -213,12 +245,9 @@ def available(self): return True def sasAvailable(self): - try: - if not self.sas: - self.sas = saspy.SASsession() - self.sas.sasver + if self.sas: return True - except: + else: return False def actualSolve(self, lp): @@ -226,9 +255,7 @@ def actualSolve(self, lp): log.debug("Running SAS") if not self.sasAvailable(): - raise PulpSolverError( - "SAS94: Cannot connect to a SAS session." - ) + raise PulpSolverError("SAS94: SAS session might have timed out.") sas = self.sas if len(lp.sos1) or len(lp.sos2): raise PulpSolverError( @@ -236,21 +263,46 @@ def actualSolve(self, lp): ) postfix = uuid4().hex[:16] - tmpMps, tmpMst = self.create_tmp_files(lp.name, "mps", "mst") + mpsName = f"pulp{postfix}.mps" + localMps = os.path.join(self.tmpDir, mpsName) + remoteMps = f"/tmp/{mpsName}" # Remote machine is always Linux + mstName = f"pulp{postfix}.mst" + localMst = os.path.join(self.tmpDir, mstName) + remoteMst = f"/tmp/{mstName}" # Remote machine is always Linux - vs = lp.writeMPS(tmpMps, with_objsense=False) + vs = lp.writeMPS(localMps, with_objsense=False) - nameLen = self._get_max_upload_len(tmpMps) + nameLen = self._get_max_upload_len(localMps) if nameLen > MAX_NAME_LENGTH: raise PulpSolverError( f"SAS94: The lengths of the variable or constraint names \ (including indices) should not exceed {MAX_NAME_LENGTH}." ) - proc = "OPTMILP" if self.mip else "OPTLP" + # If we use a remote SAS installation, need to upload the file + upload_mps = False + usedMps = localMps + if not sas.file_info(localMps, quiet=True): + sas.upload(localMps, remoteMps, overwrite=True) + usedMps = remoteMps + upload_mps = True + + # Figure out if the problem has integer variables + with_opt = self.optionsDict.pop("with", None) + if with_opt == "lp": + proc = "OPTLP" + elif with_opt == "milp": + proc = "OPTMILP" + else: + proc = "OPTMILP" if (lp.isMIP() and self.mip) else "OPTLP" - optionList = ["warmStart", "decomp", "decompmaster", - "decompsubprob", "rootnode"] + optionList = [ + "warmStart", + "decomp", + "decompmaster", + "decompsubprob", + "rootnode", + ] solverOptions = { key: self.optionsDict[key] @@ -277,14 +329,26 @@ def actualSolve(self, lp): decompsubprob_str = self._create_statement_str("decompsubprob") rootnode_str = self._create_statement_str("rootnode") - if lp.isMIP() and not self.mip: + if lp.isMIP() and (proc == "OPTLP" or not self.mip): warnings.warn( "SAS94 will solve the relaxed problem of the MILP instance." ) # Handle warmstart warmstart_str = "" + upload_pin = False if warmStart: - self._write_sol(filename=tmpMst, vs=vs) + self._write_sol(filename=localMst, vs=vs) + + # If we use a remote SAS installation, need to upload the file + usedMst = localMst + if not sas.file_info(localMst, quiet=True): + sas.upload( + localMst, + remoteMst, + overwrite=True, + ) + usedMst = remoteMst + upload_pin = True # Set the warmstart basis option if proc == "OPTMILP": @@ -296,9 +360,10 @@ def actualSolve(self, lp): getnames=yes; run; """.format( - primalin=tmpMst, + primalin=usedMst, postfix=postfix, ) + solverOptions["primalin"] = f"primalin{postfix}" elif proc == "OPTLP": pass @@ -307,12 +372,22 @@ def actualSolve(self, lp): option + "=" + str(value) for option, value in solverOptions.items() ) + # Set some SAS options to make the log more clean + sas_options = "option notes nonumber nodate nosource pagesize=max;" + # Find the version of 9.4 we are using - if sas.sasver.startswith("9.04.01M5"): + major_version = sas.sasver[0] + minor_version = sas.sasver.split("M", 1)[1][0] + if major_version == "9" and int(minor_version) < 5: + raise NotImplementedError( + "Support for SAS 9.4 M4 and earlier is not implemented." + ) + elif major_version == "9" and int(minor_version) == 5: # In 9.4M5 we have to create an MPS data set from an MPS file first # Earlier versions will not work because the MPS format in incompatible' res = sas.submit( """ + {sas_options} option notes nonumber nodate nosource pagesize=max; {warmstart} %MPS2SASD(MPSFILE="{mpsfile}", OUTDATA=mpsdata{postfix}, MAXLEN={maxLen}, FORMAT=FREE); @@ -325,9 +400,10 @@ def actualSolve(self, lp): proc delete data=mpsdata{postfix}; run; """.format( + sas_options=sas_options, warmstart=warmstart_str, postfix=postfix, - mpsfile=tmpMps, + mpsfile=usedMps, proc=proc, maxLen=min(nameLen, MAX_NAME_LENGTH), options=opt_str, @@ -341,10 +417,9 @@ def actualSolve(self, lp): ) else: # Since 9.4M6+ optlp/optmilp can read mps files directly - # TODO: Check whether there are limits for length of variable and constraint names res = sas.submit( """ - option notes nonumber nodate nosource pagesize=max; + {sas_options} {warmstart} proc {proc} mpsfile=\"{mpsfile}\" {options} primalout=primalout{postfix} dualout=dualout{postfix}; {decomp} @@ -354,10 +429,11 @@ def actualSolve(self, lp): {rootnode} run; """.format( + sas_options=sas_options, warmstart=warmstart_str, postfix=postfix, proc=proc, - mpsfile=tmpMps, + mpsfile=usedMps, options=opt_str, decomp=decomp_str, decompmaster=decompmaster_str, @@ -368,11 +444,19 @@ def actualSolve(self, lp): results="TEXT", ) - self.delete_tmp_files(tmpMps, tmpMst) + # Clean up local files + self.delete_tmp_files(localMps, localMst) + + # Clean up uploaded files + if upload_mps: + sas.file_delete(remoteMps, quiet=True) + if upload_pin: + sas.file_delete(remoteMst, quiet=True) # Store SAS output if self.msg: - print(res["LOG"]) + self._log = res["LOG"] + print(self._log) self._macro = dict( (key.strip(), value.strip()) for key, value in ( @@ -380,9 +464,7 @@ def actualSolve(self, lp): ) ) - primal_out = sas.sd2df(f"primalout{postfix}") - dual_out = sas.sd2df(f"dualout{postfix}") - + # Check for error and raise exception if self._macro.get("STATUS", "ERROR") != "OK": raise PulpSolverError( "PuLP: Error ({err_name}) \ @@ -390,6 +472,10 @@ def actualSolve(self, lp): err_name=self._macro.get("STATUS", "ERROR"), name=lp.name ) ) + + # Prepare output + primal_out = sas.sd2df(f"primalout{postfix}") + dual_out = sas.sd2df(f"dualout{postfix}") status = self._read_solution(lp, primal_out, dual_out, proc) return status @@ -424,7 +510,6 @@ def __init__( keepFiles=False, warmStart=False, timeLimit=None, - casOptions={}, **solverParams, ): """ @@ -432,9 +517,21 @@ def __init__( :param bool msg: if False, no log is shown :param bool keepFiles: if False, mps and mst files will not be saved :param bool warmStart: if False, no warmstart or initial primal solution provided - :param dict casOptions: options for cas connection. :param solverParams: SAS proc OPTMILP or OPTLP parameters """ + + # Extract cas_options connection options + self._cas_options = {} + for option in SWAT_OPTIONS: + value = solverParams.pop(option, None) + if value: + self._cas_options[option] = value + + if self._cas_options == {}: + self._cas_options["hostname"] = os.environ["CAS_SERVER"] + self._cas_options["port"] = os.environ["CAS_PORT"] + self._cas_options["authinfo"] = os.environ["CAS_AUTHINFO"] + SASsolver.__init__( self, mip=mip, @@ -442,10 +539,10 @@ def __init__( keepFiles=keepFiles, warmStart=warmStart, timeLimit=timeLimit, - casOptions=casOptions, **solverParams, ) - self.cas = None + + self.cas = swat.CAS(**self._cas_options) def __del__(self): if self.cas: @@ -457,8 +554,7 @@ def available(self): def sasAvailable(self): try: if not self.cas: - casOptions = self.optionsDict.get("casOptions", None) - self.cas = swat.CAS(**casOptions) + return False with redirect_stdout(SASLogWriter(self.msg)) as self._log_writer: # Load the optimization action set @@ -472,24 +568,27 @@ def actualSolve(self, lp): log.debug("Running SAS") if not self.sasAvailable(): - raise PulpSolverError( - """SASCAS: Cannot connect to a CAS session.""" - ) + raise PulpSolverError("""SASCAS: Cannot connect to a CAS session.""") s = self.cas if len(lp.sos1) or len(lp.sos2): raise PulpSolverError( "SASCAS: Currently SAS doesn't support SOS1 and SOS2." ) - solverOptions = { - key: self.optionsDict[key] - for key in self.optionsDict.keys() - if key not in ["warmStart", "casOptions"] - } - warmStart = self.optionsDict.get("warmStart") + warmStart = self.optionsDict.pop("warmStart", False) + # Figure out if the problem has integer variables + with_opt = self.optionsDict.pop("with", None) + if with_opt == "lp": + proc = "OPTLP" + elif with_opt == "milp": + proc = "OPTMILP" + else: + proc = "OPTMILP" if (lp.isMIP() and self.mip) else "OPTLP" + + # The options are exactly what it's left in the optionsDict + solverOptions = self.optionsDict - proc = "OPTMILP" if self.mip else "OPTLP" # Get Obj Sense if lp.sense == constants.LpMaximize: solverOptions["objsense"] = "max" @@ -579,13 +678,13 @@ def _get_output(self, lp, s, r, proc, postfix): ) # If we get solution successfully. if proc == "OPTMILP": - primal_out = s.CASTable(name=f"primalout{postfix}") - primal_out = primal_out[["_VAR_", "_VALUE_", "_STATUS_", "_R_COST_"]] + primal_out = s.CASTable(name=f"primalout{postfix}").to_frame() + primal_out = primal_out[["_VAR_", "_VALUE_"]] dual_out = None else: - primal_out = s.CASTable(name=f"primalout{postfix}") + primal_out = s.CASTable(name=f"primalout{postfix}").to_frame() primal_out = primal_out[["_VAR_", "_VALUE_", "_STATUS_", "_R_COST_"]] - dual_out = s.CASTable(name=f"dualout{postfix}") + dual_out = s.CASTable(name=f"dualout{postfix}").to_frame() dual_out = dual_out[["_ROW_", "_VALUE_", "_STATUS_", "_ACTIVITY_"]] return primal_out, dual_out diff --git a/pulp/tests/test_pulp.py b/pulp/tests/test_pulp.py index 5af77060..3f0de9f1 100644 --- a/pulp/tests/test_pulp.py +++ b/pulp/tests/test_pulp.py @@ -451,6 +451,8 @@ def test_initial_value(self): "CPLEX_PY", "COPT", "HiGHS_CMD", + "SAS94", + "SASCAS", ]: self.solver.optionsDict["warmStart"] = True print("\t Testing Initial value in MIP solution") @@ -647,6 +649,8 @@ def test_dual_variables_reduced_costs(self): PULP_CBC_CMD, YAPOSIB, PYGLPK, + SAS94, + SASCAS, ]: print("\t Testing dual variables and slacks reporting") pulpTestCheck( @@ -1599,11 +1603,29 @@ class COPTTest(BaseSolverTest.PuLPTest): solveInst = COPT -class SAS94Test(BaseSolverTest.PuLPTest): +class SASTest: + + def test_sas_with_option(self): + prob = LpProblem("test", LpMinimize) + X = LpVariable.dicts("x", [1, 2, 3], lowBound=0.0, cat="Integer") + prob += 2 * X[1] - 3 * X[2] - 4 * X[3], "obj" + prob += -2 * X[2] - 3 * X[3] >= -5, "R1" + prob += X[1] + X[2] + 2 * X[3] <= 4, "R2" + prob += X[1] + 2 * X[2] + 3 * X[3] <= 7, "R3" + self.solver.optionsDict["with"] = "lp" + pulpTestCheck( + prob, + self.solver, + [const.LpStatusOptimal], + {X[1]: 0.0, X[2]: 2.5, X[3]: 0.0}, + ) + + +class SAS94Test(BaseSolverTest.PuLPTest, SASTest): solveInst = SAS94 -class SASCASTest(BaseSolverTest.PuLPTest): +class SASCASTest(BaseSolverTest.PuLPTest, SASTest): solveInst = SASCAS From ade07e5ffafbb1877eed7371333a9b10f2b726cf Mon Sep 17 00:00:00 2001 From: phchri Date: Mon, 23 Sep 2024 14:39:10 +0200 Subject: [PATCH 10/11] allow SAS solver classes without valid connection --- pulp/apis/sas_api.py | 65 +++++++++++++++++++++----------------------- 1 file changed, 31 insertions(+), 34 deletions(-) diff --git a/pulp/apis/sas_api.py b/pulp/apis/sas_api.py index 39c035bd..c39e6320 100755 --- a/pulp/apis/sas_api.py +++ b/pulp/apis/sas_api.py @@ -184,9 +184,6 @@ def available(self): """True if SAS94 is available.""" return False - def sasAvailable(self): - return False - def actualSolve(self, lp, callback=None): """Solves a well-formulated lp problem.""" raise PulpSolverError("SAS94 : Not Available") @@ -227,14 +224,14 @@ def __init__( **solverParams, ) - # Connect to saspy + # Try to connect to saspy, if this fails, don't throw the error here. + # Instead we return False on available() to be consistent with + # other interfaces. self.sas = None try: self.sas = saspy.SASsession(**self._saspy_options) - except: - raise PulpSolverError( - "SAS94: Cannot connect to a SAS session. Try using using the cfgfile option." - ) + except Exception: + pass def __del__(self): if self.sas: @@ -242,9 +239,6 @@ def __del__(self): def available(self): """True if SAS94 is available.""" - return True - - def sasAvailable(self): if self.sas: return True else: @@ -254,8 +248,11 @@ def actualSolve(self, lp): """Solve a well formulated lp problem""" log.debug("Running SAS") - if not self.sasAvailable(): - raise PulpSolverError("SAS94: SAS session might have timed out.") + if not self.sas: + raise PulpSolverError( + "SAS94: Cannot connect to a SAS session. Try the cfgfile option or adjust options in that file." + ) + sas = self.sas if len(lp.sos1) or len(lp.sos2): raise PulpSolverError( @@ -494,9 +491,6 @@ def available(self): """True if SASCAS is available.""" return False - def sasAvailable(self): - return False - def actualSolve(self, lp, callback=None): """Solves a well-formulated lp problem.""" raise PulpSolverError("SASCAS : Not Available") @@ -519,6 +513,7 @@ def __init__( :param bool warmStart: if False, no warmstart or initial primal solution provided :param solverParams: SAS proc OPTMILP or OPTLP parameters """ + self.cas = None # Extract cas_options connection options self._cas_options = {} @@ -528,9 +523,9 @@ def __init__( self._cas_options[option] = value if self._cas_options == {}: - self._cas_options["hostname"] = os.environ["CAS_SERVER"] - self._cas_options["port"] = os.environ["CAS_PORT"] - self._cas_options["authinfo"] = os.environ["CAS_AUTHINFO"] + self._cas_options["hostname"] = os.getenv("CAS_SERVER") + self._cas_options["port"] = os.getenv("CAS_PORT") + self._cas_options["authinfo"] = os.getenv("CAS_AUTHINFO") SASsolver.__init__( self, @@ -542,33 +537,33 @@ def __init__( **solverParams, ) - self.cas = swat.CAS(**self._cas_options) + # Try to connect to SWAT, if this fails, don't throw the error here. + # Instead we return False on available() to be consistent with + # other interfaces. + try: + self.cas = swat.CAS(**self._cas_options) + except Exception: + pass def __del__(self): if self.cas: self.cas.close() def available(self): - return True - - def sasAvailable(self): - try: - if not self.cas: - return False - - with redirect_stdout(SASLogWriter(self.msg)) as self._log_writer: - # Load the optimization action set - self.cas.loadactionset("optimization") - return True - except: + if not self.cas: return False + else: + return True def actualSolve(self, lp): """Solve a well formulated lp problem""" log.debug("Running SAS") - if not self.sasAvailable(): - raise PulpSolverError("""SASCAS: Cannot connect to a CAS session.""") + if not self.cas: + raise PulpSolverError( + "SAS94: Cannot connect to a SAS session. Try the cfgfile option or adjust options in that file." + ) + s = self.cas if len(lp.sos1) or len(lp.sos2): raise PulpSolverError( @@ -603,6 +598,8 @@ def actualSolve(self, lp): status = None with redirect_stdout(SASLogWriter(self.msg)) as self._log_writer: + # Load the optimization action set + s.loadactionset("optimization") # Used for naming the data structure in SAS. postfix = uuid4().hex[:16] From 86e712c8baa383bcb68f8b30d7435e43d4740cdd Mon Sep 17 00:00:00 2001 From: phchri Date: Tue, 24 Sep 2024 10:36:09 +0200 Subject: [PATCH 11/11] disable printing and fix comment --- pulp/apis/sas_api.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pulp/apis/sas_api.py b/pulp/apis/sas_api.py index c39e6320..51c207e7 100755 --- a/pulp/apis/sas_api.py +++ b/pulp/apis/sas_api.py @@ -229,7 +229,9 @@ def __init__( # other interfaces. self.sas = None try: + saspy.logger.disabled = True self.sas = saspy.SASsession(**self._saspy_options) + saspy.logger.disabled = False except Exception: pass @@ -561,7 +563,7 @@ def actualSolve(self, lp): if not self.cas: raise PulpSolverError( - "SAS94: Cannot connect to a SAS session. Try the cfgfile option or adjust options in that file." + "SASCAS: Cannot connect to a CAS. Try setting CAS connection options." ) s = self.cas