diff --git a/.github/workflows/ccpp.yml b/.github/workflows/ccpp.yml index 59f9ecae93..f3320205bb 100644 --- a/.github/workflows/ccpp.yml +++ b/.github/workflows/ccpp.yml @@ -883,7 +883,6 @@ jobs: - name: Tests run: | - set OMP_NUM_THREADS=1 set NELSON_TERM_IS_UNICODE_SUPPORTED=TRUE set PATH=C:\Program Files\Microsoft MPI\Bin;%PATH% set TESTS_RESULT_DIR=%GITHUB_WORKSPACE%/artifacts diff --git a/modules/ipc/src/cpp/NelsonInterprocess.cpp b/modules/ipc/src/cpp/NelsonInterprocess.cpp index 5b3b88c5dd..607a9e430a 100644 --- a/modules/ipc/src/cpp/NelsonInterprocess.cpp +++ b/modules/ipc/src/cpp/NelsonInterprocess.cpp @@ -30,12 +30,14 @@ //============================================================================= namespace Nelson { //============================================================================= +constexpr std::size_t KILOBYTE = 1024; +constexpr std::size_t MEGABYTE = 1024 * KILOBYTE; #if (defined(_LP64) || defined(_WIN64)) -constexpr auto OFF_MSG_SIZE = sizeof(double) * 16 * 1024; -constexpr auto MAX_MSG_SIZE = sizeof(double) * (4096 * 4096) + OFF_MSG_SIZE; -constexpr auto MAX_NB_MSG = 4; +constexpr std::size_t OFF_MSG_SIZE = sizeof(double) * 16 * KILOBYTE; +constexpr std::size_t MAX_MSG_SIZE = sizeof(double) * (4 * MEGABYTE) + OFF_MSG_SIZE; +constexpr std::size_t MAX_NB_MSG = 4; #else -constexpr auto MAX_MSG_SIZE = sizeof(double) * (1024 * 1024); +constexpr auto MAX_MSG_SIZE = sizeof(double) * MEGABYTE; constexpr auto MAX_NB_MSG = 2; #endif constexpr auto TIMEOUT_COUNT = 20; diff --git a/modules/os_functions/src/cpp/SystemCommand.cpp b/modules/os_functions/src/cpp/SystemCommand.cpp index 426e511528..386cb649d4 100644 --- a/modules/os_functions/src/cpp/SystemCommand.cpp +++ b/modules/os_functions/src/cpp/SystemCommand.cpp @@ -17,8 +17,6 @@ #include #include #endif - -// #include #include #include #include @@ -34,6 +32,7 @@ #include "i18n.hpp" #include "PredefinedErrorMessages.hpp" #include "SystemCommandTask.hpp" +#include "nlsBuildConfig.h" //============================================================================= namespace Nelson { //============================================================================= @@ -69,60 +68,70 @@ ParallelSystemCommand(const wstringVector& commands, const std::vector& std::vector> results; size_t nbCommands = commands.size(); results.resize(nbCommands); - size_t nbThreadsMax = (size_t)NelsonConfiguration::getInstance()->getMaxNumCompThreads(); - size_t nbThreads = std::min(nbCommands, nbThreadsMax); - - std::vector threads; std::vector taskList; +#if defined(__APPLE__) or (defined(_WIN32) && not defined(_WIN64)) + for (ompIndexType k = 0; k < (ompIndexType)nbCommands; k++) { + SystemCommandTask* task = new SystemCommandTask(); + taskList.push_back(task); + } +#if not defined(__APPLE__) +#if WITH_OPENMP +#pragma omp parallel for +#endif +#endif + for (ompIndexType k = 0; k < (ompIndexType)nbCommands; k++) { + taskList[k]->evaluateCommand(commands[k], timeouts[k]); + } +#else + size_t nbThreadsMax = (size_t)NelsonConfiguration::getInstance()->getMaxNumCompThreads(); + size_t nbThreads = std::min(nbCommands, nbThreadsMax); + std::vector threadList; for (size_t k = 0; k < nbCommands; k++) { try { SystemCommandTask* task = new SystemCommandTask(); taskList.push_back(task); - threads.emplace_back([task, commands, timeouts, k]() { + threadList.emplace_back([task, commands, timeouts, k]() { task->evaluateCommand(commands[k], timeouts[k]); }); } catch (std::bad_alloc&) { Error(ERROR_MEMORY_ALLOCATION); } } - - while (true) { + bool allTasksFinished = false; + do { if (NelsonConfiguration::getInstance()->getInterruptPending(evaluatorID)) { - for (SystemCommandTask* task : taskList) { - task->terminate(); + for (size_t k = 0; k < nbCommands; k++) { + taskList[k]->terminate(); } break; } - if (withEventsLoop) { ProcessEventsDynamicFunction(); } + std::this_thread::sleep_for(std::chrono::milliseconds(uint64(1))); - std::this_thread::sleep_for(std::chrono::milliseconds(1)); + allTasksFinished = std::all_of(threadList.begin(), threadList.end(), + [](const auto& thread) { return thread.joinable(); }); - bool allTasksFinished = true; - for (SystemCommandTask* task : taskList) { - if (task->isRunning()) { - allTasksFinished = false; - break; - } - } + allTasksFinished = allTasksFinished + && std::all_of(taskList.begin(), taskList.end(), + [](const auto& task) { return !task->isRunning(); }); if (allTasksFinished) { - for (auto& thread : threads) { + for (auto& thread : threadList) { thread.join(); } break; } - } + } while (!allTasksFinished); +#endif for (size_t k = 0; k < nbCommands; k++) { if (taskList[k]) { results[k] = taskList[k]->getResult(); } } - for (SystemCommandTask* task : taskList) { if (task) { delete task; @@ -132,7 +141,8 @@ ParallelSystemCommand(const wstringVector& commands, const std::vector& taskList.clear(); return results; -} //============================================================================= +} +//============================================================================= static void initGuiDynamicLibrary() { diff --git a/modules/os_functions/src/cpp/SystemCommandTask.cpp b/modules/os_functions/src/cpp/SystemCommandTask.cpp index cb828b805a..360b58ad6c 100644 --- a/modules/os_functions/src/cpp/SystemCommandTask.cpp +++ b/modules/os_functions/src/cpp/SystemCommandTask.cpp @@ -12,16 +12,49 @@ #define _CRT_SECURE_NO_WARNINGS #define WIN32_LEAN_AND_MEAN #endif -#include -#include -#include #include +#include #include "SystemCommandTask.hpp" #include "StringHelpers.hpp" #include "characters_encoding.hpp" //============================================================================= namespace Nelson { //============================================================================= +static const int SLEEP_DURATION_MS = 10; +//============================================================================= +void +SystemCommandTask::evaluateCommand(const std::wstring& command, uint64 timeout) +{ + _beginTimePoint = std::chrono::steady_clock::now(); + _terminate = false; + _running = true; + + auto tempOutputFile + = std::make_unique(FileSystemWrapper::Path::unique_path()); + auto tempErrorFile + = std::make_unique(FileSystemWrapper::Path::unique_path()); + + try { + bool mustDetach = false; + std::wstring _command = detectDetachProcess(command, mustDetach); + std::wstring cmd = buildCommandString(_command); + + if (mustDetach) { + executeDetachedProcess(cmd); + } else { + executeAttachedProcess(cmd, *tempOutputFile, *tempErrorFile, timeout); + } + } catch (const std::exception& e) { + _message = utf8_to_wstring(e.what()); + _exitCode = -1; + } + + cleanupTempFiles(*tempOutputFile, *tempErrorFile); + + _running = false; + _duration = this->getDuration(); +} +//============================================================================= void SystemCommandTask::terminate() { @@ -39,6 +72,16 @@ SystemCommandTask::isRunning() return _running; } //============================================================================= +int +SystemCommandTask::exitCodeAbort() +{ +#ifdef _MSC_VER + return int(258); // WAIT_TIMEOUT +#else + return int(128 + SIGABRT); +#endif +} +//============================================================================= std::tuple SystemCommandTask::getResult() { @@ -57,85 +100,94 @@ SystemCommandTask::getDuration() .count(); } //============================================================================= +std::wstring +SystemCommandTask::buildCommandString(const std::wstring& _command) +{ + std::wstring argsShell = getPlatformSpecificShellArgs(); + return L"\"" + boost::process::shell().wstring() + L"\" " + argsShell + L"\"" + _command + + L"\""; +} +//============================================================================= void -SystemCommandTask::evaluateCommand(const std::wstring& command, uint64 timeout) +SystemCommandTask::executeDetachedProcess(const std::wstring& cmd) { - _beginTimePoint = std::chrono::steady_clock::now(); + boost::process::child childProcess(cmd); + childProcess.detach(); + _exitCode = 0; +} +//============================================================================= +void +SystemCommandTask::executeAttachedProcess(const std::wstring& cmd, + const FileSystemWrapper::Path& tempOutputFile, const FileSystemWrapper::Path& tempErrorFile, + uint64 timeout) +{ + boost::process::child childProcess(cmd, + boost::process::std_out > tempOutputFile.generic_string().c_str(), + boost::process::std_err > tempErrorFile.generic_string().c_str(), + boost::process::std_in < boost::process::null); - _terminate = false; - _running = true; - FileSystemWrapper::Path tempOutputFile(FileSystemWrapper::Path::unique_path()); - FileSystemWrapper::Path tempErrorFile(FileSystemWrapper::Path::unique_path()); - bool mustDetach = false; - std::wstring _command = detectDetachProcess(command, mustDetach); - std::wstring argsShell; -#ifdef _MSC_VER - argsShell = L" /a /c "; -#else - argsShell = L" -c "; -#endif - std::wstring cmd - = L"\"" + boost::process::shell().wstring() + L"\" " + argsShell + L"\"" + _command + L"\""; - if (mustDetach) { - boost::process::child childProcess(cmd); - childProcess.detach(); - _exitCode = 0; - } else { - boost::process::child childProcess(cmd, - boost::process::std_out > tempOutputFile.generic_string().c_str(), - boost::process::std_err > tempErrorFile.generic_string().c_str(), - boost::process::std_in < boost::process::null); + monitorChildProcess(childProcess, timeout); - while (childProcess.running() && !_terminate) { - std::chrono::steady_clock::time_point _currentTimePoint - = std::chrono::steady_clock::now(); - if ((timeout != 0) - && (std::chrono::duration_cast( - _currentTimePoint - _beginTimePoint) - >= std::chrono::seconds(timeout))) { - _terminate = true; - break; - } - std::this_thread::sleep_for(std::chrono::milliseconds(10)); + if (!_terminate) { + _exitCode = (int)childProcess.exit_code(); + _message = readProcessOutput(tempOutputFile, tempErrorFile); + } +} +//============================================================================= +void +SystemCommandTask::monitorChildProcess(boost::process::child& childProcess, uint64 timeout) +{ + while (childProcess.running() && !_terminate) { + auto _currentTimePoint = std::chrono::steady_clock::now(); + if (timeout != 0 + && (std::chrono::duration_cast( + _currentTimePoint - _beginTimePoint) + >= std::chrono::seconds(timeout))) { + _terminate = true; + break; } + std::this_thread::sleep_for(std::chrono::milliseconds(SLEEP_DURATION_MS)); + } - if (_terminate) { - this->_duration = this->getDuration(); - this->_exitCode = exitCodeAbort(); - FileSystemWrapper::Path::remove(tempOutputFile); - FileSystemWrapper::Path::remove(tempErrorFile); - childProcess.terminate(); - _running = false; - return; - } else { - this->_exitCode = (int)childProcess.exit_code(); - } + if (_terminate) { + _duration = this->getDuration(); + _exitCode = exitCodeAbort(); + childProcess.terminate(); + } +} +//============================================================================= +std::wstring +SystemCommandTask::readProcessOutput( + const FileSystemWrapper::Path& tempOutputFile, const FileSystemWrapper::Path& tempErrorFile) +{ + std::wstring outputResult; + if (_exitCode) { - std::wstring outputResult; - if (this->_exitCode) { - outputResult = readFile(tempErrorFile); - if (outputResult.empty()) { - outputResult = readFile(tempOutputFile); - } - } else { + outputResult = readFile(tempErrorFile); + if (outputResult.empty()) { outputResult = readFile(tempOutputFile); } - _message = outputResult; + } else { + outputResult = readFile(tempOutputFile); } + return outputResult; +} +//============================================================================= +void +SystemCommandTask::cleanupTempFiles( + const FileSystemWrapper::Path& tempOutputFile, const FileSystemWrapper::Path& tempErrorFile) +{ FileSystemWrapper::Path::remove(tempOutputFile); FileSystemWrapper::Path::remove(tempErrorFile); - - _running = false; - _duration = this->getDuration(); } //============================================================================= -int -SystemCommandTask::exitCodeAbort() +std::wstring +SystemCommandTask::getPlatformSpecificShellArgs() { #ifdef _MSC_VER - return int(258); // WAIT_TIMEOUT + return L" /a /c "; #else - return int(128 + SIGABRT); + return L" -c "; #endif } //============================================================================= @@ -174,31 +226,27 @@ std::wstring SystemCommandTask::readFile(const FileSystemWrapper::Path& filePath) { std::string result; - FILE* pFile; #ifdef _MSC_VER - pFile = _wfopen(filePath.wstring().c_str(), L"r"); + FILE* pFile = _wfopen(filePath.wstring().c_str(), L"r"); #else - pFile = fopen(filePath.string().c_str(), "r"); + FILE* pFile = fopen(filePath.string().c_str(), "r"); #endif if (pFile != nullptr) { -#define bufferSize 4096 -#define bufferSizeMax 4096 * 2 + constexpr std::streamsize bufferSize = 16384; char buffer[bufferSize]; - result.reserve(bufferSizeMax); - while (fgets(buffer, sizeof(buffer), pFile)) { + + while (fgets(buffer, bufferSize * sizeof(char), pFile)) { #ifdef _MSC_VER std::string str = std::string(buffer); - boost::replace_all(str, "\r\n", "\n"); + StringHelpers::replace_all(str, "\r\n", "\n"); OemToCharBuffA(str.c_str(), const_cast(str.c_str()), (DWORD)str.size()); result.append(str); #else result.append(buffer); #endif } - if (result.size() > 0) { - if (*result.rbegin() != '\n') { - result.append("\n"); - } + if (!result.empty() && result.back() != '\n') { + result.push_back('\n'); } fclose(pFile); } diff --git a/modules/os_functions/src/cpp/SystemCommandTask.hpp b/modules/os_functions/src/cpp/SystemCommandTask.hpp index 4076de8934..1ad5b68419 100644 --- a/modules/os_functions/src/cpp/SystemCommandTask.hpp +++ b/modules/os_functions/src/cpp/SystemCommandTask.hpp @@ -13,6 +13,9 @@ #include #include #include +#include +#include +#include #include "Types.hpp" #include "FileSystemWrapper.hpp" //============================================================================= @@ -46,17 +49,41 @@ class SystemCommandTask uint64 _duration = uint64(0); std::chrono::steady_clock::time_point _beginTimePoint; //============================================================================= + std::wstring + buildCommandString(const std::wstring& _command); + //============================================================================= + void + executeDetachedProcess(const std::wstring& cmd); + //============================================================================= + void + executeAttachedProcess(const std::wstring& cmd, const FileSystemWrapper::Path& tempOutputFile, + const FileSystemWrapper::Path& tempErrorFile, uint64 timeout); + //============================================================================= + void + monitorChildProcess(boost::process::child& childProcess, uint64 timeout); + //============================================================================= + std::wstring + readProcessOutput(const FileSystemWrapper::Path& tempOutputFile, + const FileSystemWrapper::Path& tempErrorFile); + //============================================================================= + void + cleanupTempFiles(const FileSystemWrapper::Path& tempOutputFile, + const FileSystemWrapper::Path& tempErrorFile); + //============================================================================= + std::wstring + getPlatformSpecificShellArgs(); + //============================================================================= int exitCodeAbort(); //============================================================================= std::wstring - detectDetachProcess(const std::wstring& command, bool& haveDetach); + readFile(const FileSystemWrapper::Path& filePath); //============================================================================= std::wstring - cleanCommand(const std::wstring& command); + detectDetachProcess(const std::wstring& command, bool& haveDetach); //============================================================================= std::wstring - readFile(const FileSystemWrapper::Path& filePath); + cleanCommand(const std::wstring& command); //============================================================================= }; //============================================================================= diff --git a/modules/os_functions/tests/test_system_3.m b/modules/os_functions/tests/test_system_3.m index aef0a9ea4a..13d3437d50 100644 --- a/modules/os_functions/tests/test_system_3.m +++ b/modules/os_functions/tests/test_system_3.m @@ -16,11 +16,7 @@ [s, w, d] = system(["sleep 4", "sleep 6", "sleep 9"]); t = toc(); end -if (maxNumCompThreads() > 3) - assert_istrue(t >= 8 && t < 12) -else - assert_istrue(t >= 8 && t < 25) -end +assert_istrue(t >= 8 && t < 25) assert_istrue(d(1) >= 3000 && d(1) <= 5000) assert_istrue(d(2) >= 5000 && d(2) <= 7000) assert_istrue(d(3) >= 8000 && d(2) <= 10000)