From 1a43ce5208355a2c9ba2bddf0bd9c4879265fa76 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Thu, 14 May 2020 21:03:09 +0200 Subject: [PATCH 01/36] Add list-in-list C++ regression test --- .../cpp/src/libmuscle/tests/test_data.cpp | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/libmuscle/cpp/src/libmuscle/tests/test_data.cpp b/libmuscle/cpp/src/libmuscle/tests/test_data.cpp index 88bf5dd4..0b3419ac 100644 --- a/libmuscle/cpp/src/libmuscle/tests/test_data.cpp +++ b/libmuscle/cpp/src/libmuscle/tests/test_data.cpp @@ -349,6 +349,27 @@ TEST(libmuscle_mcp_data, list_dict) { ASSERT_EQ(data[2]["test2"].as(), 87); } +TEST(libmuscle_mcp_data, list_list) { + auto list = Data::list("test1", "test2", 13); + + msgpack::sbuffer buf; + msgpack::pack(buf, Data::list(1, 2.0, list)); + auto zone = std::make_shared(); + auto data = unpack_data(zone, buf.data(), buf.size()); + + ASSERT_TRUE(data.is_a_list()); + ASSERT_EQ(data.size(), 3); + ASSERT_TRUE(data[0].is_a()); + ASSERT_EQ(data[0].as(), 1); + ASSERT_TRUE(data[1].is_a()); + ASSERT_EQ(data[1].as(), 2.0); + ASSERT_TRUE(data[2].is_a_list()); + ASSERT_EQ(data[2].size(), 3); + ASSERT_EQ(data[2][0].as(), "test1"); + ASSERT_EQ(data[2][1].as(), "test2"); + ASSERT_EQ(data[2][2].as(), 13); +} + TEST(libmuscle_mcp_data, list_dataconstref) { // regression test Data dict = Data::list(DataConstRef()); From dc24236a48ead32a9a273d7c58f8d3838bb98645 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Thu, 21 May 2020 18:16:42 +0200 Subject: [PATCH 02/36] Install mcp/data_pack so that dict and list building works --- libmuscle/cpp/build/libmuscle/Makefile | 3 ++- libmuscle/cpp/src/libmuscle/libmuscle.hpp | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/libmuscle/cpp/build/libmuscle/Makefile b/libmuscle/cpp/build/libmuscle/Makefile index c947552a..8e564564 100644 --- a/libmuscle/cpp/build/libmuscle/Makefile +++ b/libmuscle/cpp/build/libmuscle/Makefile @@ -42,7 +42,8 @@ header_root := $(CURDIR)/../../src CXXFLAGS += -I$(header_root) public_headers := libmuscle/data.hpp libmuscle/data.tpp libmuscle/instance.hpp -public_headers += libmuscle/libmuscle.hpp libmuscle/message.hpp +public_headers += libmuscle/libmuscle.hpp libmuscle/mcp/data_pack.hpp +public_headers += libmuscle/mcp/data_pack.tpp libmuscle/message.hpp public_headers += libmuscle/ports_description.hpp libmuscle/util.hpp libmuscle/util.tpp installed_headers := $(public_headers:%=$(PREFIX)/include/%) diff --git a/libmuscle/cpp/src/libmuscle/libmuscle.hpp b/libmuscle/cpp/src/libmuscle/libmuscle.hpp index 19813429..c7d74ada 100644 --- a/libmuscle/cpp/src/libmuscle/libmuscle.hpp +++ b/libmuscle/cpp/src/libmuscle/libmuscle.hpp @@ -1,5 +1,6 @@ #include #include +#include #include #include From d110841bc3bac42bd41e022f4396010e3dd3a51d Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Thu, 21 May 2020 18:17:31 +0200 Subject: [PATCH 03/36] Add missing dict/list builders to ABI --- libmuscle/cpp/build/libmuscle/libmuscle.version | 2 ++ libmuscle/cpp/build/libmuscle/libmuscle.version.in | 2 ++ libmuscle/cpp/build/libmuscle/libmuscle_mpi.version | 2 ++ 3 files changed, 6 insertions(+) diff --git a/libmuscle/cpp/build/libmuscle/libmuscle.version b/libmuscle/cpp/build/libmuscle/libmuscle.version index fef68798..96854ff1 100644 --- a/libmuscle/cpp/build/libmuscle/libmuscle.version +++ b/libmuscle/cpp/build/libmuscle/libmuscle.version @@ -120,7 +120,9 @@ "libmuscle::impl::Data::byte_array(char const*, unsigned int)"; "libmuscle::impl::Data::byte_array(unsigned int)"; "libmuscle::impl::Data::dict()"; + "libmuscle::impl::Data::init_dict_(unsigned int)"; "libmuscle::impl::Data::list()"; + "libmuscle::impl::Data::init_list_(unsigned int)"; "libmuscle::impl::Data::nils(unsigned long)"; "libmuscle::impl::Data::operator=(libmuscle::impl::Data const&)"; "libmuscle::impl::Data::operator[](unsigned long)"; diff --git a/libmuscle/cpp/build/libmuscle/libmuscle.version.in b/libmuscle/cpp/build/libmuscle/libmuscle.version.in index 5163da10..3b463354 100644 --- a/libmuscle/cpp/build/libmuscle/libmuscle.version.in +++ b/libmuscle/cpp/build/libmuscle/libmuscle.version.in @@ -120,7 +120,9 @@ "libmuscle::impl::Data::byte_array(char const*, unsigned int)"; "libmuscle::impl::Data::byte_array(unsigned int)"; "libmuscle::impl::Data::dict()"; + "libmuscle::impl::Data::init_dict_(unsigned int)"; "libmuscle::impl::Data::list()"; + "libmuscle::impl::Data::init_list_(unsigned int)"; "libmuscle::impl::Data::nils(unsigned long)"; "libmuscle::impl::Data::operator=(libmuscle::impl::Data const&)"; "libmuscle::impl::Data::operator[](unsigned long)"; diff --git a/libmuscle/cpp/build/libmuscle/libmuscle_mpi.version b/libmuscle/cpp/build/libmuscle/libmuscle_mpi.version index fa6c204a..395cb218 100644 --- a/libmuscle/cpp/build/libmuscle/libmuscle_mpi.version +++ b/libmuscle/cpp/build/libmuscle/libmuscle_mpi.version @@ -120,7 +120,9 @@ "libmuscle::impl::Data::byte_array(char const*, unsigned int)"; "libmuscle::impl::Data::byte_array(unsigned int)"; "libmuscle::impl::Data::dict()"; + "libmuscle::impl::Data::init_dict_(unsigned int)"; "libmuscle::impl::Data::list()"; + "libmuscle::impl::Data::init_list_(unsigned int)"; "libmuscle::impl::Data::nils(unsigned long)"; "libmuscle::impl::Data::operator=(libmuscle::impl::Data const&)"; "libmuscle::impl::Data::operator[](unsigned long)"; From 74b79a030ac5f461baeaf3869c7f9d250514c1fd Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Fri, 22 May 2020 14:56:51 +0200 Subject: [PATCH 04/36] Allow accessing settings as 4-byte ints (#55) --- docs/source/fortran_api.rst | 37 +++- libmuscle/cpp/build/ymmsl/ymmsl.version | 3 + .../cpp/src/libmuscle/tests/test_instance.cpp | 10 + .../src/ymmsl/bindings/ymmsl_fortran_c.cpp | 97 ++++++++++ libmuscle/cpp/src/ymmsl/settings.hpp | 14 +- libmuscle/cpp/src/ymmsl/settings.tpp | 20 ++ .../src/libmuscle/tests/test_settings.f03 | 46 +++-- libmuscle/fortran/src/ymmsl/ymmsl.f03 | 180 ++++++++++++++++++ scripts/make_ymmsl_api.py | 7 +- 9 files changed, 385 insertions(+), 29 deletions(-) diff --git a/docs/source/fortran_api.rst b/docs/source/fortran_api.rst index febc3bb1..288dd3e5 100644 --- a/docs/source/fortran_api.rst +++ b/docs/source/fortran_api.rst @@ -2302,6 +2302,22 @@ YMMSL_Settings :r is: ``.true.`` if the value is of type logical. :rtype is: logical +.. f:function:: YMMSL_Settings_is_a_int4(self, key, err_code, err_msg) + + Return whether a value is of type ``YMMSL_int4``. + + This returns ``.true.`` if the value is an integer and fits in an int4. + + If the given key does not exist, then ``err_code`` will be set to + ``YMMSL_out_of_bounds`` and the result will be invalid. + + :p YMMSL_Settings self: The Settings object to inspect. + :p character key: The name of the setting to check. + :p integer err_code: An error code output (optional). + :p character err_msg: An error message output (allocatable, optional). + :r is: ``.true.`` if the value is of type ``YMMSL_int4``. + :rtype is: logical + .. f:function:: YMMSL_Settings_is_a_int8(self, key, err_code, err_msg) Return whether a value is of type ``YMMSL_int8``. @@ -2365,9 +2381,10 @@ YMMSL_Settings If no setting with the given key exists, one is added, if one does, it is overwritten. - ``value`` may be a character (string), a logical, an 8-byte integer (e.g. - ``YMMSL_int8``), an 8-byte real number (``YMMSL_real8``), or a one- or - two-dimensional arrays of 8-byte real numbers. + ``value`` may be a character (string), a logical, a 4-byte integer (e.g. + ``YMMSL_int4``), an 8-byte integer (e.g. ``YMMSL_int8``), an 8-byte real + number (``YMMSL_real8``), or a one- or two-dimensional arrays of 8-byte real + numbers. :p YMMSL_Settings self: The Settings object to modify. :p character key: The name of the setting. @@ -2401,6 +2418,20 @@ YMMSL_Settings :r value: The value at the given index :rtype value: logical +.. f:function:: YMMSL_Settings_get_as_int4(self, key, err_code, err_msg) + + Return the value of an integer-typed setting. + + If this setting is not currently set to a integer-typed value, or the + value is out of range for an int4, then ``err_code`` will be set to + ``YMMSL_bad_cast`` and the result will be invalid. + + :p character key: The name of the setting to get. + :p integer err_code: An error code output (optional). + :p character err_msg: An error message output (allocatable, optional). + :r value: The value at the given index (YMMSL_int4) + :rtype value: integer + .. f:function:: YMMSL_Settings_get_as_int8(self, key, err_code, err_msg) Return the value of an integer-typed setting. diff --git a/libmuscle/cpp/build/ymmsl/ymmsl.version b/libmuscle/cpp/build/ymmsl/ymmsl.version index d48e63d6..32a4773b 100644 --- a/libmuscle/cpp/build/ymmsl/ymmsl.version +++ b/libmuscle/cpp/build/ymmsl/ymmsl.version @@ -122,18 +122,21 @@ YMMSL_Settings_size_; YMMSL_Settings_empty_; YMMSL_Settings_is_a_character_; + YMMSL_Settings_is_a_int4_; YMMSL_Settings_is_a_int8_; YMMSL_Settings_is_a_real8_; YMMSL_Settings_is_a_logical_; YMMSL_Settings_is_a_real8array_; YMMSL_Settings_is_a_real8array2_; YMMSL_Settings_set_character_; + YMMSL_Settings_set_int4_; YMMSL_Settings_set_int8_; YMMSL_Settings_set_real8_; YMMSL_Settings_set_logical_; YMMSL_Settings_set_real8array_; YMMSL_Settings_set_real8array2_; YMMSL_Settings_get_as_character_; + YMMSL_Settings_get_as_int4_; YMMSL_Settings_get_as_int8_; YMMSL_Settings_get_as_real8_; YMMSL_Settings_get_as_logical_; diff --git a/libmuscle/cpp/src/libmuscle/tests/test_instance.cpp b/libmuscle/cpp/src/libmuscle/tests/test_instance.cpp index 6cbdeedf..65dce00b 100644 --- a/libmuscle/cpp/src/libmuscle/tests/test_instance.cpp +++ b/libmuscle/cpp/src/libmuscle/tests/test_instance.cpp @@ -154,13 +154,23 @@ TEST(libmuscle_instance, get_setting) { Settings settings; settings["test1"] = "test"; settings["test2"] = {1.0, 2.0}; + settings["test3"] = 10; + settings["test4"] = 10000000000l; // does not fit 32 bits + settings["test5"] = 10.0; + settings["test6"] = 1.0f / 3.0f; // not exactly representable TestInstance::settings_manager_(instance).base = settings; ASSERT_TRUE(instance.get_setting("test1").is_a()); ASSERT_EQ(instance.get_setting("test1").as(), "test"); ASSERT_EQ(instance.get_setting_as("test1"), "test"); + ASSERT_EQ(instance.get_setting_as>("test2"), std::vector({1.0, 2.0})); + ASSERT_EQ(instance.get_setting_as("test3"), 10l); + ASSERT_EQ(instance.get_setting_as("test4"), 10000000000l); + ASSERT_EQ(static_cast(instance.get_setting_as("test3")), 10); + ASSERT_THROW(instance.get_setting_as("test4"), std::bad_cast); + ASSERT_THROW(instance.get_setting("testx"), std::out_of_range); ASSERT_THROW(instance.get_setting_as("test1"), std::bad_cast); } diff --git a/libmuscle/cpp/src/ymmsl/bindings/ymmsl_fortran_c.cpp b/libmuscle/cpp/src/ymmsl/bindings/ymmsl_fortran_c.cpp index 57d97d95..f642e515 100644 --- a/libmuscle/cpp/src/ymmsl/bindings/ymmsl_fortran_c.cpp +++ b/libmuscle/cpp/src/ymmsl/bindings/ymmsl_fortran_c.cpp @@ -89,6 +89,51 @@ bool YMMSL_Settings_is_a_character_(std::intptr_t self, char * key, std::size_t } } +bool YMMSL_Settings_is_a_int4_(std::intptr_t self, char * key, std::size_t key_size, int * err_code, char ** err_msg, std::size_t * err_msg_len) { + Settings * self_p = reinterpret_cast(self); + std::string key_s(key, key_size); + try { + *err_code = 0; + bool result = self_p->at(key_s).is_a(); + return result; + } + catch (std::domain_error const & e) { + *err_code = 1; + static std::string msg; + msg = e.what(); + *err_msg = const_cast(msg.data()); + *err_msg_len = msg.size(); + } + catch (std::out_of_range const & e) { + *err_code = 2; + static std::string msg; + msg = e.what(); + *err_msg = const_cast(msg.data()); + *err_msg_len = msg.size(); + } + catch (std::logic_error const & e) { + *err_code = 3; + static std::string msg; + msg = e.what(); + *err_msg = const_cast(msg.data()); + *err_msg_len = msg.size(); + } + catch (std::runtime_error const & e) { + *err_code = 4; + static std::string msg; + msg = e.what(); + *err_msg = const_cast(msg.data()); + *err_msg_len = msg.size(); + } + catch (std::bad_cast const & e) { + *err_code = 5; + static std::string msg; + msg = e.what(); + *err_msg = const_cast(msg.data()); + *err_msg_len = msg.size(); + } +} + bool YMMSL_Settings_is_a_int8_(std::intptr_t self, char * key, std::size_t key_size, int * err_code, char ** err_msg, std::size_t * err_msg_len) { Settings * self_p = reinterpret_cast(self); std::string key_s(key, key_size); @@ -322,6 +367,13 @@ void YMMSL_Settings_set_character_(std::intptr_t self, char * key, std::size_t k return; } +void YMMSL_Settings_set_int4_(std::intptr_t self, char * key, std::size_t key_size, int32_t value) { + Settings * self_p = reinterpret_cast(self); + std::string key_s(key, key_size); + (*self_p)[key_s] = value; + return; +} + void YMMSL_Settings_set_int8_(std::intptr_t self, char * key, std::size_t key_size, int64_t value) { Settings * self_p = reinterpret_cast(self); std::string key_s(key, key_size); @@ -411,6 +463,51 @@ void YMMSL_Settings_get_as_character_(std::intptr_t self, char * key, std::size_ } } +int32_t YMMSL_Settings_get_as_int4_(std::intptr_t self, char * key, std::size_t key_size, int * err_code, char ** err_msg, std::size_t * err_msg_len) { + Settings * self_p = reinterpret_cast(self); + std::string key_s(key, key_size); + try { + *err_code = 0; + int32_t result = self_p->at(key_s).as(); + return result; + } + catch (std::domain_error const & e) { + *err_code = 1; + static std::string msg; + msg = e.what(); + *err_msg = const_cast(msg.data()); + *err_msg_len = msg.size(); + } + catch (std::out_of_range const & e) { + *err_code = 2; + static std::string msg; + msg = e.what(); + *err_msg = const_cast(msg.data()); + *err_msg_len = msg.size(); + } + catch (std::logic_error const & e) { + *err_code = 3; + static std::string msg; + msg = e.what(); + *err_msg = const_cast(msg.data()); + *err_msg_len = msg.size(); + } + catch (std::runtime_error const & e) { + *err_code = 4; + static std::string msg; + msg = e.what(); + *err_msg = const_cast(msg.data()); + *err_msg_len = msg.size(); + } + catch (std::bad_cast const & e) { + *err_code = 5; + static std::string msg; + msg = e.what(); + *err_msg = const_cast(msg.data()); + *err_msg_len = msg.size(); + } +} + int64_t YMMSL_Settings_get_as_int8_(std::intptr_t self, char * key, std::size_t key_size, int * err_code, char ** err_msg, std::size_t * err_msg_len) { Settings * self_p = reinterpret_cast(self); std::string key_s(key, key_size); diff --git a/libmuscle/cpp/src/ymmsl/settings.hpp b/libmuscle/cpp/src/ymmsl/settings.hpp index fde216ec..4c233ef6 100644 --- a/libmuscle/cpp/src/ymmsl/settings.hpp +++ b/libmuscle/cpp/src/ymmsl/settings.hpp @@ -129,8 +129,14 @@ class SettingValue { /** Return whether this SettingValue holds a value of the given type. * - * @param T A valid type, being one of std::string, int64_t, double, - * bool, std::vector, or + * Note that for int32_t, this function will return true only if the + * value is integer and fits in an int32_t. + * + * Since int and long are usually equivalent to int32_t or int64_t, + * you can use those values too. + * + * @param T A valid type, being one of std::string, int32_t, int64_t, + * double, bool, std::vector, or * std::vector>. */ template @@ -140,8 +146,8 @@ class SettingValue { * * Only call if is_a() returns true. * - * @param T A valid type, being one of std::string, int64_t, double, - * bool, std::vector, or + * @param T A valid type, being one of std::string, int32_t, int64_t, + * double, bool, std::vector, or * std::vector>. * * @throw std::bad_cast if the type of this value does not match the diff --git a/libmuscle/cpp/src/ymmsl/settings.tpp b/libmuscle/cpp/src/ymmsl/settings.tpp index 6e13ef36..733f214f 100644 --- a/libmuscle/cpp/src/ymmsl/settings.tpp +++ b/libmuscle/cpp/src/ymmsl/settings.tpp @@ -1,5 +1,8 @@ // Template implementation. Do not include directly! +#include + + namespace ymmsl { namespace impl { template<> @@ -12,6 +15,16 @@ inline bool SettingValue::is_a() const { return type_ == Type_::INT; } +template<> +inline bool SettingValue::is_a() const { + if (type_ != Type_::INT) return false; + + bool too_small = (int_value_ < std::numeric_limits::min()); + bool too_big = (int_value_ > std::numeric_limits::max()); + + return !(too_small || too_big); +} + template<> inline bool SettingValue::is_a() const { return type_ == Type_::FLOAT; @@ -52,6 +65,13 @@ inline int64_t SettingValue::as() const { return int_value_; } +template<> +inline int32_t SettingValue::as() const { + if (!is_a()) + throw std::bad_cast(); + return int_value_; +} + template<> inline double SettingValue::as() const { if (!is_a()) diff --git a/libmuscle/fortran/src/libmuscle/tests/test_settings.f03 b/libmuscle/fortran/src/libmuscle/tests/test_settings.f03 index 5de4dc68..375c2a4c 100644 --- a/libmuscle/fortran/src/libmuscle/tests/test_settings.f03 +++ b/libmuscle/fortran/src/libmuscle/tests/test_settings.f03 @@ -66,11 +66,12 @@ subroutine test_settings_set_get_as ra3 = reshape((/1.0d0, 2.0d0, 3.0d0, 4.0d0, 5.0d0, 6.0d0/), (/3, 2/)) call YMMSL_Settings_set(s1, 'key1', 'value1') - call YMMSL_Settings_set(s1, 'key2', 42424242424242_YMMSL_int8) - call YMMSL_Settings_set(s1, 'key3', .false.) - call YMMSL_Settings_set(s1, 'key4', 13.13d0) - call YMMSL_Settings_set(s1, 'key5', ra1) - call YMMSL_Settings_set(s1, 'key6', ra3) + call YMMSL_Settings_set(s1, 'key2', 242424242_YMMSL_int4) + call YMMSL_Settings_set(s1, 'key3', 42424242424242_YMMSL_int8) + call YMMSL_Settings_set(s1, 'key4', .false.) + call YMMSL_Settings_set(s1, 'key5', 13.13d0) + call YMMSL_Settings_set(s1, 'key6', ra1) + call YMMSL_Settings_set(s1, 'key7', ra3) call assert_true(YMMSL_Settings_contains(s1, 'key1')) call assert_true(YMMSL_Settings_contains(s1, 'key2')) @@ -78,15 +79,18 @@ subroutine test_settings_set_get_as call assert_true(YMMSL_Settings_contains(s1, 'key4')) call assert_true(YMMSL_Settings_contains(s1, 'key5')) call assert_true(YMMSL_Settings_contains(s1, 'key6')) + call assert_true(YMMSL_Settings_contains(s1, 'key7')) call assert_false(YMMSL_Settings_contains(s1, 'nokey')) call assert_eq_character(YMMSL_Settings_get_as_character(s1, 'key1'), 'value1') - call assert_eq_int8(YMMSL_Settings_get_as_int8(s1, 'key2'), 42424242424242_YMMSL_int8) - call assert_eq_logical(YMMSL_Settings_get_as_logical(s1, 'key3'), .false.) - call assert_eq_real8(YMMSL_Settings_get_as_real8(s1, 'key4'), 13.13d0) - call YMMSL_Settings_get_as_real8array(s1, 'key5', ra2) + call assert_eq_int4(YMMSL_Settings_get_as_int4(s1, 'key2'), 242424242_YMMSL_int4) + call assert_eq_int8(YMMSL_Settings_get_as_int8(s1, 'key2'), 242424242_YMMSL_int8) + call assert_eq_int8(YMMSL_Settings_get_as_int8(s1, 'key3'), 42424242424242_YMMSL_int8) + call assert_eq_logical(YMMSL_Settings_get_as_logical(s1, 'key4'), .false.) + call assert_eq_real8(YMMSL_Settings_get_as_real8(s1, 'key5'), 13.13d0) + call YMMSL_Settings_get_as_real8array(s1, 'key6', ra2) call assert_eq_real8array(ra2, ra1) - call YMMSL_Settings_get_as_real8array2(s1, 'key6', ra4) + call YMMSL_Settings_get_as_real8array2(s1, 'key7', ra4) call assert_eq_real8array2(ra4, ra3) call YMMSL_Settings_free(s1) @@ -109,18 +113,22 @@ subroutine test_settings_is_a ra2 = reshape((/1.0d0, 2.0d0, 3.0d0, 4.0d0, 5.0d0, 6.0d0/), (/3, 2/)) call YMMSL_Settings_set(s1, 'key1', 'value1') - call YMMSL_Settings_set(s1, 'key2', 42424242424242_YMMSL_int8) - call YMMSL_Settings_set(s1, 'key3', .false.) - call YMMSL_Settings_set(s1, 'key4', 13.13d0) - call YMMSL_Settings_set(s1, 'key5', ra1) - call YMMSL_Settings_set(s1, 'key6', ra2) + call YMMSL_Settings_set(s1, 'key2', 242424242_YMMSL_int4) + call YMMSL_Settings_set(s1, 'key3', 42424242424242_YMMSL_int8) + call YMMSL_Settings_set(s1, 'key4', .false.) + call YMMSL_Settings_set(s1, 'key5', 13.13d0) + call YMMSL_Settings_set(s1, 'key6', ra1) + call YMMSL_Settings_set(s1, 'key7', ra2) call assert_true(YMMSL_Settings_is_a_character(s1, 'key1')) + call assert_true(YMMSL_Settings_is_a_int4(s1, 'key2')) call assert_true(YMMSL_Settings_is_a_int8(s1, 'key2')) - call assert_true(YMMSL_Settings_is_a_logical(s1, 'key3')) - call assert_true(YMMSL_Settings_is_a_real8(s1, 'key4')) - call assert_true(YMMSL_Settings_is_a_real8array(s1, 'key5')) - call assert_true(YMMSL_Settings_is_a_real8array2(s1, 'key6')) + call assert_false(YMMSL_Settings_is_a_int4(s1, 'key3')) + call assert_true(YMMSL_Settings_is_a_int8(s1, 'key3')) + call assert_true(YMMSL_Settings_is_a_logical(s1, 'key4')) + call assert_true(YMMSL_Settings_is_a_real8(s1, 'key5')) + call assert_true(YMMSL_Settings_is_a_real8array(s1, 'key6')) + call assert_true(YMMSL_Settings_is_a_real8array2(s1, 'key7')) call assert_false(YMMSL_Settings_is_a_int8(s1, 'key1')) call assert_false(YMMSL_Settings_is_a_logical(s1, 'key1')) diff --git a/libmuscle/fortran/src/ymmsl/ymmsl.f03 b/libmuscle/fortran/src/ymmsl/ymmsl.f03 index cdeec328..07e0db91 100644 --- a/libmuscle/fortran/src/ymmsl/ymmsl.f03 +++ b/libmuscle/fortran/src/ymmsl/ymmsl.f03 @@ -41,12 +41,14 @@ module ymmsl public :: YMMSL_Settings_size public :: YMMSL_Settings_empty public :: YMMSL_Settings_is_a_character + public :: YMMSL_Settings_is_a_int4 public :: YMMSL_Settings_is_a_int8 public :: YMMSL_Settings_is_a_real8 public :: YMMSL_Settings_is_a_logical public :: YMMSL_Settings_is_a_real8array public :: YMMSL_Settings_is_a_real8array2 public :: YMMSL_Settings_set_character + public :: YMMSL_Settings_set_int4 public :: YMMSL_Settings_set_int8 public :: YMMSL_Settings_set_real8 public :: YMMSL_Settings_set_logical @@ -54,6 +56,7 @@ module ymmsl public :: YMMSL_Settings_set_real8array2 public :: YMMSL_Settings_set public :: YMMSL_Settings_get_as_character + public :: YMMSL_Settings_get_as_int4 public :: YMMSL_Settings_get_as_int8 public :: YMMSL_Settings_get_as_real8 public :: YMMSL_Settings_get_as_logical @@ -121,6 +124,24 @@ logical (c_bool) function YMMSL_Settings_is_a_character_( & integer (c_size_t), intent(out) :: err_msg_len end function YMMSL_Settings_is_a_character_ + logical (c_bool) function YMMSL_Settings_is_a_int4_( & + self, & + key, & + key_size, & + err_code, & + err_msg, & + err_msg_len) & + bind(C, name="YMMSL_Settings_is_a_int4_") + + use iso_c_binding + integer (c_intptr_t), value, intent(in) :: self + character, intent(in) :: key + integer (c_size_t), value, intent(in) :: key_size + integer (c_int), intent(out) :: err_code + type (c_ptr), intent(out) :: err_msg + integer (c_size_t), intent(out) :: err_msg_len + end function YMMSL_Settings_is_a_int4_ + logical (c_bool) function YMMSL_Settings_is_a_int8_( & self, & key, & @@ -227,6 +248,20 @@ subroutine YMMSL_Settings_set_character_( & integer (c_size_t), value, intent(in) :: value_size end subroutine YMMSL_Settings_set_character_ + subroutine YMMSL_Settings_set_int4_( & + self, & + key, & + key_size, & + value) & + bind(C, name="YMMSL_Settings_set_int4_") + + use iso_c_binding + integer (c_intptr_t), value, intent(in) :: self + character, intent(in) :: key + integer (c_size_t), value, intent(in) :: key_size + integer (c_int32_t), value, intent(in) :: value + end subroutine YMMSL_Settings_set_int4_ + subroutine YMMSL_Settings_set_int8_( & self, & key, & @@ -323,6 +358,24 @@ subroutine YMMSL_Settings_get_as_character_( & integer (c_size_t), intent(out) :: err_msg_len end subroutine YMMSL_Settings_get_as_character_ + integer (c_int32_t) function YMMSL_Settings_get_as_int4_( & + self, & + key, & + key_size, & + err_code, & + err_msg, & + err_msg_len) & + bind(C, name="YMMSL_Settings_get_as_int4_") + + use iso_c_binding + integer (c_intptr_t), value, intent(in) :: self + character, intent(in) :: key + integer (c_size_t), value, intent(in) :: key_size + integer (c_int), intent(out) :: err_code + type (c_ptr), intent(out) :: err_msg + integer (c_size_t), intent(out) :: err_msg_len + end function YMMSL_Settings_get_as_int4_ + integer (c_int64_t) function YMMSL_Settings_get_as_int8_( & self, & key, & @@ -477,6 +530,7 @@ end subroutine YMMSL_Settings_key_ interface YMMSL_Settings_set module procedure & YMMSL_Settings_set_character, & + YMMSL_Settings_set_int4, & YMMSL_Settings_set_int8, & YMMSL_Settings_set_real8, & YMMSL_Settings_set_logical, & @@ -609,6 +663,62 @@ function YMMSL_Settings_is_a_character( & YMMSL_Settings_is_a_character = ret_val end function YMMSL_Settings_is_a_character + function YMMSL_Settings_is_a_int4( & + self, & + key, & + err_code, & + err_msg) + implicit none + type(YMMSL_Settings), intent(in) :: self + character (len=*), intent(in) :: key + integer, optional, intent(out) :: err_code + character(:), allocatable, optional, intent(out) :: err_msg + logical :: YMMSL_Settings_is_a_int4 + + logical (c_bool) :: ret_val + integer (c_int) :: err_code_v + type (c_ptr) :: err_msg_v + integer (c_size_t) :: err_msg_len_v + character (c_char), dimension(:), pointer :: err_msg_f + character(:), allocatable :: err_msg_p + integer (c_size_t) :: err_msg_i + + ret_val = YMMSL_Settings_is_a_int4_( & + self%ptr, & + key, int(len(key), c_size_t), & + err_code_v, & + err_msg_v, & + err_msg_len_v) + + if (err_code_v .ne. 0) then + if (present(err_code)) then + err_code = err_code_v + if (present(err_msg)) then + call c_f_pointer(err_msg_v, err_msg_f, (/err_msg_len_v/)) + allocate (character(err_msg_len_v) :: err_msg) + do err_msg_i = 1, err_msg_len_v + err_msg(err_msg_i:err_msg_i) = err_msg_f(err_msg_i) + end do + end if + return + else + call c_f_pointer(err_msg_v, err_msg_f, (/err_msg_len_v/)) + allocate (character(err_msg_len_v) :: err_msg_p) + do err_msg_i = 1, err_msg_len_v + err_msg_p(err_msg_i:err_msg_i) = err_msg_f(err_msg_i) + end do + print *, err_msg_p + stop + end if + else + if (present(err_code)) then + err_code = 0 + end if + end if + + YMMSL_Settings_is_a_int4 = ret_val + end function YMMSL_Settings_is_a_int4 + function YMMSL_Settings_is_a_int8( & self, & key, & @@ -904,6 +1014,21 @@ subroutine YMMSL_Settings_set_character( & value, int(len(value), c_size_t)) end subroutine YMMSL_Settings_set_character + subroutine YMMSL_Settings_set_int4( & + self, & + key, & + value) + implicit none + type(YMMSL_Settings), intent(in) :: self + character (len=*), intent(in) :: key + integer (YMMSL_int4), intent(in) :: value + + call YMMSL_Settings_set_int4_( & + self%ptr, & + key, int(len(key), c_size_t), & + value) + end subroutine YMMSL_Settings_set_int4 + subroutine YMMSL_Settings_set_int8( & self, & key, & @@ -1044,6 +1169,61 @@ function YMMSL_Settings_get_as_character( & end do end function YMMSL_Settings_get_as_character + function YMMSL_Settings_get_as_int4( & + self, & + key, & + err_code, & + err_msg) + implicit none + type(YMMSL_Settings), intent(in) :: self + character (len=*), intent(in) :: key + integer, optional, intent(out) :: err_code + character(:), allocatable, optional, intent(out) :: err_msg + integer (YMMSL_int4) :: YMMSL_Settings_get_as_int4 + + integer (c_int32_t) :: ret_val + integer (c_int) :: err_code_v + type (c_ptr) :: err_msg_v + integer (c_size_t) :: err_msg_len_v + character (c_char), dimension(:), pointer :: err_msg_f + character(:), allocatable :: err_msg_p + integer (c_size_t) :: err_msg_i + + ret_val = YMMSL_Settings_get_as_int4_( & + self%ptr, & + key, int(len(key), c_size_t), & + err_code_v, & + err_msg_v, & + err_msg_len_v) + if (err_code_v .ne. 0) then + if (present(err_code)) then + err_code = err_code_v + if (present(err_msg)) then + call c_f_pointer(err_msg_v, err_msg_f, (/err_msg_len_v/)) + allocate (character(err_msg_len_v) :: err_msg) + do err_msg_i = 1, err_msg_len_v + err_msg(err_msg_i:err_msg_i) = err_msg_f(err_msg_i) + end do + end if + return + else + call c_f_pointer(err_msg_v, err_msg_f, (/err_msg_len_v/)) + allocate (character(err_msg_len_v) :: err_msg_p) + do err_msg_i = 1, err_msg_len_v + err_msg_p(err_msg_i:err_msg_i) = err_msg_f(err_msg_i) + end do + print *, err_msg_p + stop + end if + else + if (present(err_code)) then + err_code = 0 + end if + end if + + YMMSL_Settings_get_as_int4 = ret_val + end function YMMSL_Settings_get_as_int4 + function YMMSL_Settings_get_as_int8( & self, & key, & diff --git a/scripts/make_ymmsl_api.py b/scripts/make_ymmsl_api.py index 07d50df4..9fea59e6 100755 --- a/scripts/make_ymmsl_api.py +++ b/scripts/make_ymmsl_api.py @@ -32,21 +32,22 @@ MemFun(Sizet('size'), 'size'), MemFun(Bool('empty'), 'empty'), MemFunTmpl( - [String(), Int64t(), Double(), Bool(), VecDbl(), Vec2Dbl()], + [String(), Int32t(), Int64t(), Double(), Bool(), VecDbl(), Vec2Dbl()], Bool(), 'is_a', [String('key')], True, cpp_chain_call=lambda **kwargs: 'self_p->at({}).is_a<{}>()'.format( kwargs['cpp_args'], kwargs['tpl_type'])), IndexAssignmentOperator('set_character', [String('key'), String('value')]), + IndexAssignmentOperator('set_int4', [String('key'), Int32t('value')]), IndexAssignmentOperator('set_int8', [String('key'), Int64t('value')]), IndexAssignmentOperator('set_real8', [String('key'), Double('value')]), IndexAssignmentOperator('set_logical', [String('key'), Bool('value')]), IndexAssignmentOperator('set_real8array', [String('key'), VecDbl('value')]), IndexAssignmentOperator('set_real8array2', [String('key'), Vec2Dbl('value')]), OverloadSet('set', [ - 'set_character', 'set_int8', 'set_real8', 'set_logical', + 'set_character', 'set_int4', 'set_int8', 'set_real8', 'set_logical', 'set_real8array', 'set_real8array2']), MemFunTmpl( - [String(), Int64t(), Double(), Bool(), VecDbl('value'), + [String(), Int32t(), Int64t(), Double(), Bool(), VecDbl('value'), Vec2Dbl('value') ], T(), 'get_as', [String('key')], True, From 79d5acdc0b3a4347952a5fe5dac842ee1eca37b7 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Fri, 22 May 2020 16:18:32 +0200 Subject: [PATCH 05/36] Allow int-valued parameters to be read as double --- docs/source/fortran_api.rst | 8 +++++++- libmuscle/cpp/src/libmuscle/tests/test_instance.cpp | 4 ++++ libmuscle/cpp/src/ymmsl/settings.hpp | 4 +++- libmuscle/cpp/src/ymmsl/settings.tpp | 4 +++- libmuscle/fortran/src/libmuscle/tests/test_settings.f03 | 1 + 5 files changed, 18 insertions(+), 3 deletions(-) diff --git a/docs/source/fortran_api.rst b/docs/source/fortran_api.rst index 288dd3e5..345ac43f 100644 --- a/docs/source/fortran_api.rst +++ b/docs/source/fortran_api.rst @@ -2336,6 +2336,9 @@ YMMSL_Settings Return whether a value is of type ``YMMSL_real8``. + This will also return ``.true.`` if the value is an integer, even if + converting it would lose precision. + If the given key does not exist, then ``err_code`` will be set to ``YMMSL_out_of_bounds`` and the result will be invalid. @@ -2450,7 +2453,10 @@ YMMSL_Settings Return the value of a real-typed setting. - If this setting is not currently set to a real-typed value, + This will also work if the setting is integer-typed in which case it + will be converted, with possible loss of precision. + + If this setting is not currently set to a real- or integer-typed value, then ``err_code`` will be set to ``YMMSL_bad_cast`` and the result will be invalid. diff --git a/libmuscle/cpp/src/libmuscle/tests/test_instance.cpp b/libmuscle/cpp/src/libmuscle/tests/test_instance.cpp index 65dce00b..d6537296 100644 --- a/libmuscle/cpp/src/libmuscle/tests/test_instance.cpp +++ b/libmuscle/cpp/src/libmuscle/tests/test_instance.cpp @@ -171,6 +171,10 @@ TEST(libmuscle_instance, get_setting) { ASSERT_EQ(static_cast(instance.get_setting_as("test3")), 10); ASSERT_THROW(instance.get_setting_as("test4"), std::bad_cast); + ASSERT_EQ(instance.get_setting_as("test5"), 10.0); + ASSERT_EQ(instance.get_setting_as("test6"), 1.0f / 3.0f); + ASSERT_EQ(instance.get_setting_as("test3"), 10.0); + ASSERT_THROW(instance.get_setting("testx"), std::out_of_range); ASSERT_THROW(instance.get_setting_as("test1"), std::bad_cast); } diff --git a/libmuscle/cpp/src/ymmsl/settings.hpp b/libmuscle/cpp/src/ymmsl/settings.hpp index 4c233ef6..f2fe42d1 100644 --- a/libmuscle/cpp/src/ymmsl/settings.hpp +++ b/libmuscle/cpp/src/ymmsl/settings.hpp @@ -130,7 +130,9 @@ class SettingValue { /** Return whether this SettingValue holds a value of the given type. * * Note that for int32_t, this function will return true only if the - * value is integer and fits in an int32_t. + * value is integer and fits in an int32_t. For double, it will return + * true if the value is integer, even if converting it to a double + * would reduce precision. * * Since int and long are usually equivalent to int32_t or int64_t, * you can use those values too. diff --git a/libmuscle/cpp/src/ymmsl/settings.tpp b/libmuscle/cpp/src/ymmsl/settings.tpp index 733f214f..99e6b5a0 100644 --- a/libmuscle/cpp/src/ymmsl/settings.tpp +++ b/libmuscle/cpp/src/ymmsl/settings.tpp @@ -27,7 +27,7 @@ inline bool SettingValue::is_a() const { template<> inline bool SettingValue::is_a() const { - return type_ == Type_::FLOAT; + return (type_ == Type_::FLOAT) || (type_ == Type_::INT); } template<> @@ -76,6 +76,8 @@ template<> inline double SettingValue::as() const { if (!is_a()) throw std::bad_cast(); + if (is_a()) + return static_cast(int_value_); return float_value_; } diff --git a/libmuscle/fortran/src/libmuscle/tests/test_settings.f03 b/libmuscle/fortran/src/libmuscle/tests/test_settings.f03 index 375c2a4c..38919fdb 100644 --- a/libmuscle/fortran/src/libmuscle/tests/test_settings.f03 +++ b/libmuscle/fortran/src/libmuscle/tests/test_settings.f03 @@ -88,6 +88,7 @@ subroutine test_settings_set_get_as call assert_eq_int8(YMMSL_Settings_get_as_int8(s1, 'key3'), 42424242424242_YMMSL_int8) call assert_eq_logical(YMMSL_Settings_get_as_logical(s1, 'key4'), .false.) call assert_eq_real8(YMMSL_Settings_get_as_real8(s1, 'key5'), 13.13d0) + call assert_eq_real8(YMMSL_Settings_get_as_real8(s1, 'key2'), 242424242d0) call YMMSL_Settings_get_as_real8array(s1, 'key6', ra2) call assert_eq_real8array(ra2, ra1) call YMMSL_Settings_get_as_real8array2(s1, 'key7', ra4) From 4c113ff7b638e06d5d3aab41a80f73e0454f2818 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Fri, 22 May 2020 16:19:40 +0200 Subject: [PATCH 06/36] Try to make build test run on push to release branches --- .github/workflows/ci_ubuntu16.04.yaml | 3 +++ .github/workflows/ci_ubuntu18.04.yaml | 3 +++ .github/workflows/ci_ubuntu19.10.yaml | 3 +++ .github/workflows/ci_ubuntu20.04.yaml | 3 +++ 4 files changed, 12 insertions(+) diff --git a/.github/workflows/ci_ubuntu16.04.yaml b/.github/workflows/ci_ubuntu16.04.yaml index 1b29991f..161a4787 100644 --- a/.github/workflows/ci_ubuntu16.04.yaml +++ b/.github/workflows/ci_ubuntu16.04.yaml @@ -4,6 +4,9 @@ name: native_compatibility_ubuntu16.04 on: schedule: - cron: '0 1 * * 0' + push: + branches: + - 'release-*' jobs: build: runs-on: ubuntu-latest diff --git a/.github/workflows/ci_ubuntu18.04.yaml b/.github/workflows/ci_ubuntu18.04.yaml index 86c79e8c..f22c54c2 100644 --- a/.github/workflows/ci_ubuntu18.04.yaml +++ b/.github/workflows/ci_ubuntu18.04.yaml @@ -4,6 +4,9 @@ name: native_compatibility_ubuntu18.04 on: schedule: - cron: '0 2 * * 0' + push: + branches: + - 'release-*' jobs: build: runs-on: ubuntu-latest diff --git a/.github/workflows/ci_ubuntu19.10.yaml b/.github/workflows/ci_ubuntu19.10.yaml index 6f220106..e1f7913b 100644 --- a/.github/workflows/ci_ubuntu19.10.yaml +++ b/.github/workflows/ci_ubuntu19.10.yaml @@ -4,6 +4,9 @@ name: native_compatibility_ubuntu19.10 on: schedule: - cron: '0 3 * * 0' + push: + branches: + - 'release-*' jobs: build: runs-on: ubuntu-latest diff --git a/.github/workflows/ci_ubuntu20.04.yaml b/.github/workflows/ci_ubuntu20.04.yaml index c46af0f9..449543d3 100644 --- a/.github/workflows/ci_ubuntu20.04.yaml +++ b/.github/workflows/ci_ubuntu20.04.yaml @@ -4,6 +4,9 @@ name: native_compatibility_ubuntu20.04 on: schedule: - cron: '0 4 * * 0' + push: + branches: + - 'release-*' jobs: build: runs-on: ubuntu-latest From 35dc71d286f5e4a6733f72989b1734988a0da9c5 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Fri, 22 May 2020 16:36:24 +0200 Subject: [PATCH 07/36] Update release procedure --- docs/source/releasing.rst | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/docs/source/releasing.rst b/docs/source/releasing.rst index eada37b8..f4fbd66c 100644 --- a/docs/source/releasing.rst +++ b/docs/source/releasing.rst @@ -55,7 +55,7 @@ shows up: .. code-block:: bash - python setup.py build_sphinx + make docs It may give some warnings about missing references, that's a known issue and normally harmless. Next, point your web browser to @@ -67,8 +67,8 @@ Run tests --------- Before we make a commit, the tests should be run, and this is a good idea anyway -if we're making a release. So run ``python setup.py test`` and check that -everything is in order. +if we're making a release. So run ``make test`` and check that everything is in +order. Commit the version update ------------------------- @@ -136,8 +136,13 @@ can start using it. To build, use: .. code-block:: bash + rm -r ./build python3 setup.py sdist bdist_wheel +Note that we remove ``./build``, which is the build directory setuptools uses, +to ensure that we're doing a clean build, I've seen some weird mixes of versions +on occasion so it's better to be safe than sorry. + We can then check to see if everything is okay using .. code-block:: bash @@ -150,6 +155,13 @@ and if all seems well, we can upload to PyPI: twine upload dist/muscle3-x.y.z* +Announce release +---------------- + +Announce the release in the usual places, so that people know it exists. There +should be a short release message listing new features and fixed bugs, and don't +forget to thank everyone who contributed! + Merge the release branch back into develop ------------------------------------------ From 281c75b8907da5b2d5150c36472705e0916cc2bd Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Tue, 30 Jun 2020 22:21:43 +0200 Subject: [PATCH 08/36] Fix use-after-free in DCR::elements() (#62) --- libmuscle/cpp/src/libmuscle/data.cpp | 8 ++++++-- libmuscle/cpp/src/libmuscle/data.hpp | 3 +++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/libmuscle/cpp/src/libmuscle/data.cpp b/libmuscle/cpp/src/libmuscle/data.cpp index 4637d380..c7b0245c 100644 --- a/libmuscle/cpp/src/libmuscle/data.cpp +++ b/libmuscle/cpp/src/libmuscle/data.cpp @@ -310,6 +310,7 @@ DataConstRef::DataConstRef(Settings const & settings) void DataConstRef::reseat(DataConstRef const & target) { mp_zones_ = target.mp_zones_; mp_obj_ = target.mp_obj_; + obj_cache_ = target.obj_cache_; } template <> @@ -718,8 +719,10 @@ DataConstRef DataConstRef::grid_dict_() const { if (oh.get().type != msgpack::type::MAP) throw std::runtime_error("Invalid grid format. Bug in MUSCLE 3?"); - auto zone = std::make_shared(); - return DataConstRef(mcp::unpack_data(zone, ext.data(), ext.size())); + if (!obj_cache_) + obj_cache_ = std::make_shared( + mcp::unpack_data(mp_zones_->at(0), ext.data(), ext.size())); + return *obj_cache_; } /* This is here in the .cpp and instantiated explicitly, because it requires the @@ -853,6 +856,7 @@ Data & Data::operator=(Data const & rhs) { if (mp_zones_ != rhs.mp_zones_) mp_zones_->insert(mp_zones_->end(), rhs.mp_zones_->cbegin(), rhs.mp_zones_->cend()); + obj_cache_ = rhs.obj_cache_; } return *this; } diff --git a/libmuscle/cpp/src/libmuscle/data.hpp b/libmuscle/cpp/src/libmuscle/data.hpp index 02b83321..c4cd02c7 100644 --- a/libmuscle/cpp/src/libmuscle/data.hpp +++ b/libmuscle/cpp/src/libmuscle/data.hpp @@ -462,6 +462,9 @@ class DataConstRef { Zones_ mp_zones_; msgpack::object * mp_obj_; + // cache for extracted complex object, e.g. Settings, Grid + mutable std::shared_ptr obj_cache_; + // create DCR pointing to the given object and sharing the given zone DataConstRef( msgpack::object * data, From 9917f643fc739a31bb0cde852e7d44a5729a6c07 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Tue, 30 Jun 2020 22:25:59 +0200 Subject: [PATCH 09/36] Use valgrind when running tests if available (#62) --- .github/workflows/ci.yaml | 2 +- .github/workflows/ci_ubuntu16.04.yaml | 2 +- .github/workflows/ci_ubuntu18.04.yaml | 2 +- .github/workflows/ci_ubuntu19.10.yaml | 2 +- .github/workflows/ci_ubuntu20.04.yaml | 2 +- libmuscle/cpp/build/check_tools.make | 20 ++++++++++++++++++++ libmuscle/cpp/build/libmuscle/tests/Makefile | 2 +- 7 files changed, 26 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 80ba2e99..4c6f136d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -15,7 +15,7 @@ jobs: - name: Install dependencies run: | - sudo apt-get install -y build-essential cmake gfortran libopenmpi-dev pkg-config wget + sudo apt-get install -y build-essential cmake gfortran libopenmpi-dev pkg-config wget valgrind sudo apt-get install -y libssl-dev zlib1g-dev pip install ymmsl==0.10.1 diff --git a/.github/workflows/ci_ubuntu16.04.yaml b/.github/workflows/ci_ubuntu16.04.yaml index 161a4787..7d783789 100644 --- a/.github/workflows/ci_ubuntu16.04.yaml +++ b/.github/workflows/ci_ubuntu16.04.yaml @@ -15,4 +15,4 @@ jobs: - uses: actions/checkout@v2 - name: Run tests on Ubuntu 16.04 - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:16.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install ymmsl==0.10.1 && make test_examples' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:16.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install ymmsl==0.10.1 && make test_examples' diff --git a/.github/workflows/ci_ubuntu18.04.yaml b/.github/workflows/ci_ubuntu18.04.yaml index f22c54c2..3d9b313f 100644 --- a/.github/workflows/ci_ubuntu18.04.yaml +++ b/.github/workflows/ci_ubuntu18.04.yaml @@ -15,4 +15,4 @@ jobs: - uses: actions/checkout@v2 - name: Run tests on Ubuntu 18.04 - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:18.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install ymmsl==0.10.1 && make test_examples' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:18.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install ymmsl==0.10.1 && make test_examples' diff --git a/.github/workflows/ci_ubuntu19.10.yaml b/.github/workflows/ci_ubuntu19.10.yaml index e1f7913b..560d5284 100644 --- a/.github/workflows/ci_ubuntu19.10.yaml +++ b/.github/workflows/ci_ubuntu19.10.yaml @@ -15,4 +15,4 @@ jobs: - uses: actions/checkout@v2 - name: Run tests on Ubuntu 19.10 - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:19.10 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install ymmsl==0.10.1 && make test_examples' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:19.10 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install ymmsl==0.10.1 && make test_examples' diff --git a/.github/workflows/ci_ubuntu20.04.yaml b/.github/workflows/ci_ubuntu20.04.yaml index 449543d3..ea711557 100644 --- a/.github/workflows/ci_ubuntu20.04.yaml +++ b/.github/workflows/ci_ubuntu20.04.yaml @@ -15,4 +15,4 @@ jobs: - uses: actions/checkout@v2 - name: Run tests on Ubuntu 20.04 - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:20.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install ymmsl==0.10.1 && make test_examples' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:20.04 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install ymmsl==0.10.1 && make test_examples' diff --git a/libmuscle/cpp/build/check_tools.make b/libmuscle/cpp/build/check_tools.make index 892f4dde..7a0dc965 100644 --- a/libmuscle/cpp/build/check_tools.make +++ b/libmuscle/cpp/build/check_tools.make @@ -84,6 +84,26 @@ else $(info - Will extract archives using $(TAR).) endif +# Check for valgrind (for testing for memory leaks) +$(info ) +$(info Looking for valgrind...) +tool_var := VALGRIND +include $(TOOLDIR)/check_override.make + +tool_command := valgrind +include $(TOOLDIR)/detect_tool.make + +ifeq ($(VALGRIND), valgrind) + export VALGRIND := valgrind --leak-check=full --error-exitcode=1 +endif + +ifndef VALGRIND + $(warning - Could not find valgrind, so tests will run without it.) + $(warning - To fix this, install valgrind and if necessary set VALGRIND to point to it.) +else + $(info - Will check for leaks using $(VALGRIND).) +endif + # Check number of cores ifndef NCORES NCORES := $(shell nproc 2>/dev/null || echo 2) diff --git a/libmuscle/cpp/build/libmuscle/tests/Makefile b/libmuscle/cpp/build/libmuscle/tests/Makefile index 7d6a322f..e285659a 100644 --- a/libmuscle/cpp/build/libmuscle/tests/Makefile +++ b/libmuscle/cpp/build/libmuscle/tests/Makefile @@ -93,5 +93,5 @@ test_dep_lib_paths := $(subst $(space),:,$(foreach DIR,$(DEP_DIRS),$(DIR)/lib)) .PHONY: run_test% run_test%: test% - export LD_LIBRARY_PATH=$(test_dep_lib_paths) ; ./$< + export LD_LIBRARY_PATH=$(test_dep_lib_paths) ; $(VALGRIND) ./$< From ad66691f63e32ba3c4788f5f88973c9844c33045 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Wed, 1 Jul 2020 08:53:00 +0200 Subject: [PATCH 10/36] Try to fix Python 3.5 CI (#64) --- .github/workflows/ci_python3.5.1.yaml | 4 ++-- .github/workflows/ci_python3.5.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci_python3.5.1.yaml b/.github/workflows/ci_python3.5.1.yaml index 57799ba5..ca80540e 100644 --- a/.github/workflows/ci_python3.5.1.yaml +++ b/.github/workflows/ci_python3.5.1.yaml @@ -15,5 +15,5 @@ jobs: path: ${{ github.workspace }}/.eggs key: python-compatibility-3.5.1-eggs - - name: Run Python tests on 3.5.1 latest - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.5.1 /bin/bash -c 'cd /home/muscle3 && pip install -U pip setuptools wheel ymmsl==0.10.1 && make test_python_only' + - name: Run Python tests on 3.5.1 + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.5.1 /bin/bash -c 'apt-get update && apt-get -y install python3-numpy && cd /home/muscle3 && pip install -U pip setuptools wheel ymmsl==0.10.1 && make test_python_only' diff --git a/.github/workflows/ci_python3.5.yaml b/.github/workflows/ci_python3.5.yaml index bb4232eb..d87ddca4 100644 --- a/.github/workflows/ci_python3.5.yaml +++ b/.github/workflows/ci_python3.5.yaml @@ -16,4 +16,4 @@ jobs: key: python-compatibility-3.5-eggs - name: Run Python tests on 3.5 latest - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.5 /bin/bash -c 'cd /home/muscle3 && pip install ymmsl==0.10.1 && make test_python_only' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.5 /bin/bash -c 'apt-get update && apt-get -y install python3-numpy && cd /home/muscle3 && pip install ymmsl==0.10.1 && make test_python_only' From 12ed45074884c4a0a7a19a62f6a57aa4749dbc65 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Wed, 1 Jul 2020 09:07:50 +0200 Subject: [PATCH 11/36] Try to fix Python 3.5 CI again (#64) --- .github/workflows/ci_python3.5.1.yaml | 2 +- .github/workflows/ci_python3.5.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_python3.5.1.yaml b/.github/workflows/ci_python3.5.1.yaml index ca80540e..88fb4e62 100644 --- a/.github/workflows/ci_python3.5.1.yaml +++ b/.github/workflows/ci_python3.5.1.yaml @@ -16,4 +16,4 @@ jobs: key: python-compatibility-3.5.1-eggs - name: Run Python tests on 3.5.1 - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.5.1 /bin/bash -c 'apt-get update && apt-get -y install python3-numpy && cd /home/muscle3 && pip install -U pip setuptools wheel ymmsl==0.10.1 && make test_python_only' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.5.1 /bin/bash -c 'cd /home/muscle3 && pip install -U pip setuptools wheel numpy ymmsl==0.10.1 && make test_python_only' diff --git a/.github/workflows/ci_python3.5.yaml b/.github/workflows/ci_python3.5.yaml index d87ddca4..3d79cf5f 100644 --- a/.github/workflows/ci_python3.5.yaml +++ b/.github/workflows/ci_python3.5.yaml @@ -16,4 +16,4 @@ jobs: key: python-compatibility-3.5-eggs - name: Run Python tests on 3.5 latest - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.5 /bin/bash -c 'apt-get update && apt-get -y install python3-numpy && cd /home/muscle3 && pip install ymmsl==0.10.1 && make test_python_only' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.5 /bin/bash -c 'cd /home/muscle3 && pip install numpy ymmsl==0.10.1 && make test_python_only' From b8bd1cfa6131ad810cfb7f2d9fa932df50809d2c Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Wed, 1 Jul 2020 09:11:59 +0200 Subject: [PATCH 12/36] Try to fix Python 3.5.1 CI (#64) --- .github/workflows/ci_python3.5.1.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_python3.5.1.yaml b/.github/workflows/ci_python3.5.1.yaml index 88fb4e62..9f0ac8e2 100644 --- a/.github/workflows/ci_python3.5.1.yaml +++ b/.github/workflows/ci_python3.5.1.yaml @@ -16,4 +16,4 @@ jobs: key: python-compatibility-3.5.1-eggs - name: Run Python tests on 3.5.1 - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.5.1 /bin/bash -c 'cd /home/muscle3 && pip install -U pip setuptools wheel numpy ymmsl==0.10.1 && make test_python_only' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.5.1 /bin/bash -c 'cd /home/muscle3 && pip install -U pip setuptools wheel numpy<1.19 ymmsl==0.10.1 && make test_python_only' From de36b987e6355431406ed23c2b4276734c8230e3 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Wed, 1 Jul 2020 09:17:09 +0200 Subject: [PATCH 13/36] Add quotes to Python 3.5.1 CI fix (#64) --- .github/workflows/ci_python3.5.1.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_python3.5.1.yaml b/.github/workflows/ci_python3.5.1.yaml index 9f0ac8e2..414110fb 100644 --- a/.github/workflows/ci_python3.5.1.yaml +++ b/.github/workflows/ci_python3.5.1.yaml @@ -16,4 +16,4 @@ jobs: key: python-compatibility-3.5.1-eggs - name: Run Python tests on 3.5.1 - run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.5.1 /bin/bash -c 'cd /home/muscle3 && pip install -U pip setuptools wheel numpy<1.19 ymmsl==0.10.1 && make test_python_only' + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" python:3.5.1 /bin/bash -c 'cd /home/muscle3 && pip install -U pip setuptools wheel "numpy<1.19" ymmsl==0.10.1 && make test_python_only' From 578c98ec0584e3b520d42c51d184343b87217b2d Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Fri, 17 Jul 2020 22:50:46 +0200 Subject: [PATCH 14/36] Switch to pytest-flake8 and fix newly found issues --- integration_test/conftest.py | 2 +- integration_test/test_all.py | 5 +--- integration_test/test_cpp_mmp_client.py | 2 -- integration_test/test_cpp_tcp_client.py | 3 --- integration_test/test_cpp_tcp_server.py | 3 --- integration_test/test_duplication_mapper.py | 7 ++---- integration_test/test_logging.py | 1 - integration_test/test_parameter_overlays.py | 3 +-- libmuscle/python/libmuscle/__init__.py | 3 --- libmuscle/python/libmuscle/communicator.py | 15 ++++++------ libmuscle/python/libmuscle/instance.py | 24 +++++++++---------- libmuscle/python/libmuscle/logging.py | 4 ---- .../libmuscle/manager/instance_registry.py | 2 +- libmuscle/python/libmuscle/manager/logger.py | 1 - .../python/libmuscle/manager/mmp_server.py | 6 ++--- .../python/libmuscle/manager/test/conftest.py | 4 ++-- .../libmuscle/manager/test/test_logger.py | 1 - .../manager/test/test_mmp_servicer.py | 2 +- .../python/libmuscle/mcp/pipe_multiplexer.py | 2 +- libmuscle/python/libmuscle/mcp/server.py | 2 -- libmuscle/python/libmuscle/mcp/tcp_client.py | 1 - libmuscle/python/libmuscle/mcp/tcp_server.py | 1 - .../mcp/test/test_direct_communication.py | 4 ---- .../libmuscle/mcp/test/test_direct_server.py | 7 +----- .../mcp/test/test_pipe_communication.py | 2 -- .../mcp/test/test_tcp_communication.py | 4 ---- .../libmuscle/mcp/test/test_tcp_server.py | 7 ------ libmuscle/python/libmuscle/outbox.py | 3 --- libmuscle/python/libmuscle/port.py | 2 +- libmuscle/python/libmuscle/profiler.py | 7 +++--- libmuscle/python/libmuscle/profiling.py | 12 ++++------ .../libmuscle/test/test_communicator.py | 1 - libmuscle/python/libmuscle/test/test_grid.py | 8 +++---- .../python/libmuscle/test/test_instance.py | 6 ++--- .../python/libmuscle/test/test_mmp_client.py | 2 +- setup.cfg | 4 ++-- setup.py | 1 - 37 files changed, 50 insertions(+), 114 deletions(-) diff --git a/integration_test/conftest.py b/integration_test/conftest.py index 46bac069..a64b0717 100644 --- a/integration_test/conftest.py +++ b/integration_test/conftest.py @@ -8,7 +8,7 @@ import yatiml import ymmsl -import integration_test.include_libmuscle +import integration_test.include_libmuscle # noqa: F401 from libmuscle.manager.instance_registry import InstanceRegistry from libmuscle.manager.logger import Logger diff --git a/integration_test/test_all.py b/integration_test/test_all.py index 780c0b32..f314396a 100644 --- a/integration_test/test_all.py +++ b/integration_test/test_all.py @@ -1,11 +1,8 @@ from collections import OrderedDict -import sys -from typing import List import numpy as np -import pytest from ymmsl import (ComputeElement, Conduit, Configuration, Model, Operator, - Reference, Settings) + Settings) from libmuscle import Grid, Instance, Message from libmuscle.runner import run_simulation diff --git a/integration_test/test_cpp_mmp_client.py b/integration_test/test_cpp_mmp_client.py index 29b31b1f..4cc60577 100644 --- a/integration_test/test_cpp_mmp_client.py +++ b/integration_test/test_cpp_mmp_client.py @@ -5,13 +5,11 @@ import ymmsl from ymmsl import Port, Reference -from libmuscle.logging import LogLevel, LogMessage, Timestamp from libmuscle.manager.instance_registry import InstanceRegistry from libmuscle.manager.logger import Logger from libmuscle.manager.mmp_server import MMPServer from libmuscle.manager.manager import elements_for_model from libmuscle.manager.topology_store import TopologyStore -from libmuscle.mmp_client import MMPClient from libmuscle.operator import Operator from .conftest import skip_if_python_only diff --git a/integration_test/test_cpp_tcp_client.py b/integration_test/test_cpp_tcp_client.py index 23c61f19..273b7f63 100644 --- a/integration_test/test_cpp_tcp_client.py +++ b/integration_test/test_cpp_tcp_client.py @@ -3,11 +3,8 @@ import subprocess from unittest.mock import MagicMock -import msgpack - from libmuscle.mcp.tcp_server import TcpServer from libmuscle.mcp.message import Message -from libmuscle.post_office import PostOffice from ymmsl import Reference, Settings diff --git a/integration_test/test_cpp_tcp_server.py b/integration_test/test_cpp_tcp_server.py index cd1a4e2f..d4a87f3a 100644 --- a/integration_test/test_cpp_tcp_server.py +++ b/integration_test/test_cpp_tcp_server.py @@ -1,8 +1,5 @@ from pathlib import Path import subprocess -from unittest.mock import MagicMock - -import msgpack from libmuscle.mcp.tcp_client import TcpClient from libmuscle.mcp.message import Message diff --git a/integration_test/test_duplication_mapper.py b/integration_test/test_duplication_mapper.py index dba5ffdf..36dbf3ba 100644 --- a/integration_test/test_duplication_mapper.py +++ b/integration_test/test_duplication_mapper.py @@ -1,8 +1,5 @@ -from typing import List - -import pytest -from ymmsl import (ComputeElement, Conduit, Configuration, Operator, Reference, - Model, Settings) +from ymmsl import (ComputeElement, Conduit, Configuration, Operator, Model, + Settings) from libmuscle import Instance, Message from libmuscle.runner import run_simulation diff --git a/integration_test/test_logging.py b/integration_test/test_logging.py index 4b4a3fc0..8a51ce99 100644 --- a/integration_test/test_logging.py +++ b/integration_test/test_logging.py @@ -9,7 +9,6 @@ from libmuscle.manager.manager import elements_for_model from libmuscle.manager.topology_store import TopologyStore from libmuscle.mmp_client import MMPClient -from libmuscle.operator import Operator def do_logging_test(caplog): diff --git a/integration_test/test_parameter_overlays.py b/integration_test/test_parameter_overlays.py index f0d9ef17..a4211cef 100644 --- a/integration_test/test_parameter_overlays.py +++ b/integration_test/test_parameter_overlays.py @@ -1,8 +1,7 @@ from collections import OrderedDict -import pytest from ymmsl import (ComputeElement, Conduit, Configuration, Model, Operator, - Reference, Settings) + Settings) from libmuscle import Instance, Message from libmuscle.runner import run_simulation diff --git a/libmuscle/python/libmuscle/__init__.py b/libmuscle/python/libmuscle/__init__.py index 4b5407ce..707dbdae 100644 --- a/libmuscle/python/libmuscle/__init__.py +++ b/libmuscle/python/libmuscle/__init__.py @@ -1,6 +1,3 @@ -import os -import pathlib - from libmuscle.communicator import Message from libmuscle.grid import Grid from libmuscle.instance import Instance diff --git a/libmuscle/python/libmuscle/communicator.py b/libmuscle/python/libmuscle/communicator.py index 5614a6cc..e9789ee7 100644 --- a/libmuscle/python/libmuscle/communicator.py +++ b/libmuscle/python/libmuscle/communicator.py @@ -1,5 +1,4 @@ -import msgpack -from typing import Any, Dict, List, Optional, Tuple, Union, cast +from typing import Any, Dict, List, Optional, Tuple, cast from ymmsl import Conduit, Identifier, Operator, Reference, Settings from libmuscle.endpoint import Endpoint @@ -11,7 +10,7 @@ from libmuscle.post_office import PostOffice from libmuscle.port import Port from libmuscle.profiler import Profiler -from libmuscle.profiling import ProfileEvent, ProfileEventType +from libmuscle.profiling import ProfileEventType MessageObject = Any @@ -35,7 +34,7 @@ class Message: # actually goes out on the wire, see libmuscle.mcp.Message for that. def __init__(self, timestamp: float, next_timestamp: Optional[float], data: MessageObject, - settings: Optional[Settings]=None + settings: Optional[Settings] = None ) -> None: """Create a Message. @@ -176,7 +175,7 @@ def get_port(self, port_name: str) -> Port: def send_message( self, port_name: str, message: Message, - slot: Optional[int]=None) -> None: + slot: Optional[int] = None) -> None: """Send a message and settings to the outside world. Sending is non-blocking, a copy of the message will be made @@ -226,8 +225,8 @@ def send_message( profile_event.port_length = port.get_length() profile_event.message_size = len(encoded_message) - def receive_message(self, port_name: str, slot: Optional[int]=None, - default: Optional[Message]=None + def receive_message(self, port_name: str, slot: Optional[int] = None, + default: Optional[Message] = None ) -> Message: """Receive a message and attached settings overlay. @@ -305,7 +304,7 @@ def receive_message(self, port_name: str, slot: Optional[int]=None, return message - def close_port(self, port_name: str, slot: Optional[int]=None + def close_port(self, port_name: str, slot: Optional[int] = None ) -> None: """Closes the given port. diff --git a/libmuscle/python/libmuscle/instance.py b/libmuscle/python/libmuscle/instance.py index 517ab936..1323d0a8 100644 --- a/libmuscle/python/libmuscle/instance.py +++ b/libmuscle/python/libmuscle/instance.py @@ -1,13 +1,11 @@ from copy import copy import logging -from pathlib import Path import sys -from typing import cast, Dict, List, Optional, Tuple, Union -from typing_extensions import Type +from typing import cast, Dict, List, Optional, Tuple import grpc -from ymmsl import (Conduit, Identifier, Operator, SettingValue, Port, - Reference, Settings) +from ymmsl import (Identifier, Operator, SettingValue, Port, Reference, + Settings) from libmuscle.communicator import Communicator, Message from libmuscle.settings_manager import SettingsManager @@ -15,7 +13,7 @@ from libmuscle.mcp.message import ClosePort from libmuscle.mmp_client import MMPClient from libmuscle.profiler import Profiler -from libmuscle.profiling import ProfileEvent, ProfileEventType +from libmuscle.profiling import ProfileEventType from libmuscle.util import extract_log_file_location @@ -25,7 +23,7 @@ class Instance: This class provides a low-level send/receive API for the instance to use. """ - def __init__(self, ports: Optional[Dict[Operator, List[str]]]=None + def __init__(self, ports: Optional[Dict[Operator, List[str]]] = None ) -> None: """Create an Instance. @@ -67,7 +65,7 @@ def __init__(self, ports: Optional[Dict[Operator, List[str]]]=None self._register() self._connect() - def reuse_instance(self, apply_overlay: bool=True) -> bool: + def reuse_instance(self, apply_overlay: bool = True) -> bool: """Decide whether to run this instance again. In a multiscale simulation, instances get reused all the time. @@ -253,7 +251,7 @@ def set_port_length(self, port: str, length: int) -> None: self._communicator.get_port(port).set_length(length) def send(self, port_name: str, message: Message, - slot: Optional[int]=None) -> None: + slot: Optional[int] = None) -> None: """Send a message to the outside world. Sending is non-blocking, a copy of the message will be made @@ -271,8 +269,8 @@ def send(self, port_name: str, message: Message, self._communicator.send_message(port_name, message, slot) - def receive(self, port_name: str, slot: Optional[int]=None, - default: Optional[Message]=None + def receive(self, port_name: str, slot: Optional[int] = None, + default: Optional[Message] = None ) -> Message: """Receive a message from the outside world. @@ -304,8 +302,8 @@ def receive(self, port_name: str, slot: Optional[int]=None, return self.__receive_message(port_name, slot, default, False) def receive_with_settings( - self, port_name: str, slot: Optional[int]=None, - default: Optional[Message]=None + self, port_name: str, slot: Optional[int] = None, + default: Optional[Message] = None ) -> Message: """Receive a message with attached settings overlay. diff --git a/libmuscle/python/libmuscle/logging.py b/libmuscle/python/libmuscle/logging.py index 917b0347..e40a6635 100644 --- a/libmuscle/python/libmuscle/logging.py +++ b/libmuscle/python/libmuscle/logging.py @@ -1,13 +1,9 @@ -import datetime from enum import Enum import logging from typing import Dict -from typing_extensions import NewType -from ymmsl import Operator import muscle_manager_protocol.muscle_manager_protocol_pb2 as mmp -from libmuscle.operator import operator_to_grpc from libmuscle.timestamp import Timestamp diff --git a/libmuscle/python/libmuscle/manager/instance_registry.py b/libmuscle/python/libmuscle/manager/instance_registry.py index d1d265fb..4cd9e47d 100644 --- a/libmuscle/python/libmuscle/manager/instance_registry.py +++ b/libmuscle/python/libmuscle/manager/instance_registry.py @@ -1,7 +1,7 @@ from enum import Enum from threading import Condition from typing import Dict # noqa -from typing import cast, List +from typing import List from ymmsl import Port, Reference diff --git a/libmuscle/python/libmuscle/manager/logger.py b/libmuscle/python/libmuscle/manager/logger.py index 3e2d6056..0c9b2411 100644 --- a/libmuscle/python/libmuscle/manager/logger.py +++ b/libmuscle/python/libmuscle/manager/logger.py @@ -1,7 +1,6 @@ import logging from libmuscle.logging import LogLevel, Timestamp -from libmuscle.operator import Operator from libmuscle.util import extract_log_file_location diff --git a/libmuscle/python/libmuscle/manager/mmp_server.py b/libmuscle/python/libmuscle/manager/mmp_server.py index fe65f7a6..14e9775f 100644 --- a/libmuscle/python/libmuscle/manager/mmp_server.py +++ b/libmuscle/python/libmuscle/manager/mmp_server.py @@ -1,5 +1,4 @@ from concurrent import futures -import logging import time import socket from typing import cast, Generator, List @@ -12,7 +11,6 @@ from libmuscle.manager.instance_registry import InstanceRegistry from libmuscle.manager.logger import Logger from libmuscle.manager.topology_store import TopologyStore -from libmuscle.operator import operator_from_grpc from libmuscle.util import (conduit_to_grpc, generate_indices, instance_indices, instance_to_kernel) @@ -136,7 +134,7 @@ def RegisterInstance( self.__log(LogLevel.INFO, 'Registered instance {}'.format( request.instance_name)) return mmp.RegistrationResult(status=mmp.RESULT_STATUS_SUCCESS) - except ValueError as e: + except ValueError: return mmp.RegistrationResult( status=mmp.RESULT_STATUS_ERROR, error_message=('An instance with name {} was already' @@ -197,7 +195,7 @@ def DeregisterInstance(self, request: mmp.DeregistrationRequest, self.__log(LogLevel.INFO, 'Deregistered instance {}'.format( request.instance_name)) return mmp.DeregistrationResult(status=mmp.RESULT_STATUS_SUCCESS) - except ValueError as e: + except ValueError: return mmp.DeregistrationResult( status=mmp.RESULT_STATUS_ERROR, error_message=('No instance with name {} was registered' diff --git a/libmuscle/python/libmuscle/manager/test/conftest.py b/libmuscle/python/libmuscle/manager/test/conftest.py index 2b0900e1..8d05a800 100644 --- a/libmuscle/python/libmuscle/manager/test/conftest.py +++ b/libmuscle/python/libmuscle/manager/test/conftest.py @@ -1,6 +1,6 @@ import pytest -from ymmsl import (ComputeElement, Conduit, Configuration, Identifier, Model, - Reference, Settings) +from ymmsl import (ComputeElement, Conduit, Configuration, Model, Reference, + Settings) from libmuscle.manager.instance_registry import InstanceRegistry from libmuscle.manager.logger import Logger diff --git a/libmuscle/python/libmuscle/manager/test/test_logger.py b/libmuscle/python/libmuscle/manager/test/test_logger.py index e734df0f..1c4f32ba 100644 --- a/libmuscle/python/libmuscle/manager/test/test_logger.py +++ b/libmuscle/python/libmuscle/manager/test/test_logger.py @@ -1,7 +1,6 @@ import logging from libmuscle.logging import LogLevel, Timestamp -from libmuscle.operator import Operator from libmuscle.manager.logger import Logger diff --git a/libmuscle/python/libmuscle/manager/test/test_mmp_servicer.py b/libmuscle/python/libmuscle/manager/test/test_mmp_servicer.py index 9c1854cb..08e1c91c 100644 --- a/libmuscle/python/libmuscle/manager/test/test_mmp_servicer.py +++ b/libmuscle/python/libmuscle/manager/test/test_mmp_servicer.py @@ -1,5 +1,5 @@ from google.protobuf.timestamp_pb2 import Timestamp -from ymmsl import Operator, Reference +from ymmsl import Operator from libmuscle.manager.mmp_server import MMPServicer import muscle_manager_protocol.muscle_manager_protocol_pb2 as mmp diff --git a/libmuscle/python/libmuscle/mcp/pipe_multiplexer.py b/libmuscle/python/libmuscle/mcp/pipe_multiplexer.py index 36b059b9..79e6b94a 100644 --- a/libmuscle/python/libmuscle/mcp/pipe_multiplexer.py +++ b/libmuscle/python/libmuscle/mcp/pipe_multiplexer.py @@ -3,7 +3,7 @@ # The below line seems to help avoid crashes, something to do with # a background thread in the library and forking threaded processes. from multiprocessing import resource_sharer # type: ignore -from typing import Dict, List, Tuple +from typing import Dict, List import uuid from ymmsl import Reference diff --git a/libmuscle/python/libmuscle/mcp/server.py b/libmuscle/python/libmuscle/mcp/server.py index 68060e11..6ee98af1 100644 --- a/libmuscle/python/libmuscle/mcp/server.py +++ b/libmuscle/python/libmuscle/mcp/server.py @@ -1,5 +1,3 @@ -from typing import Dict - from ymmsl import Reference from libmuscle.post_office import PostOffice diff --git a/libmuscle/python/libmuscle/mcp/tcp_client.py b/libmuscle/python/libmuscle/mcp/tcp_client.py index db38290c..7c535c56 100644 --- a/libmuscle/python/libmuscle/mcp/tcp_client.py +++ b/libmuscle/python/libmuscle/mcp/tcp_client.py @@ -1,6 +1,5 @@ import socket -import msgpack from typing import Optional from ymmsl import Reference diff --git a/libmuscle/python/libmuscle/mcp/tcp_server.py b/libmuscle/python/libmuscle/mcp/tcp_server.py index 6e8aeae2..5385ce48 100644 --- a/libmuscle/python/libmuscle/mcp/tcp_server.py +++ b/libmuscle/python/libmuscle/mcp/tcp_server.py @@ -3,7 +3,6 @@ from typing import cast, List, Optional, Tuple from typing_extensions import Type -import msgpack import netifaces from ymmsl import Reference diff --git a/libmuscle/python/libmuscle/mcp/test/test_direct_communication.py b/libmuscle/python/libmuscle/mcp/test/test_direct_communication.py index f8f3bfe8..b2fe60d2 100644 --- a/libmuscle/python/libmuscle/mcp/test/test_direct_communication.py +++ b/libmuscle/python/libmuscle/mcp/test/test_direct_communication.py @@ -1,7 +1,3 @@ -from unittest.mock import patch -from typing import Dict - -import pytest from ymmsl import Reference from libmuscle.mcp.direct_client import DirectClient diff --git a/libmuscle/python/libmuscle/mcp/test/test_direct_server.py b/libmuscle/python/libmuscle/mcp/test/test_direct_server.py index 103fe3e5..ea9f9b7d 100644 --- a/libmuscle/python/libmuscle/mcp/test/test_direct_server.py +++ b/libmuscle/python/libmuscle/mcp/test/test_direct_server.py @@ -1,11 +1,6 @@ -from typing import Dict - -import pytest from ymmsl import Reference -from libmuscle.outbox import Outbox -from libmuscle.post_office import PostOffice -from libmuscle.mcp.direct_server import DirectServer, registered_servers +from libmuscle.mcp.direct_server import registered_servers from libmuscle.mcp.message import Message diff --git a/libmuscle/python/libmuscle/mcp/test/test_pipe_communication.py b/libmuscle/python/libmuscle/mcp/test/test_pipe_communication.py index 3992e4c5..cf725b2a 100644 --- a/libmuscle/python/libmuscle/mcp/test/test_pipe_communication.py +++ b/libmuscle/python/libmuscle/mcp/test/test_pipe_communication.py @@ -2,8 +2,6 @@ import time from ymmsl import Reference -import pytest - from libmuscle.mcp.pipe_client import PipeClient from libmuscle.mcp.pipe_server import PipeServer import libmuscle.mcp.pipe_multiplexer as mux diff --git a/libmuscle/python/libmuscle/mcp/test/test_tcp_communication.py b/libmuscle/python/libmuscle/mcp/test/test_tcp_communication.py index 8c3f84ea..0d4c5b3e 100644 --- a/libmuscle/python/libmuscle/mcp/test/test_tcp_communication.py +++ b/libmuscle/python/libmuscle/mcp/test/test_tcp_communication.py @@ -1,11 +1,7 @@ -import time from ymmsl import Reference -import pytest - from libmuscle.mcp.tcp_client import TcpClient from libmuscle.mcp.tcp_server import TcpServer -from libmuscle.mcp.message import Message def test_send_receive(receiver, post_office): diff --git a/libmuscle/python/libmuscle/mcp/test/test_tcp_server.py b/libmuscle/python/libmuscle/mcp/test/test_tcp_server.py index 1c063a42..274eebd2 100644 --- a/libmuscle/python/libmuscle/mcp/test/test_tcp_server.py +++ b/libmuscle/python/libmuscle/mcp/test/test_tcp_server.py @@ -1,14 +1,7 @@ import socket -from typing import Dict -import msgpack -import pytest from ymmsl import Reference -from libmuscle.outbox import Outbox -from libmuscle.post_office import PostOffice -from libmuscle.mcp.message import Message - def test_create(tcp_server): assert tcp_server._instance_id == Reference('test_sender') diff --git a/libmuscle/python/libmuscle/outbox.py b/libmuscle/python/libmuscle/outbox.py index 042a7a6f..26190b02 100644 --- a/libmuscle/python/libmuscle/outbox.py +++ b/libmuscle/python/libmuscle/outbox.py @@ -1,7 +1,4 @@ from queue import Queue -from typing import List - -from ymmsl import Reference class Outbox: diff --git a/libmuscle/python/libmuscle/port.py b/libmuscle/python/libmuscle/port.py index c826d3ea..d15364aa 100644 --- a/libmuscle/python/libmuscle/port.py +++ b/libmuscle/python/libmuscle/port.py @@ -1,5 +1,5 @@ from typing import List, Optional -from ymmsl import Identifier, Operator, Reference +from ymmsl import Identifier, Operator import ymmsl from libmuscle.operator import operator_from_grpc, operator_to_grpc diff --git a/libmuscle/python/libmuscle/profiler.py b/libmuscle/python/libmuscle/profiler.py index aa2713ea..04ab390b 100644 --- a/libmuscle/python/libmuscle/profiler.py +++ b/libmuscle/python/libmuscle/profiler.py @@ -3,7 +3,6 @@ from ymmsl import Port, Reference -import muscle_manager_protocol.muscle_manager_protocol_pb2 as mmp from libmuscle.mmp_client import MMPClient from libmuscle.profiling import ProfileEvent, ProfileEventType from libmuscle.timestamp import Timestamp @@ -23,9 +22,9 @@ def __init__(self, instance_id: Reference, manager: MMPClient) -> None: self._manager = manager self._events = list() # type: List[ProfileEvent] - def start(self, event_type: ProfileEventType, port: Optional[Port]=None, - port_length: Optional[int]=None, slot: Optional[int]=None, - message_size: Optional[int]=None + def start(self, event_type: ProfileEventType, port: Optional[Port] = None, + port_length: Optional[int] = None, slot: Optional[int] = None, + message_size: Optional[int] = None ) -> ProfileEvent: """Start measuring an event. diff --git a/libmuscle/python/libmuscle/profiling.py b/libmuscle/python/libmuscle/profiling.py index a459e312..f535d5aa 100644 --- a/libmuscle/python/libmuscle/profiling.py +++ b/libmuscle/python/libmuscle/profiling.py @@ -1,12 +1,10 @@ -import datetime from enum import Enum import time -from typing import Dict, List, Optional +from typing import Dict, Optional import muscle_manager_protocol.muscle_manager_protocol_pb2 as mmp from ymmsl import Port, Reference -from libmuscle.operator import operator_to_grpc from libmuscle.port import optional_port_to_grpc from libmuscle.timestamp import Timestamp @@ -98,10 +96,10 @@ def __init__( start_time: Timestamp, stop_time: Timestamp, event_type: ProfileEventType, - port: Optional[Port]=None, - port_length: Optional[int]=None, - slot: Optional[int]=None, - message_size: Optional[int]=None + port: Optional[Port] = None, + port_length: Optional[int] = None, + slot: Optional[int] = None, + message_size: Optional[int] = None ) -> None: self.instance_id = instance_id diff --git a/libmuscle/python/libmuscle/test/test_communicator.py b/libmuscle/python/libmuscle/test/test_communicator.py index 8723a311..5e939629 100644 --- a/libmuscle/python/libmuscle/test/test_communicator.py +++ b/libmuscle/python/libmuscle/test/test_communicator.py @@ -6,7 +6,6 @@ from ymmsl import Conduit, Identifier, Operator, Reference, Settings -import msgpack import pytest from unittest.mock import patch, MagicMock diff --git a/libmuscle/python/libmuscle/test/test_grid.py b/libmuscle/python/libmuscle/test/test_grid.py index 0db64741..a24462e3 100644 --- a/libmuscle/python/libmuscle/test/test_grid.py +++ b/libmuscle/python/libmuscle/test/test_grid.py @@ -6,11 +6,11 @@ def test_grid() -> None: a = np.array([[1, 2, 3], [4, 5, 6]]) - grid = Grid(a) - grid = Grid(a, ['x', 'y']) + _ = Grid(a) + _ = Grid(a, ['x', 'y']) with pytest.raises(ValueError): - grid = Grid(a, ['x']) + _ = Grid(a, ['x']) with pytest.raises(ValueError): - grid = Grid(a, ['x', 'y', 'z']) + _ = Grid(a, ['x', 'y', 'z']) diff --git a/libmuscle/python/libmuscle/test/test_instance.py b/libmuscle/python/libmuscle/test/test_instance.py index d00c2fa5..1949cb9c 100644 --- a/libmuscle/python/libmuscle/test/test_instance.py +++ b/libmuscle/python/libmuscle/test/test_instance.py @@ -1,9 +1,9 @@ import sys -from typing import Generator, List +from typing import Generator from unittest.mock import MagicMock, patch import pytest -from ymmsl import Conduit, Operator, Reference, Settings +from ymmsl import Operator, Reference, Settings from libmuscle.communicator import Message from libmuscle.instance import Instance @@ -61,7 +61,7 @@ def instance(sys_argv_instance): @pytest.fixture def instance2(sys_argv_instance): with patch('libmuscle.instance.MMPClient') as mmp_client, \ - patch('libmuscle.instance.Communicator') as comm_type: + patch('libmuscle.instance.Communicator'): mmp_client_object = MagicMock() mmp_client_object.request_peers.return_value = (None, None, None) mmp_client.return_value = mmp_client_object diff --git a/libmuscle/python/libmuscle/test/test_mmp_client.py b/libmuscle/python/libmuscle/test/test_mmp_client.py index 66516767..ee627b36 100644 --- a/libmuscle/python/libmuscle/test/test_mmp_client.py +++ b/libmuscle/python/libmuscle/test/test_mmp_client.py @@ -1,4 +1,4 @@ -from unittest.mock import MagicMock, patch +from unittest.mock import patch import pytest from ymmsl import Conduit, Port, Reference diff --git a/setup.cfg b/setup.cfg index 637e67b4..641f70aa 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,8 +4,8 @@ test = pytest [tool:pytest] testpaths = muscle_manager libmuscle/python integration_test -addopts = --mypy --pep8 --cov --cov-report xml --cov-report term-missing -pep8ignore = +addopts = --mypy --flake8 --cov --cov-report xml --cov-report term-missing +flake8-ignore = setup.py E501 muscle_manager/protocol/*.py ALL libmuscle/manager_protocol/*.py ALL diff --git a/setup.py b/setup.py index 156f2d13..14077644 100644 --- a/setup.py +++ b/setup.py @@ -65,7 +65,6 @@ 'mypy', 'pytest>=3.5', 'pytest-cov', - 'pytest-pep8', 'pytest-flake8', 'pytest-mypy' ], From f04591a32f0712400054509e1cc2df2882bdc3ce Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Wed, 12 Aug 2020 07:14:04 +0200 Subject: [PATCH 15/36] Work around setuptools dependency issue --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 14077644..ab46642b 100644 --- a/setup.py +++ b/setup.py @@ -56,7 +56,7 @@ 'pytest-runner', # dependencies for `python setup.py build_sphinx` 'breathe', - 'sphinx', + 'sphinx<3.2', 'recommonmark', 'sphinx-rtd-theme' ], @@ -72,7 +72,7 @@ 'dev': [ 'grpcio-tools==1.17.1', 'mypy-protobuf', - 'sphinx', + 'sphinx<3.2', 'sphinx_rtd_theme', 'sphinx-fortran', 'yapf', From e1651382d6a344c32e22a5b7d7c31a2d97667432 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Sun, 23 Aug 2020 18:18:05 +0200 Subject: [PATCH 16/36] Update Python compatibility in setup.py metadata --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ab46642b..501aeb86 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,9 @@ 'License :: OSI Approved :: Apache Software License', 'Operating System :: POSIX :: Linux', 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6'], + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8'], # packages=['muscle_manager', 'muscle_manager_protocol', 'libmuscle', 'libmuscle.mcp'], packages=_muscle3_packages, From 540e1b4269f1a8adda7dbfaaf54351621cd7640e Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Sun, 23 Aug 2020 18:18:26 +0200 Subject: [PATCH 17/36] Limit Python protobuf version to <4 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 501aeb86..226069dc 100644 --- a/setup.py +++ b/setup.py @@ -50,7 +50,7 @@ 'msgpack', 'netifaces', 'numpy>=1.12', - 'protobuf>=3.10.0', + 'protobuf>=3.10.0, <4', 'typing_extensions', 'ymmsl==0.10.1' # Also in CI, update there as well ], From 0eb09d76a523062fc1be6369115c00b1e49ea815 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Sun, 23 Aug 2020 18:27:59 +0200 Subject: [PATCH 18/36] Remove test install when running make clean/distclean --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index dff04cad..94791818 100644 --- a/Makefile +++ b/Makefile @@ -116,6 +116,7 @@ clean: cd scripts && $(MAKE) clean cd docs/source/examples && $(MAKE) clean rm -rf ./build + rm -rf $(CURDIR)/libmuscle/build/test_install/* .PHONY: distclean distclean: @@ -124,6 +125,7 @@ distclean: cd scripts && $(MAKE) distclean cd docs/source/examples && $(MAKE) clean rm -rf ./build + rm -rf $(CURDIR)/libmuscle/build/test_install/* .PHONY: fortran fortran: cpp From 5f23ddb667d66e96801fa02e79d752da66fc6a78 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Sun, 23 Aug 2020 18:31:31 +0200 Subject: [PATCH 19/36] Have CI exit on error in Python tests --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 641f70aa..4742cd5b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,7 +4,7 @@ test = pytest [tool:pytest] testpaths = muscle_manager libmuscle/python integration_test -addopts = --mypy --flake8 --cov --cov-report xml --cov-report term-missing +addopts = --mypy --flake8 --cov --cov-report xml --cov-report term-missing -x flake8-ignore = setup.py E501 muscle_manager/protocol/*.py ALL From f7bf4e15c871ca63b70d492d776437c02c56b7e0 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Mon, 7 Sep 2020 22:26:52 +0200 Subject: [PATCH 20/36] Patch protobuf build error on newer linkers --- libmuscle/cpp/build/protobuf/.gitignore | 1 + libmuscle/cpp/build/protobuf/Makefile | 1 + .../build/protobuf/protobuf_issue_5144.patch | 36 +++++++++++++++++++ 3 files changed, 38 insertions(+) create mode 100644 libmuscle/cpp/build/protobuf/protobuf_issue_5144.patch diff --git a/libmuscle/cpp/build/protobuf/.gitignore b/libmuscle/cpp/build/protobuf/.gitignore index a0991ff4..7213c2a0 100644 --- a/libmuscle/cpp/build/protobuf/.gitignore +++ b/libmuscle/cpp/build/protobuf/.gitignore @@ -1,3 +1,4 @@ * !.gitignore !Makefile +!*.patch diff --git a/libmuscle/cpp/build/protobuf/Makefile b/libmuscle/cpp/build/protobuf/Makefile index d5d228b1..863347a4 100644 --- a/libmuscle/cpp/build/protobuf/Makefile +++ b/libmuscle/cpp/build/protobuf/Makefile @@ -20,6 +20,7 @@ protobuf-cpp-$(protobuf_VERSION).tar.gz: protobuf-$(protobuf_VERSION): protobuf-cpp-$(protobuf_VERSION).tar.gz $(TAR) xf protobuf-cpp-$(protobuf_VERSION).tar.gz + patch -p0 Date: Sat, 12 Sep 2020 20:32:53 +0200 Subject: [PATCH 21/36] Set C++ remote log level immediately after receiving global settings --- libmuscle/cpp/src/libmuscle/instance.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/libmuscle/cpp/src/libmuscle/instance.cpp b/libmuscle/cpp/src/libmuscle/instance.cpp index 11faed51..bc4d8823 100644 --- a/libmuscle/cpp/src/libmuscle/instance.cpp +++ b/libmuscle/cpp/src/libmuscle/instance.cpp @@ -132,6 +132,7 @@ Instance::Impl::Impl( communicator_.reset(new Communicator(name_(), index_(), ports, *logger_, 0)); register_(); connect_(); + set_log_level_(); #ifdef MUSCLE_ENABLE_MPI auto sbase_data = Data(settings_manager_.base); msgpack::sbuffer sbuf; From f2cecd8012476243d9d739926e490adad300deeb Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Sat, 12 Sep 2020 20:40:44 +0200 Subject: [PATCH 22/36] Make GitHub CI update package info before installing --- .github/workflows/ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4c6f136d..1f2527ba 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -15,6 +15,7 @@ jobs: - name: Install dependencies run: | + sudo apt-get update sudo apt-get install -y build-essential cmake gfortran libopenmpi-dev pkg-config wget valgrind sudo apt-get install -y libssl-dev zlib1g-dev pip install ymmsl==0.10.1 From 7de9234af5a5d2a751e3d4f478be64a0bc32934f Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Sun, 13 Sep 2020 22:40:48 +0200 Subject: [PATCH 23/36] Implement logging in libmuscle-Python --- libmuscle/python/libmuscle/communicator.py | 26 +++++++++++ libmuscle/python/libmuscle/instance.py | 51 ++++++++++++++++++++-- 2 files changed, 73 insertions(+), 4 deletions(-) diff --git a/libmuscle/python/libmuscle/communicator.py b/libmuscle/python/libmuscle/communicator.py index e9789ee7..49f3ec06 100644 --- a/libmuscle/python/libmuscle/communicator.py +++ b/libmuscle/python/libmuscle/communicator.py @@ -1,3 +1,4 @@ +import logging from typing import Any, Dict, List, Optional, Tuple, cast from ymmsl import Conduit, Identifier, Operator, Reference, Settings @@ -13,6 +14,9 @@ from libmuscle.profiling import ProfileEventType +_logger = logging.getLogger(__name__) + + MessageObject = Any @@ -187,8 +191,10 @@ def send_message( slot: The slot to send the message on, if any. """ if slot is None: + _logger.info('Sending message on {}'.format(port_name)) slot_list = [] # type: List[int] else: + _logger.info('Sending message on {}[{}]'.format(port_name, slot)) slot_list = [slot] slot_length = self._ports[port_name].get_length() if slot_length <= slot: @@ -254,8 +260,11 @@ def receive_message(self, port_name: str, slot: Optional[int] = None, connected. """ if slot is None: + _logger.info('Waiting for message on {}'.format(port_name)) slot_list = [] # type: List[int] else: + _logger.info('Waiting for message on {}[{}]'.format( + port_name, slot)) slot_list = [slot] recv_endpoint = self.__get_endpoint(port_name, slot_list) @@ -267,6 +276,9 @@ def receive_message(self, port_name: str, slot: Optional[int] = None, ' given. Either specify a default, or' ' connect a sending component to this' ' port.').format(port_name)) + _logger.info( + 'No message received on {} as it is not connected'.format( + port_name)) return default if port_name in self._ports: @@ -302,6 +314,16 @@ def receive_message(self, port_name: str, slot: Optional[int] = None, profile_event.port_length = port.get_length() profile_event.message_size = len(mcp_message_bytes) + if slot is None: + _logger.info('Received message on {}'.format(port_name)) + if isinstance(mcp_message.data, ClosePort): + _logger.info('Port {} is now closed'.format(port_name)) + else: + _logger.info('Received message on {}[{}]'.format(port_name, slot)) + if isinstance(mcp_message.data, ClosePort): + _logger.info('Port {}[{}] is now closed'.format( + port_name, slot)) + return message def close_port(self, port_name: str, slot: Optional[int] = None @@ -316,6 +338,10 @@ def close_port(self, port_name: str, slot: Optional[int] = None port_name: The name of the port to close. """ message = Message(float('inf'), None, ClosePort(), Settings()) + if slot is None: + _logger.info('Closing port {}'.format(port_name)) + else: + _logger.info('Closing port {}[{}]'.format(port_name, slot)) self.send_message(port_name, message, slot) def shutdown(self) -> None: diff --git a/libmuscle/python/libmuscle/instance.py b/libmuscle/python/libmuscle/instance.py index 1323d0a8..13338b2d 100644 --- a/libmuscle/python/libmuscle/instance.py +++ b/libmuscle/python/libmuscle/instance.py @@ -17,6 +17,9 @@ from libmuscle.util import extract_log_file_location +_logger = logging.getLogger(__name__) + + class Instance: """Represents a compute element instance in a MUSCLE3 simulation. @@ -64,6 +67,7 @@ def __init__(self, ports: Optional[Dict[Operator, List[str]]] = None self._register() self._connect() + self._set_log_level() def reuse_instance(self, apply_overlay: bool = True) -> bool: """Decide whether to run this instance again. @@ -105,6 +109,8 @@ def reuse_instance(self, apply_overlay: bool = True) -> bool: # At least emit a warning. self.__pre_receive_f_init(apply_overlay) + self._set_log_level() + ports = self._communicator.list_ports() f_init_not_connected = all( [not self.is_connected(port) @@ -408,9 +414,9 @@ def __set_up_logging(self) -> None: logging.getLogger().addHandler(local_handler) if self.__manager is not None: - mmp_handler = MuscleManagerHandler(id_str, logging.WARNING, - self.__manager) - logging.getLogger().addHandler(mmp_handler) + self._mmp_handler = MuscleManagerHandler(id_str, logging.WARNING, + self.__manager) + logging.getLogger().addHandler(self._mmp_handler) def __receive_message( self, port_name: str, slot: Optional[int], @@ -578,6 +584,7 @@ def pre_receive(port_name: str, slot: Optional[int]) -> None: self._f_init_cache = dict() ports = self._communicator.list_ports() for port_name in ports.get(Operator.F_INIT, []): + _logger.info('Pre-receiving on port {}'.format(port_name)) port = self._communicator.get_port(port_name) if not port.is_connected(): continue @@ -590,6 +597,42 @@ def pre_receive(port_name: str, slot: Optional[int]) -> None: for slot in range(1, port.get_length()): pre_receive(port_name, slot) + def _set_log_level(self) -> None: + """Sets the remote log level. + + This is the minimum level a message must have to be sent to + the manager. It gets this from the muscle_remote_log_level + setting. + + Note that this also sets the global log level to this level + if it is currently higher, otherwise we still get no output. + + """ + try: + log_level_str = cast( + str, self.get_setting('muscle_remote_log_level', 'str')) + level_map = { + 'CRITICAL': logging.CRITICAL, + 'ERROR': logging.ERROR, + 'WARNING': logging.WARNING, + 'INFO': logging.INFO, + 'DEBUG': logging.DEBUG} + + if log_level_str.upper() not in level_map: + _logger.warning( + ('muscle_remote_log_level is set to {}, which is not a' + ' valid log level. Please use one of DEBUG, INFO,' + ' WARNING, ERROR, or CRITICAL').format(log_level_str)) + return + + log_level = level_map[log_level_str] + self._mmp_handler.setLevel(log_level) + if not logging.getLogger().isEnabledFor(log_level): + logging.getLogger().setLevel(log_level) + except KeyError: + # muscle_remote_log_level not set, do nothing and keep the default + pass + def __apply_overlay(self, message: Message) -> None: """Sets local overlay if we don't already have one. @@ -700,7 +743,7 @@ def __shutdown(self, message: str) -> None: that we're shutting down, and deregisters from the manager. """ if not self.__is_shut_down: - logging.critical(message) + _logger.critical(message) self.__close_ports() self._communicator.shutdown() self._deregister() From 6196353758742172ddf2ba4a05760d5682364d0a Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Sun, 13 Sep 2020 22:42:48 +0200 Subject: [PATCH 24/36] Set manager default log level to NOTSET (so log everything) --- libmuscle/python/libmuscle/manager/logger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libmuscle/python/libmuscle/manager/logger.py b/libmuscle/python/libmuscle/manager/logger.py index 0c9b2411..9f942b3d 100644 --- a/libmuscle/python/libmuscle/manager/logger.py +++ b/libmuscle/python/libmuscle/manager/logger.py @@ -29,7 +29,7 @@ def __init__(self) -> None: logging.getLogger().addHandler(self._local_handler) # hardwired for now - logging.getLogger().setLevel(logging.INFO) + logging.getLogger().setLevel(logging.NOTSET) logging.getLogger('yatiml.loader').setLevel(logging.WARNING) logging.getLogger('yatiml.dumper').setLevel(logging.WARNING) From a6e8d76f7f758d0338f03cd951cb57f84b6dc388 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Mon, 14 Sep 2020 09:40:54 +0200 Subject: [PATCH 25/36] Try to fix CI test_logging failure --- integration_test/test_logging.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/integration_test/test_logging.py b/integration_test/test_logging.py index 8a51ce99..a3398e4a 100644 --- a/integration_test/test_logging.py +++ b/integration_test/test_logging.py @@ -36,11 +36,11 @@ def do_logging_test(caplog): ) # create server - logger = Logger() ymmsl_doc = ymmsl.load(ymmsl_text) expected_elements = elements_for_model(ymmsl_doc.model) instance_registry = InstanceRegistry(expected_elements) topology_store = TopologyStore(ymmsl_doc) + logger = Logger() server = MMPServer(logger, ymmsl_doc.settings, instance_registry, topology_store) @@ -49,14 +49,14 @@ def do_logging_test(caplog): message = LogMessage( instance_id='test_logging', timestamp=Timestamp(2.0), - level=LogLevel.CRITICAL, + level=LogLevel.DEBUG, text='Integration testing') # log and check client.submit_log_message(message) assert caplog.records[0].name == 'test_logging' assert caplog.records[0].time_stamp == '1970-01-01T00:00:02Z' - assert caplog.records[0].levelname == 'CRITICAL' + assert caplog.records[0].levelname == 'DEBUG' assert caplog.records[0].message == 'Integration testing' server.stop() From 6141c70e6597d3af93a1f746905209440e42d895 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Mon, 14 Sep 2020 17:51:43 +0200 Subject: [PATCH 26/36] Rework manager log level handling --- integration_test/test_cpp_mmp_client.py | 2 +- integration_test/test_logging.py | 4 ++-- libmuscle/python/libmuscle/manager/logger.py | 7 +++---- libmuscle/python/libmuscle/manager/test/test_logger.py | 4 ++-- .../python/libmuscle/manager/test/test_mmp_servicer.py | 2 +- 5 files changed, 9 insertions(+), 10 deletions(-) diff --git a/integration_test/test_cpp_mmp_client.py b/integration_test/test_cpp_mmp_client.py index 4cc60577..1c51de0a 100644 --- a/integration_test/test_cpp_mmp_client.py +++ b/integration_test/test_cpp_mmp_client.py @@ -83,7 +83,7 @@ def mock_remove(name: Reference): assert result.returncode == 0 # check submit_log_message - assert caplog.records[0].name == 'test_logging' + assert caplog.records[0].name == 'instances.test_logging' assert caplog.records[0].time_stamp == '1970-01-01T00:00:02Z' assert caplog.records[0].levelname == 'CRITICAL' assert caplog.records[0].message == 'Integration testing' diff --git a/integration_test/test_logging.py b/integration_test/test_logging.py index a3398e4a..1c92a694 100644 --- a/integration_test/test_logging.py +++ b/integration_test/test_logging.py @@ -36,11 +36,11 @@ def do_logging_test(caplog): ) # create server + logger = Logger() ymmsl_doc = ymmsl.load(ymmsl_text) expected_elements = elements_for_model(ymmsl_doc.model) instance_registry = InstanceRegistry(expected_elements) topology_store = TopologyStore(ymmsl_doc) - logger = Logger() server = MMPServer(logger, ymmsl_doc.settings, instance_registry, topology_store) @@ -54,7 +54,7 @@ def do_logging_test(caplog): # log and check client.submit_log_message(message) - assert caplog.records[0].name == 'test_logging' + assert caplog.records[0].name == 'instances.test_logging' assert caplog.records[0].time_stamp == '1970-01-01T00:00:02Z' assert caplog.records[0].levelname == 'DEBUG' assert caplog.records[0].message == 'Integration testing' diff --git a/libmuscle/python/libmuscle/manager/logger.py b/libmuscle/python/libmuscle/manager/logger.py index 9f942b3d..b91f321f 100644 --- a/libmuscle/python/libmuscle/manager/logger.py +++ b/libmuscle/python/libmuscle/manager/logger.py @@ -29,9 +29,8 @@ def __init__(self) -> None: logging.getLogger().addHandler(self._local_handler) # hardwired for now - logging.getLogger().setLevel(logging.NOTSET) - logging.getLogger('yatiml.loader').setLevel(logging.WARNING) - logging.getLogger('yatiml.dumper').setLevel(logging.WARNING) + logging.getLogger('instances').setLevel(1) + logging.getLogger('yatiml').setLevel(logging.WARNING) def close(self) -> None: logging.getLogger().removeHandler(self._local_handler) @@ -52,7 +51,7 @@ def log_message( level: The log level of the message. text: The message text. """ - logger = logging.getLogger(instance_id) + logger = logging.getLogger('instances.{}'.format(instance_id)) logger.log( level.as_python_level(), text, diff --git a/libmuscle/python/libmuscle/manager/test/test_logger.py b/libmuscle/python/libmuscle/manager/test/test_logger.py index 1c4f32ba..01f1166b 100644 --- a/libmuscle/python/libmuscle/manager/test/test_logger.py +++ b/libmuscle/python/libmuscle/manager/test/test_logger.py @@ -16,10 +16,10 @@ def test_create_logger(): def test_log_message(logger, caplog): logger.log_message( - 'test_log_message', Timestamp(123.0), + 'test_instance', Timestamp(123.0), LogLevel.CRITICAL, 'Testing the logging system') - assert caplog.records[0].name == 'test_log_message' + assert caplog.records[0].name == 'instances.test_instance' assert caplog.records[0].time_stamp == '1970-01-01T00:02:03Z' assert caplog.records[0].levelname == 'CRITICAL' assert caplog.records[0].message == 'Testing the logging system' diff --git a/libmuscle/python/libmuscle/manager/test/test_mmp_servicer.py b/libmuscle/python/libmuscle/manager/test/test_mmp_servicer.py index 08e1c91c..091914ae 100644 --- a/libmuscle/python/libmuscle/manager/test/test_mmp_servicer.py +++ b/libmuscle/python/libmuscle/manager/test/test_mmp_servicer.py @@ -20,7 +20,7 @@ def test_log_message(mmp_servicer, caplog): text='Testing log message') result = mmp_servicer.SubmitLogMessage(message, None) assert isinstance(result, mmp.LogResult) - assert caplog.records[0].name == 'test_instance_id' + assert caplog.records[0].name == 'instances.test_instance_id' assert caplog.records[0].time_stamp == '1970-01-01T00:00:00Z' assert caplog.records[0].levelname == 'WARNING' assert caplog.records[0].message == ( From a15cedd8f03c77bbec6769dfad6eeb93bcc0b66b Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Tue, 22 Sep 2020 19:10:41 +0200 Subject: [PATCH 27/36] Print more information from Makefile to help debug build problems --- libmuscle/cpp/build/check_tools.make | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/libmuscle/cpp/build/check_tools.make b/libmuscle/cpp/build/check_tools.make index 7a0dc965..14e12471 100644 --- a/libmuscle/cpp/build/check_tools.make +++ b/libmuscle/cpp/build/check_tools.make @@ -1,6 +1,17 @@ # Make module that verifies that we have all needed tools $(info ) +# Output some information about the environment +$(info Environment information:) + +$(info Variables:) +$(info $(.VARIABLES)) +$(info ) +$(info Make invocation: $(MAKE)) +$(info Make command goals: $(MAKECMDGOALS)) +$(info Make flags: $(MAKEFLAGS)) +$(info ) + # Check Python version $(info Looking for Python...) _python_version := $(shell python3 --version || echo NOTFOUND) From b8dc914b3b2ceadd777b948061f386d03e0fd4ac Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Wed, 23 Sep 2020 14:43:02 +0200 Subject: [PATCH 28/36] Make separate ymmsl.pc and ymmsl_fortran.pc (#59) --- Makefile | 2 +- docs/source/examples/cpp/build/Makefile | 12 +++++----- docs/source/examples/fortran/build/Makefile | 13 ++++++----- docs/source/installing.rst.in | 4 ++-- libmuscle/cpp/build/libmuscle/Makefile | 6 +++-- libmuscle/cpp/build/ymmsl/Makefile | 24 +++++++++++++++++++- libmuscle/fortran/build/libmuscle/Makefile | 6 +++-- libmuscle/fortran/build/ymmsl/Makefile | 25 +++++++++++++++++++-- 8 files changed, 72 insertions(+), 20 deletions(-) diff --git a/Makefile b/Makefile index 94791818..575756fa 100644 --- a/Makefile +++ b/Makefile @@ -85,7 +85,7 @@ install: all @echo " You can also use pkg-config. Add $(PREFIX)/lib/pkgconfig" @echo '* to your PKG_CONFIG_PATH environment variable and use module *' @echo '* names libmuscle, libmuscle_mpi, libmuscle_fortran or *' - @echo '* libmuscle_mpi_fortran. *' + @echo '* libmuscle_mpi_fortran and ymmsl or ymmsl_fortran. *' @echo '* *' @echo '* If the directory you installed MUSCLE 3 in is not in your *' @echo "* system's library search path, then you have to set *" diff --git a/docs/source/examples/cpp/build/Makefile b/docs/source/examples/cpp/build/Makefile index da31e583..8bf60256 100644 --- a/docs/source/examples/cpp/build/Makefile +++ b/docs/source/examples/cpp/build/Makefile @@ -1,10 +1,12 @@ CXX ?= g++ -MPICXX := mpic++ -CXXFLAGS += -std=c++14 -g $(shell pkg-config --cflags libmuscle) -MPI_CXXFLAGS := -std=c++14 -g $(shell pkg-config --cflags libmuscle_mpi) +CXXFLAGS += -std=c++14 -g $(shell pkg-config --cflags libmuscle ymmsl) +LDFLAGS += $(shell pkg-config --libs libmuscle ymmsl) -MPI_LDFLAGS := $(LDFLAGS) $(shell pkg-config --libs libmuscle_mpi) -LDFLAGS += $(shell pkg-config --libs libmuscle) +ifdef MUSCLE_ENABLE_MPI + MPICXX := mpic++ + MPI_CXXFLAGS := -std=c++14 -g $(shell pkg-config --cflags libmuscle_mpi ymmsl) + MPI_LDFLAGS := $(LDFLAGS) $(shell pkg-config --libs libmuscle_mpi ymmsl) +endif binaries := reaction diffusion mc_driver load_balancer diff --git a/docs/source/examples/fortran/build/Makefile b/docs/source/examples/fortran/build/Makefile index 049d7948..06a6dd25 100644 --- a/docs/source/examples/fortran/build/Makefile +++ b/docs/source/examples/fortran/build/Makefile @@ -1,9 +1,12 @@ FC ?= gfortran -MPIFC ?= mpifort -FFLAGS += -std=f2003 -g $(shell pkg-config --cflags libmuscle_fortran) +FFLAGS += -std=f2003 -g $(shell pkg-config --cflags libmuscle_fortran ymmsl_fortran) +LDFLAGS := $(shell pkg-config --libs libmuscle_fortran ymmsl_fortran) -MPI_LDFLAGS := $(shell pkg-config --libs libmuscle_mpi_fortran) -LDFLAGS := $(shell pkg-config --libs libmuscle_fortran) +ifdef MUSCLE_ENABLE_MPI + MPIFC ?= mpifort + MPI_FFLAGS := -std=2003 -g $(shell pkg-config --cflags libmuscle_mpi_fortran ymmsl_fortran) + MPI_LDFLAGS := $(shell pkg-config --libs libmuscle_mpi_fortran ymmsl_fortran) +endif binaries := reaction diffusion mc_driver load_balancer @@ -26,5 +29,5 @@ clean: LD_LIBRARY_PATH=$(MUSCLE3_HOME)/lib:$(LD_LIBRARY_PATH) $(FC) $(FFLAGS) -o $@ $^ $(LDFLAGS) %_mpi: ../%_mpi.f03 - LD_LIBRARY_PATH=$(MUSCLE3_HOME)/lib:$(LD_LIBRARY_PATH) $(MPIFC) $(FFLAGS) -o $@ $^ $(MPI_LDFLAGS) + LD_LIBRARY_PATH=$(MUSCLE3_HOME)/lib:$(LD_LIBRARY_PATH) $(MPIFC) $(MPI_FFLAGS) -o $@ $^ $(MPI_LDFLAGS) diff --git a/docs/source/installing.rst.in b/docs/source/installing.rst.in index e2173f25..d57974c1 100644 --- a/docs/source/installing.rst.in +++ b/docs/source/installing.rst.in @@ -305,7 +305,7 @@ libraries in case you need them. If you link statically, then you must add the MUSCLE 3 also supports ``pkg-config``. To use ``pkg-config``, add ``/lib/pkgconfig`` to your ``PKG_CONFIG_PATH`` and use the module names -``libmuscle`` or ``libmuscle_mpi``. +``ymmsl`` and either ``libmuscle`` or ``libmuscle_mpi``. There's one more thing: the directory that you've installed MUSCLE into is probably not in your system's library search path, and as a result the dynamic @@ -397,7 +397,7 @@ the libmuscle library instead: MUSCLE 3 also supports ``pkg-config``. To use ``pkg-config``, add ``/lib/pkgconfig`` to your ``PKG_CONFIG_PATH`` and use the module names -``libmuscle_fortran`` or ``libmuscle_mpi_fortran``. +``ymmsl_fortran`` and either ``libmuscle_fortran`` or ``libmuscle_mpi_fortran``. There's one more thing: the directory that you've installed MUSCLE into is probably not in your system's library search path, and as a result the dynamic diff --git a/libmuscle/cpp/build/libmuscle/Makefile b/libmuscle/cpp/build/libmuscle/Makefile index 8e564564..06c77679 100644 --- a/libmuscle/cpp/build/libmuscle/Makefile +++ b/libmuscle/cpp/build/libmuscle/Makefile @@ -210,9 +210,10 @@ libmuscle.pc: @echo 'Description: Library for MUSCLE 3' >>$@ @echo 'URL: https://muscle3.readthedocs.io' >>$@ @echo 'Version: $(muscle_version)' >>$@ + @echo 'Requires: ymmsl = $(muscle_version)' >>$@ @echo 'Requires.private: grpc++ >= 1.24.3, protobuf >= 3.10.0, msgpack >= 3.1.0' >>$@ @echo 'Cflags: -I$${includedir}' >>$@ - @echo 'Libs: -L$${libdir} -lymmsl -lmuscle' >>$@ + @echo 'Libs: -L$${libdir} -lmuscle' >>$@ libmuscle_mpi.pc: @echo 'prefix=$(PREFIX)' >$@ @@ -224,7 +225,8 @@ libmuscle_mpi.pc: @echo 'Description: Library for MUSCLE 3' >>$@ @echo 'URL: https://muscle3.readthedocs.io' >>$@ @echo 'Version: $(muscle_version)' >>$@ + @echo 'Requires: ymmsl = $(muscle_version)' >>$@ @echo 'Requires.private: grpc++ >= 1.24.3, protobuf >= 3.10.0, msgpack >= 3.1.0' >>$@ @echo 'Cflags: -I$${includedir} -DMUSCLE_ENABLE_MPI' >>$@ - @echo 'Libs: -L$${libdir} -lymmsl -lmuscle_mpi' >>$@ + @echo 'Libs: -L$${libdir} -lmuscle_mpi' >>$@ diff --git a/libmuscle/cpp/build/ymmsl/Makefile b/libmuscle/cpp/build/ymmsl/Makefile index ea95e890..01687538 100644 --- a/libmuscle/cpp/build/ymmsl/Makefile +++ b/libmuscle/cpp/build/ymmsl/Makefile @@ -18,6 +18,9 @@ public_headers := ymmsl/compute_element.hpp ymmsl/identity.hpp ymmsl/model.hpp public_headers += ymmsl/settings.hpp ymmsl/settings.tpp ymmsl/ymmsl.hpp installed_headers := $(public_headers:%=$(PREFIX)/include/%) +pkg_config_files := ymmsl.pc +installed_pkg_config_files := $(pkg_config_files:%=$(PREFIX)/lib/pkgconfig/%) + CXXFLAGS += -Wall -pedantic -std=c++14 -I$(hdrdir) export CXXFLAGS DEBUGFLAGS := -g @@ -38,6 +41,7 @@ test: .PHONY: clean clean: rm -f libymmsl.a libymmsl.so libymmsl_d.a libymmsl_d.so + rm -f $(pkg_config_files) rm -rf objs $(MAKE) -C tests clean @@ -46,7 +50,7 @@ distclean: clean $(MAKE) -C tests distclean .PHONY: install -install: all $(installed_headers) +install: all $(installed_headers) $(installed_pkg_config_files) @echo @echo Installing libymmsl to $(PREFIX) @mkdir -p $(PREFIX)/lib @@ -54,6 +58,7 @@ install: all $(installed_headers) cp libymmsl_d.a $(PREFIX)/lib cp libymmsl.so $(PREFIX)/lib cp libymmsl_d.so $(PREFIX)/lib + for pc in $(PREFIX)/lib/pkgconfig/* ; do sed -i -e 's@^prefix=.*$$@prefix=$(PREFIX)@' $$pc ; done cleantargets := clean distclean @@ -103,3 +108,20 @@ $(PREFIX)/include/%.tpp: $(hdrdir)/%.tpp @mkdir -p $(@D) cp $< $@ +$(PREFIX)/lib/pkgconfig/%.pc: %.pc + @mkdir -p $(@D) + cp $< $@ + +ymmsl.pc: + @echo 'prefix=$(PREFIX)' >$@ + @echo 'exec_prefix=$${prefix}' >>$@ + @echo 'includedir=$${prefix}/include' >>$@ + @echo 'libdir=$${exec_prefix}/lib' >>$@ + @echo >>$@ + @echo 'Name: yMMSL for C++' >>$@ + @echo 'Description: Partial yMMSL support library for MUSCLE 3' >>$@ + @echo 'URL: https://muscle3.readthedocs.io' >>$@ + @echo 'Version: $(muscle_version)' >>$@ + @echo 'Cflags: -I$${includedir}' >>$@ + @echo 'Libs: -L$${libdir} -lymmsl' >>$@ + diff --git a/libmuscle/fortran/build/libmuscle/Makefile b/libmuscle/fortran/build/libmuscle/Makefile index f5dcda51..8d3da959 100644 --- a/libmuscle/fortran/build/libmuscle/Makefile +++ b/libmuscle/fortran/build/libmuscle/Makefile @@ -121,9 +121,10 @@ libmuscle_fortran.pc: @echo 'Description: Library for MUSCLE 3 - Fortran version' >>$@ @echo 'URL: https://muscle3.readthedocs.io' >>$@ @echo 'Version: $(muscle_version)' >>$@ + @echo 'Requires: ymmsl_fortran = $(muscle_version)' >>$@ @echo 'Requires.private: libmuscle = $(muscle_version)' >>$@ @echo 'Cflags: -I$${includedir}' >>$@ - @echo 'Libs: -L$${libdir} -lymmsl_fortran -lmuscle_fortran' >>$@ + @echo 'Libs: -L$${libdir} -lmuscle_fortran' >>$@ libmuscle_mpi_fortran.pc: @echo 'prefix=$(PREFIX)' >$@ @@ -135,7 +136,8 @@ libmuscle_mpi_fortran.pc: @echo 'Description: Library for MUSCLE 3 - Fortran-MPI version' >>$@ @echo 'URL: https://muscle3.readthedocs.io' >>$@ @echo 'Version: $(muscle_version)' >>$@ + @echo 'Requires: ymmsl_fortran = $(muscle_version)' >>$@ @echo 'Requires.private: libmuscle_mpi = $(muscle_version)' >>$@ @echo 'Cflags: -I$${includedir}' >>$@ - @echo 'Libs: -L$${libdir} -lymmsl_fortran -lmuscle_mpi_fortran' >>$@ + @echo 'Libs: -L$${libdir} -lmuscle_mpi_fortran' >>$@ diff --git a/libmuscle/fortran/build/ymmsl/Makefile b/libmuscle/fortran/build/ymmsl/Makefile index 858c892c..9b1d105c 100644 --- a/libmuscle/fortran/build/ymmsl/Makefile +++ b/libmuscle/fortran/build/ymmsl/Makefile @@ -6,10 +6,12 @@ objects := $(sources:$(srcdir)/%.f03=%.o) lobjects := $(sources:$(srcdir)/%.f03=%.lo) libs := libymmsl_fortran.a libymmsl_fortran.so modules := $(sources:$(srcdir)/%.f03=%.mod) +pkg_config_files := ymmsl_fortran.pc installed_sources := $(sources:$(srcdir)/%=$(PREFIX)/include/%) installed_modules := $(modules:%=$(PREFIX)/include/%) installed_libs := $(libs:%=$(PREFIX)/lib/%) +installed_pkg_config_files := $(pkg_config_files:%=$(PREFIX)/lib/pkgconfig/%) LDFLAGS := -L../../../cpp/build/ymmsl -lymmsl FFLAGS := -std=f2003 @@ -28,14 +30,14 @@ test: tests .PHONY: clean clean: - rm -f $(objects) $(modules) $(libs) + rm -f $(objects) $(modules) $(libs) $(pkg_config_files) rm -f mod_dump/*.mod .PHONY: distclean distclean: clean .PHONY: install -install: $(installed_sources) $(installed_modules) $(installed_libs) +install: $(installed_sources) $(installed_modules) $(installed_libs) $(installed_pkg_config_files) %.o: $(srcdir)/%.f03 $(FC) -c $(FFLAGS) $^ -o $@ $(LDFLAGS) @@ -62,3 +64,22 @@ $(PREFIX)/include/%: % $(PREFIX)/lib/%: % @mkdir -p $(@D) cp $< $@ + +$(PREFIX)/lib/pkgconfig/%: % + @mkdir -p $(@D) + cp $< $@ + +ymmsl_fortran.pc: + @echo 'prefix=$(PREFIX)' >$@ + @echo 'exec_prefix=$${prefix}' >>$@ + @echo 'includedir=$${prefix}/include' >>$@ + @echo 'libdir=$${exec_prefix}/lib' >>$@ + @echo >>$@ + @echo 'Name: yMMSL for Fortran' >>$@ + @echo 'Description: Partial yMMSL support library for MUSCLE 3' >>$@ + @echo 'URL: https://muscle3.readthedocs.io' >>$@ + @echo 'Version: $(muscle_version)' >>$@ + @echo 'Requires.private: ymmsl = $(muscle_version)' >>$@ + @echo 'Cflags: -I$${includedir}' >>$@ + @echo 'Libs: -L$${libdir} -lymmsl_fortran' >>$@ + From 20eff90e345727bc71c9d515516bd52348bb80e1 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Wed, 23 Sep 2020 15:24:07 +0200 Subject: [PATCH 29/36] Fix missing sphinx-fortran dependency for local doc build --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index 226069dc..4c771b49 100644 --- a/setup.py +++ b/setup.py @@ -59,6 +59,8 @@ # dependencies for `python setup.py build_sphinx` 'breathe', 'sphinx<3.2', + 'sphinx_rtd_theme', + 'sphinx-fortran', 'recommonmark', 'sphinx-rtd-theme' ], From b3f03378e3aae3dd328749f49a3516528e10b7b1 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Wed, 23 Sep 2020 15:24:56 +0200 Subject: [PATCH 30/36] Show how to configure logging in Python tutorial --- docs/source/tutorial.rst | 44 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/docs/source/tutorial.rst b/docs/source/tutorial.rst index 67b71c0a..681189de 100644 --- a/docs/source/tutorial.rst +++ b/docs/source/tutorial.rst @@ -544,3 +544,47 @@ usual Python way in your models, and MUSCLE 3 will automatically take care of writing them to the log file. Any messages at level ``WARNING`` or higher will be sent to the manager log as well. This helps give an overview of what went wrong in a single place in case of errors. + +Note that by default, Python (and MUSCLE 3) only logs messages at level +``WARNING`` or higher. So if you add a statement like + +.. code-block:: python + + logging.info('Some useful information') + +then you'll find that it will not show up in either of the log files, because +``INFO`` is a lower level than ``WARNING``. To get more or less local log output +(e.g. in ``muscle3.macro.log``), you can use one of these commands: + +.. code-block:: python + + # least output + logging.getLogger().setLevel(logging.CRITICAL) + logging.getLogger().setLevel(logging.ERROR) + logging.getLogger().setLevel(logging.WARNING) + logging.getLogger().setLevel(logging.INFO) + logging.getLogger().setLevel(logging.DEBUG) + # most output + +The minimum log level for sending a message to the central log file +(``muscle_manager.log``) is set by the ``muscle_remote_log_level`` setting. For +example, if you change the example to read + +.. code-block:: python + + settings = Settings(OrderedDict([ + ('muscle_remote_log_level', 'DEBUG'), + ('micro.t_max', 2.469136e-6), + ('micro.dt', 2.469136e-8), + ('macro.t_max', 1.234568e-4), + ('macro.dt', 2.469136e-6), + ('x_max', 1.01), + ('dx', 0.01), + ('k', -4.05e4), # reaction parameter + ('d', 4.05e-2) # diffusion parameter + ])) + +then all log messages from all submodels will be sent to the manager log. This +does slow down the simulation if you have many log statements, so it's good to +use this to debug and to learn, but do increase the level again for production +runs. From b402432649dc044e2dedea5f3bd3a718d94d0a43 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Wed, 23 Sep 2020 17:55:24 +0200 Subject: [PATCH 31/36] Fix various small linting issues --- .../cpp/src/libmuscle/bindings/cmdlineargs.hpp | 2 +- libmuscle/cpp/src/libmuscle/data.hpp | 2 +- libmuscle/cpp/src/libmuscle/mcp/tcp_client.cpp | 3 +-- libmuscle/cpp/src/libmuscle/mcp/tcp_server.cpp | 9 ++++----- libmuscle/cpp/src/libmuscle/mmp_client.hpp | 2 +- .../libmuscle/tests/mocks/mock_mmp_client.hpp | 2 +- libmuscle/cpp/src/libmuscle/tests/test_util.cpp | 4 ++-- libmuscle/cpp/src/ymmsl/settings.cpp | 4 ++-- libmuscle/cpp/src/ymmsl/settings.hpp | 4 ++-- scripts/api_generator.py | 17 ++++++++++------- scripts/make_libmuscle_api.py | 6 +----- scripts/make_ymmsl_api.py | 5 +---- scripts/tests/cmdlineargs.hpp | 2 +- scripts/tests/make_echolib_api.py | 2 -- 14 files changed, 28 insertions(+), 36 deletions(-) diff --git a/libmuscle/cpp/src/libmuscle/bindings/cmdlineargs.hpp b/libmuscle/cpp/src/libmuscle/bindings/cmdlineargs.hpp index 3c00ef97..6a64aaf2 100644 --- a/libmuscle/cpp/src/libmuscle/bindings/cmdlineargs.hpp +++ b/libmuscle/cpp/src/libmuscle/bindings/cmdlineargs.hpp @@ -9,7 +9,7 @@ namespace libmuscle { namespace impl { namespace bindings { // Simple helper class for passing command line args from Fortran to C++. class CmdLineArgs { public: - CmdLineArgs(int count); + explicit CmdLineArgs(int count); void set_arg(int i, std::string const & arg); diff --git a/libmuscle/cpp/src/libmuscle/data.hpp b/libmuscle/cpp/src/libmuscle/data.hpp index c4cd02c7..30407693 100644 --- a/libmuscle/cpp/src/libmuscle/data.hpp +++ b/libmuscle/cpp/src/libmuscle/data.hpp @@ -476,7 +476,7 @@ class DataConstRef { Zones_ const & zones); // create DCR sharing the given zone - DataConstRef(std::shared_ptr const & zone); + explicit DataConstRef(std::shared_ptr const & zone); // create DCR with given data packed as ext type DataConstRef(char ext_type_id, DataConstRef const & data); diff --git a/libmuscle/cpp/src/libmuscle/mcp/tcp_client.cpp b/libmuscle/cpp/src/libmuscle/mcp/tcp_client.cpp index 01068cd9..a79eeac5 100644 --- a/libmuscle/cpp/src/libmuscle/mcp/tcp_client.cpp +++ b/libmuscle/cpp/src/libmuscle/mcp/tcp_client.cpp @@ -68,9 +68,8 @@ int connect(std::string const & address) { // try to connect to each in turn until we find one that works addrinfo * p; - int socket_fd; for (p = address_info.get(); p != nullptr; p = p->ai_next) { - socket_fd = socket(p->ai_family, p->ai_socktype, p->ai_protocol); + int socket_fd = socket(p->ai_family, p->ai_socktype, p->ai_protocol); if (socket_fd == -1) continue; err_code = connect(socket_fd, p->ai_addr, p->ai_addrlen); diff --git a/libmuscle/cpp/src/libmuscle/mcp/tcp_server.cpp b/libmuscle/cpp/src/libmuscle/mcp/tcp_server.cpp index e4045f3b..95384355 100644 --- a/libmuscle/cpp/src/libmuscle/mcp/tcp_server.cpp +++ b/libmuscle/cpp/src/libmuscle/mcp/tcp_server.cpp @@ -35,7 +35,7 @@ class TcpServerWorker { * * @param post_office The PostOffice to get messages from. */ - TcpServerWorker(PostOffice & post_office) + explicit TcpServerWorker(PostOffice & post_office) : post_office_(post_office) , shutting_down_(false) , connections_() @@ -296,12 +296,12 @@ TcpServer::AddrInfoList_ TcpServer::get_address_info_( std::vector TcpServer::create_sockets_(addrinfo const * addresses) const { std::vector result; - int err = 0; for (addrinfo const *p = addresses; p != nullptr; p = p->ai_next) { int sockfd; if ((sockfd = socket(p->ai_family, p->ai_socktype, p->ai_protocol)) == -1) continue; + int err = 0; if ((err = bind(sockfd, p->ai_addr, p->ai_addrlen)) == -1) { ::close(sockfd); continue; @@ -324,19 +324,18 @@ std::string TcpServer::get_address_string_(int sockfd) const { getsockname(sockfd, reinterpret_cast(&bound_addr), &addr_len); char addr_buf[INET6_ADDRSTRLEN]; - int port = 0; auto family = reinterpret_cast(&bound_addr)->sa_family; if (family == AF_INET) { auto ipv4_addr = reinterpret_cast(&bound_addr); inet_ntop(AF_INET, &(ipv4_addr->sin_addr), addr_buf, INET6_ADDRSTRLEN); - port = ntohs(ipv4_addr->sin_port); + int port = ntohs(ipv4_addr->sin_port); location = std::string(addr_buf) + ":" + std::to_string(port); } else if (family == AF_INET6) { auto ipv6_addr = reinterpret_cast(&bound_addr); inet_ntop(AF_INET6, &(ipv6_addr->sin6_addr), addr_buf, INET6_ADDRSTRLEN); - port = ntohs(ipv6_addr->sin6_port); + int port = ntohs(ipv6_addr->sin6_port); location = "[" + std::string(addr_buf) + "]:" + std::to_string(port); } else diff --git a/libmuscle/cpp/src/libmuscle/mmp_client.hpp b/libmuscle/cpp/src/libmuscle/mmp_client.hpp index 0bac14d5..743c90f3 100644 --- a/libmuscle/cpp/src/libmuscle/mmp_client.hpp +++ b/libmuscle/cpp/src/libmuscle/mmp_client.hpp @@ -32,7 +32,7 @@ class MMPClient { * * @param location A connection string of the form hostname:port. */ - MMPClient(std::string const & location); + explicit MMPClient(std::string const & location); /** Send a log message to the manager. * diff --git a/libmuscle/cpp/src/libmuscle/tests/mocks/mock_mmp_client.hpp b/libmuscle/cpp/src/libmuscle/tests/mocks/mock_mmp_client.hpp index 9135743c..c8dcfdb2 100644 --- a/libmuscle/cpp/src/libmuscle/tests/mocks/mock_mmp_client.hpp +++ b/libmuscle/cpp/src/libmuscle/tests/mocks/mock_mmp_client.hpp @@ -15,7 +15,7 @@ namespace libmuscle { namespace impl { class MockMMPClient { public: - MockMMPClient(std::string const & location); + explicit MockMMPClient(std::string const & location); void submit_log_message(LogMessage const & message); diff --git a/libmuscle/cpp/src/libmuscle/tests/test_util.cpp b/libmuscle/cpp/src/libmuscle/tests/test_util.cpp index 5b103712..18106df9 100644 --- a/libmuscle/cpp/src/libmuscle/tests/test_util.cpp +++ b/libmuscle/cpp/src/libmuscle/tests/test_util.cpp @@ -58,13 +58,13 @@ struct OptMock { ++rhs.move_constructed_from; } - OptMock const & operator=(OptMock const & rhs) { + OptMock & operator=(OptMock const & rhs) { ++copy_assigned_to; ++rhs.copy_assigned_from; return *this; } - OptMock const & operator=(OptMock && rhs) { + OptMock & operator=(OptMock && rhs) { ++move_assigned_to; ++rhs.move_assigned_from; return *this; diff --git a/libmuscle/cpp/src/ymmsl/settings.cpp b/libmuscle/cpp/src/ymmsl/settings.cpp index 3b7f68dd..5d2f5257 100644 --- a/libmuscle/cpp/src/ymmsl/settings.cpp +++ b/libmuscle/cpp/src/ymmsl/settings.cpp @@ -81,14 +81,14 @@ SettingValue::SettingValue(SettingValue && other) other.deactivate_(); } -SettingValue const & SettingValue::operator=(SettingValue const & other) { +SettingValue & SettingValue::operator=(SettingValue const & other) { deactivate_(); copy_value_from_(other); type_ = other.type_; return *this; } -SettingValue const & SettingValue::operator=(SettingValue && other) { +SettingValue & SettingValue::operator=(SettingValue && other) { deactivate_(); move_value_from_(std::move(other)); type_ = other.type_; diff --git a/libmuscle/cpp/src/ymmsl/settings.hpp b/libmuscle/cpp/src/ymmsl/settings.hpp index f2fe42d1..ed64e874 100644 --- a/libmuscle/cpp/src/ymmsl/settings.hpp +++ b/libmuscle/cpp/src/ymmsl/settings.hpp @@ -101,11 +101,11 @@ class SettingValue { /** Copy-assigns a SettingValue. */ - SettingValue const & operator=(SettingValue const & other); + SettingValue & operator=(SettingValue const & other); /** Move-assigns a SettingValue. */ - SettingValue const & operator=(SettingValue && other); + SettingValue & operator=(SettingValue && other); /** Destructs a SettingValue. */ diff --git a/scripts/api_generator.py b/scripts/api_generator.py index 8a7ffc11..2823c4fc 100644 --- a/scripts/api_generator.py +++ b/scripts/api_generator.py @@ -2,7 +2,7 @@ from copy import copy import textwrap -from typing import Callable, Dict, List, Optional, Tuple, Union +from typing import Dict, List, Optional, Tuple, Union error_codes = { @@ -129,7 +129,7 @@ def fc_return(self) -> str: def f_call_c(self, result_name: str, call: str) -> str: return ' call {}\n'.format(call) - def f_return_result(self, result_name: str, call: str) -> str: + def f_return_result(self, return_name: str, result_name: str) -> str: return '' @@ -233,7 +233,7 @@ def f_return_result(self, return_name: str, result_name: str) -> str: def fi_type(self) -> str: return self._regular_type( - ['real (c_double), dimension(*)'.format(self.ns_prefix), + ['real (c_double), dimension(*)', ('integer (c_size_t), value', '_size')]) def fi_ret_type(self) -> str: @@ -386,7 +386,7 @@ def f_return_result(self, return_name: str, result_name: str) -> str: def fi_type(self) -> str: return self._regular_type( - ['integer (c_size_t), dimension(*)'.format(self.ns_prefix), + ['integer (c_size_t), dimension(*)', ('integer (c_size_t), value', '_size')]) def fi_ret_type(self) -> str: @@ -429,7 +429,7 @@ def __init__( name: Name of the parameter. """ if name is None: - name = self.elem_type.name + name = elem_type.name self.elem_type = elem_type else: self.elem_type = copy(elem_type) @@ -1524,8 +1524,11 @@ class Constructor(MemFun): This generates code suitable for a constructor, rather than the default code. """ - def __init__(self, params: List[Par] = list(), name: str = 'create', **args - ) -> None: + def __init__( + self, params: Optional[List[Par]] = None, name: str = 'create', + **args) -> None: + if params is None: + params = list() super().__init__(Obj(''), name, params, **args) def __copy__(self) -> 'Constructor': diff --git a/scripts/make_libmuscle_api.py b/scripts/make_libmuscle_api.py index cd7fd427..33a17a52 100755 --- a/scripts/make_libmuscle_api.py +++ b/scripts/make_libmuscle_api.py @@ -5,8 +5,6 @@ from textwrap import indent from typing import Dict, List, Optional -import api_generator - from api_generator import ( API, Array, AssignmentOperator, Bool, Bytes, Char, Class, Constructor, Destructor, Double, Enum, EnumVal, Float, IndexAssignmentOperator, Int, @@ -33,7 +31,7 @@ def __init__(self, with_names: bool) -> None: This creates a set of named constructors, one for each combination of five element types and seven dimensions. If - with_types is True, every instance has n additional string + with_names is True, every instance has n additional string arguments for index names, where n is the number of dimensions of the array it accepts. @@ -55,7 +53,6 @@ def __init__(self, with_names: bool) -> None: typ.name = 'data_array' typ.set_ns_prefix({}, 'LIBMUSCLE') - instance_ret_type = Obj('') instance_params = [Array(1, copy(typ), 'data_array')] if not with_names: instance_name = 'grid_{}_a'.format(typ.tname()) @@ -115,7 +112,6 @@ def __init__(self, with_names: bool) -> None: # generate instances for typ in self.types: for ndims in range(1, 8): - instance_ret_type = Obj('') instance_params = [Array(ndims, copy(typ), 'data_array')] if with_names: for i in range(1, ndims+1): diff --git a/scripts/make_ymmsl_api.py b/scripts/make_ymmsl_api.py index 9fea59e6..656f17d6 100755 --- a/scripts/make_ymmsl_api.py +++ b/scripts/make_ymmsl_api.py @@ -3,15 +3,12 @@ import argparse from textwrap import indent -import api_generator - from api_generator import ( API, AssignmentOperator, Bool, Bytes, Char, Class, Constructor, Destructor, Double, Enum, EnumVal, EqualsOperator, Float, IndexAssignmentOperator, Int, Int16t, Int32t, Int64t, MemFun, MemFunTmpl, NamedConstructor, Namespace, Obj, OverloadSet, - ShiftedIndexAssignmentOperator, Sizet, String, T, VecDbl, Vec2Dbl, - Void) + Sizet, String, T, VecDbl, Vec2Dbl, Void) # These need to kept in sync with the values in the C++ implementation diff --git a/scripts/tests/cmdlineargs.hpp b/scripts/tests/cmdlineargs.hpp index b63ab0f5..15f6e839 100644 --- a/scripts/tests/cmdlineargs.hpp +++ b/scripts/tests/cmdlineargs.hpp @@ -9,7 +9,7 @@ namespace echolib { namespace impl { // Simple helper class for passing command line args from Fortran to C++. class CmdLineArgs { public: - CmdLineArgs(int count); + explicit CmdLineArgs(int count); void set_arg(int i, std::string const & arg); diff --git a/scripts/tests/make_echolib_api.py b/scripts/tests/make_echolib_api.py index e1ee22fa..1db884e2 100755 --- a/scripts/tests/make_echolib_api.py +++ b/scripts/tests/make_echolib_api.py @@ -2,8 +2,6 @@ import argparse -import api_generator - from api_generator import ( API, Bool, Bytes, Class, Constructor, Destructor, Double, Enum, EnumVal, Int, Int64t, MemFun, MemFunTmpl, Namespace, Obj, OverloadSet, From 8a35eca6fca1a895bd6c43625ec828a543b0b9e3 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Wed, 23 Sep 2020 20:06:57 +0200 Subject: [PATCH 32/36] Add Ubuntu 20.10 to the CI build tests --- .github/workflows/ci_ubuntu20.10.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 .github/workflows/ci_ubuntu20.10.yaml diff --git a/.github/workflows/ci_ubuntu20.10.yaml b/.github/workflows/ci_ubuntu20.10.yaml new file mode 100644 index 00000000..285eb940 --- /dev/null +++ b/.github/workflows/ci_ubuntu20.10.yaml @@ -0,0 +1,18 @@ +# Run Continuous Integration for the latest Ubuntu release +# This mainly checks for issues/regressions in the native build +name: native_compatibility_ubuntu20.10 +on: + schedule: + - cron: '0 4 * * 0' + push: + branches: + - 'release-*' +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Run tests on Ubuntu 20.10 + run: docker run -v "${GITHUB_WORKSPACE}:/home/muscle3" --env LC_ALL=C.UTF-8 --env LANG=C.UTF-8 --env DEBIAN_FRONTEND=noninteractive ubuntu:20.10 /bin/bash -c 'apt-get update && apt-get -y dist-upgrade && apt-get -y install build-essential cmake gfortran valgrind libopenmpi-dev pkg-config python3 python3-pip python3-venv curl && apt-get -y remove libssl-dev zlib1g-dev && pip3 install -U pip setuptools wheel && cd /home/muscle3 && pip3 install ymmsl==0.10.1 && make test_examples' From e026afdc63e97e166a4aeb1ee952ab3cb4dcad89 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Wed, 23 Sep 2020 20:17:37 +0200 Subject: [PATCH 33/36] Add version 0.3.2 to the change log --- CHANGELOG.rst | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 25dbf0d2..2994d239 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,6 +5,32 @@ Change Log All notable changes to this project will be documented in this file. This project adheres to `Semantic Versioning `_. +0.3.2 +***** + +Improved +-------- + +* Accessing settings from C++ now more flexible +* Python produces more detailed logs to aid in debugging +* Improved pkg-config set-up +* Improved build system output to help find problems +* Documentation on logging in Python +* Protobuf dependency build now more compatible + +Fixed +----- + +* C++ list/dict building functions +* C++ use-after-free when receiving grids + +Thanks +------ + +* Pavel for testing and reporting issues +* Dongwei for testing and reporting issues + + 0.3.1 ***** From 139d5e74955cb4d29bfe5bda5d2e0e8c1080a635 Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Wed, 23 Sep 2020 21:19:27 +0200 Subject: [PATCH 34/36] Hotfix examples MPI builds --- docs/source/examples/cpp/build/Makefile | 2 +- docs/source/examples/fortran/build/Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/examples/cpp/build/Makefile b/docs/source/examples/cpp/build/Makefile index 8bf60256..14e6a558 100644 --- a/docs/source/examples/cpp/build/Makefile +++ b/docs/source/examples/cpp/build/Makefile @@ -5,7 +5,7 @@ LDFLAGS += $(shell pkg-config --libs libmuscle ymmsl) ifdef MUSCLE_ENABLE_MPI MPICXX := mpic++ MPI_CXXFLAGS := -std=c++14 -g $(shell pkg-config --cflags libmuscle_mpi ymmsl) - MPI_LDFLAGS := $(LDFLAGS) $(shell pkg-config --libs libmuscle_mpi ymmsl) + MPI_LDFLAGS := $(shell pkg-config --libs libmuscle_mpi ymmsl) endif diff --git a/docs/source/examples/fortran/build/Makefile b/docs/source/examples/fortran/build/Makefile index 06a6dd25..578672ee 100644 --- a/docs/source/examples/fortran/build/Makefile +++ b/docs/source/examples/fortran/build/Makefile @@ -4,7 +4,7 @@ LDFLAGS := $(shell pkg-config --libs libmuscle_fortran ymmsl_fortran) ifdef MUSCLE_ENABLE_MPI MPIFC ?= mpifort - MPI_FFLAGS := -std=2003 -g $(shell pkg-config --cflags libmuscle_mpi_fortran ymmsl_fortran) + MPI_FFLAGS := -std=f2003 -g $(shell pkg-config --cflags libmuscle_mpi_fortran ymmsl_fortran) MPI_LDFLAGS := $(shell pkg-config --libs libmuscle_mpi_fortran ymmsl_fortran) endif From 86ea141ca103e992f488ad5fe7a94b0876f8f62e Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Wed, 23 Sep 2020 21:20:06 +0200 Subject: [PATCH 35/36] Set release version to 0.3.2 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 00f87844..d15723fb 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.2-dev +0.3.2 From 26f5d61f83dd84f9f4f893d109db58b80d38aeef Mon Sep 17 00:00:00 2001 From: Lourens Veen Date: Wed, 23 Sep 2020 21:53:15 +0200 Subject: [PATCH 36/36] Update badges to point to master --- README.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index ee6e82b0..d5adf666 100644 --- a/README.rst +++ b/README.rst @@ -1,11 +1,11 @@ .. image:: https://github.com/multiscale/muscle3/raw/develop/docs/source/muscle3_logo_readme.png :alt: MUSCLE 3 -.. image:: https://readthedocs.org/projects/muscle3/badge/?version=develop - :target: https://muscle3.readthedocs.io/en/develop/?badge=develop +.. image:: https://readthedocs.org/projects/muscle3/badge/?version=master + :target: https://muscle3.readthedocs.io/en/develop/?badge=master :alt: Documentation Build Status -.. image:: https://github.com/multiscale/muscle3/workflows/continuous_integration/badge.svg?branch=develop +.. image:: https://github.com/multiscale/muscle3/workflows/continuous_integration/badge.svg?branch=master :target: https://github.com/multiscale/muscle3/actions :alt: Build Status @@ -17,8 +17,8 @@ :target: https://www.codacy.com/app/LourensVeen/muscle3 :alt: Codacy Grade -.. image:: https://requires.io/github/multiscale/muscle3/requirements.svg?branch=develop - :target: https://requires.io/github/multiscale/muscle3/requirements/?branch=develop +.. image:: https://requires.io/github/multiscale/muscle3/requirements.svg?branch=master + :target: https://requires.io/github/multiscale/muscle3/requirements/?branch=master :alt: Requirements Status .. image:: https://zenodo.org/badge/122876985.svg