From a7c7c5973f89f43f6f7b7138156663d399394e96 Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Thu, 30 Jun 2022 13:58:30 +0200 Subject: [PATCH 01/49] add boost as a dependency --- CMakeLists.txt | 1 + lib/CMakeLists.txt | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a75f485e..227c887b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -26,6 +26,7 @@ if (NOT DEFINED LF_REACTOR_CPP_SUFFIX) endif() find_package (Threads) +find_package (Boost 1.79 COMPONENTS graph REQUIRED) set(DEFAULT_BUILD_TYPE "Release") if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index f8b6c9a1..103454fc 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -34,7 +34,8 @@ else() target_compile_options(${LIB_TARGET} PRIVATE -Wall -Wextra -pedantic -Werror) endif() -target_link_libraries(${LIB_TARGET} ${CMAKE_THREAD_LIBS_INIT}) +target_link_libraries(${LIB_TARGET} LINK_PUBLIC ${CMAKE_THREAD_LIBS_INIT}) +target_link_libraries(${LIB_TARGET} LINK_PUBLIC ${Boost_LIBRARIES}) if(REACTOR_CPP_TRACE) target_link_libraries(${LIB_TARGET} LTTng::UST) endif() From 77add13dc7909fb8cc9a631ad65d29a03b8d1a5c Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Thu, 30 Jun 2022 15:24:23 +0200 Subject: [PATCH 02/49] boost graph hello world --- include/reactor-cpp/dependency_graph.hh | 10 ++++++++ lib/CMakeLists.txt | 3 ++- lib/dependency_graph.cc | 33 +++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 include/reactor-cpp/dependency_graph.hh create mode 100644 lib/dependency_graph.cc diff --git a/include/reactor-cpp/dependency_graph.hh b/include/reactor-cpp/dependency_graph.hh new file mode 100644 index 00000000..1a470dc9 --- /dev/null +++ b/include/reactor-cpp/dependency_graph.hh @@ -0,0 +1,10 @@ +#ifndef REACTOR_CPP_DEPENDENCY_GRAPH_HH +#define REACTOR_CPP_DEPENDENCY_GRAPH_HH + +namespace reactor { + +void generate_test_graph(); + +} // namespace reactor + +#endif diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index 103454fc..95f25892 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -1,6 +1,7 @@ set(SOURCE_FILES action.cc assert.cc + dependency_graph.cc environment.cc logical_time.cc port.cc @@ -35,7 +36,7 @@ else() endif() target_link_libraries(${LIB_TARGET} LINK_PUBLIC ${CMAKE_THREAD_LIBS_INIT}) -target_link_libraries(${LIB_TARGET} LINK_PUBLIC ${Boost_LIBRARIES}) +target_link_libraries(${LIB_TARGET} LINK_PRIVATE ${Boost_LIBRARIES}) if(REACTOR_CPP_TRACE) target_link_libraries(${LIB_TARGET} LTTng::UST) endif() diff --git a/lib/dependency_graph.cc b/lib/dependency_graph.cc new file mode 100644 index 00000000..94635ffa --- /dev/null +++ b/lib/dependency_graph.cc @@ -0,0 +1,33 @@ +#include + +#include "reactor-cpp/dependency_graph.hh" +#include +#include +#include +#include + +using namespace boost; + +void reactor::generate_test_graph() { + // create a typedef for the Graph type + using Graph = adjacency_list; + + // Make convenient labels for the vertices + enum { A, B, C, D, E, N }; + const int num_vertices = N; + + // writing out the edges in the graph + using Edge = std::pair; + Edge edge_array[] = {Edge(A, B), Edge(A, D), Edge(C, A), Edge(D, C), Edge(C, E), Edge(B, D), Edge(D, E)}; // NOLINT + + // declare a graph object + Graph g(num_vertices); + + // add the edges to the graph object + for (auto& i : edge_array) { + add_edge(i.first, i.second, g); + } + + std::ofstream dot_file("test.dot"); + write_graphviz(dot_file, g); +} From 3395587d0027228006a017c26de49305a8d03b86 Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Fri, 1 Jul 2022 09:25:19 +0200 Subject: [PATCH 03/49] add all reactions to the new dependency graph --- include/reactor-cpp/dependency_graph.hh | 4 ++- lib/dependency_graph.cc | 41 ++++++++++++++----------- lib/environment.cc | 3 ++ 3 files changed, 29 insertions(+), 19 deletions(-) diff --git a/include/reactor-cpp/dependency_graph.hh b/include/reactor-cpp/dependency_graph.hh index 1a470dc9..eec8070f 100644 --- a/include/reactor-cpp/dependency_graph.hh +++ b/include/reactor-cpp/dependency_graph.hh @@ -1,9 +1,11 @@ #ifndef REACTOR_CPP_DEPENDENCY_GRAPH_HH #define REACTOR_CPP_DEPENDENCY_GRAPH_HH +#include "reactor-cpp/reactor.hh" + namespace reactor { -void generate_test_graph(); +void generate_dependency_graph(std::set& top_level_reactors); } // namespace reactor diff --git a/lib/dependency_graph.cc b/lib/dependency_graph.cc index 94635ffa..6d33600a 100644 --- a/lib/dependency_graph.cc +++ b/lib/dependency_graph.cc @@ -1,33 +1,38 @@ #include #include "reactor-cpp/dependency_graph.hh" -#include -#include -#include +#include "reactor-cpp/reaction.hh" +#include "reactor-cpp/reactor.hh" +#include #include using namespace boost; -void reactor::generate_test_graph() { - // create a typedef for the Graph type - using Graph = adjacency_list; +namespace reactor { - // Make convenient labels for the vertices - enum { A, B, C, D, E, N }; - const int num_vertices = N; +using DependencyGraph = directed_graph<>; +using ReactionToVertexMap = std::map; - // writing out the edges in the graph - using Edge = std::pair; - Edge edge_array[] = {Edge(A, B), Edge(A, D), Edge(C, A), Edge(D, C), Edge(C, E), Edge(B, D), Edge(D, E)}; // NOLINT +void populate_graph_with_reactions(DependencyGraph& graph, ReactionToVertexMap& vertex_map, Reactor* reactor) { + for (auto* reaction : reactor->reactions()) { + vertex_map[reaction] = graph.add_vertex(); + } + for (auto* sub_reactor : reactor->reactors()) { + populate_graph_with_reactions(graph, vertex_map, sub_reactor); + } +} - // declare a graph object - Graph g(num_vertices); +void generate_dependency_graph(std::set& top_level_reactors) { - // add the edges to the graph object - for (auto& i : edge_array) { - add_edge(i.first, i.second, g); + DependencyGraph graph{}; + ReactionToVertexMap vertex_map{}; + + for (auto* reactor : top_level_reactors) { + populate_graph_with_reactions(graph, vertex_map, reactor); } std::ofstream dot_file("test.dot"); - write_graphviz(dot_file, g); + write_graphviz(dot_file, graph); } + +} // namespace reactor diff --git a/lib/environment.cc b/lib/environment.cc index 0429e7da..7b974021 100644 --- a/lib/environment.cc +++ b/lib/environment.cc @@ -14,6 +14,7 @@ #include "reactor-cpp/action.hh" #include "reactor-cpp/assert.hh" +#include "reactor-cpp/dependency_graph.hh" #include "reactor-cpp/logging.hh" #include "reactor-cpp/port.hh" #include "reactor-cpp/reaction.hh" @@ -46,6 +47,8 @@ void Environment::assemble() { build_dependency_graph(reactor); } calculate_indexes(); + + generate_dependency_graph(top_level_reactors_); } void Environment::build_dependency_graph(Reactor* reactor) { // NOLINT From 8b5923db4222fd300d66a25bde42522a8cf54a3e Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Fri, 1 Jul 2022 09:58:03 +0200 Subject: [PATCH 04/49] sort reactions within reactor by priority --- include/reactor-cpp/reactor.hh | 9 ++++++++- lib/reactor.cc | 6 +++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/include/reactor-cpp/reactor.hh b/include/reactor-cpp/reactor.hh index 255754b1..6b1c1d76 100644 --- a/include/reactor-cpp/reactor.hh +++ b/include/reactor-cpp/reactor.hh @@ -54,10 +54,17 @@ public: class Reactor : public ReactorElement { // NOLINT private: + /** + * Compare two reactions by their priority (order within a reactor). + * + * The output is only valid if both reactions have the same container + */ + static auto compare_reaction_priority(const Reaction* a, const Reaction* b) -> bool; + std::set actions_{}; std::set inputs_{}; std::set outputs_{}; - std::set reactions_{}; + std::set reactions_{&compare_reaction_priority}; std::set reactors_{}; void register_action(BaseAction* action); diff --git a/lib/reactor.cc b/lib/reactor.cc index b3bf0c09..88e252a2 100644 --- a/lib/reactor.cc +++ b/lib/reactor.cc @@ -29,7 +29,7 @@ ReactorElement::ReactorElement(const std::string& name, ReactorElement::Type typ // when this constructor is executed. dynamic_cast only works for // completely constructed objects. Technically, the casts here return // invalid pointers as the objects they point to do not yet - // exists. However, we are good as long as we only store the pointer and do + // exists. However, we are good as long as we only store the pointer and do // not dereference it before construction is completeted. // It works, but maybe there is some nicer way of doing this... switch (type) { @@ -173,4 +173,8 @@ auto Reactor::get_elapsed_physical_time() const noexcept -> Duration { return get_physical_time() - environment()->start_time(); } +inline auto Reactor::compare_reaction_priority(const Reaction* a, const Reaction* b) -> bool { + return a->priority() < b->priority(); +} + } // namespace reactor From f2306e33d8a50190f45b7e6ea36a823436692382 Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Fri, 1 Jul 2022 10:58:27 +0200 Subject: [PATCH 05/49] add dependencies within each reactor (priorities) --- lib/dependency_graph.cc | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/lib/dependency_graph.cc b/lib/dependency_graph.cc index 6d33600a..8302b4ea 100644 --- a/lib/dependency_graph.cc +++ b/lib/dependency_graph.cc @@ -11,9 +11,9 @@ using namespace boost; namespace reactor { using DependencyGraph = directed_graph<>; -using ReactionToVertexMap = std::map; +using ReactionToVertexMap = std::map; -void populate_graph_with_reactions(DependencyGraph& graph, ReactionToVertexMap& vertex_map, Reactor* reactor) { +void populate_graph_with_reactions(DependencyGraph& graph, ReactionToVertexMap& vertex_map, const Reactor* reactor) { for (auto* reaction : reactor->reactions()) { vertex_map[reaction] = graph.add_vertex(); } @@ -22,6 +22,25 @@ void populate_graph_with_reactions(DependencyGraph& graph, ReactionToVertexMap& } } +void populate_graph_with_prioty_edges(DependencyGraph& graph, const ReactionToVertexMap& vertex_map, + const Reactor* reactor) { + const auto& reactions = reactor->reactions(); + + if (reactions.size() > 1) { + auto iterator = reactions.begin(); + auto next = std::next(iterator); + while (next != reactions.end()) { + graph.add_edge(vertex_map.at(*iterator), vertex_map.at(*next)); + iterator = next; + next = std::next(iterator); + } + } + + for (auto* sub_reactor : reactor->reactors()) { + populate_graph_with_prioty_edges(graph, vertex_map, sub_reactor); + } +} + void generate_dependency_graph(std::set& top_level_reactors) { DependencyGraph graph{}; @@ -29,6 +48,7 @@ void generate_dependency_graph(std::set& top_level_reactors) { for (auto* reactor : top_level_reactors) { populate_graph_with_reactions(graph, vertex_map, reactor); + populate_graph_with_prioty_edges(graph, vertex_map, reactor); } std::ofstream dot_file("test.dot"); From b16526f98a8d082aa824ec2df64a1daebdef534b Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Mon, 4 Jul 2022 11:57:17 +0200 Subject: [PATCH 06/49] add data dependencies to graph --- lib/dependency_graph.cc | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/lib/dependency_graph.cc b/lib/dependency_graph.cc index 8302b4ea..24ac5b03 100644 --- a/lib/dependency_graph.cc +++ b/lib/dependency_graph.cc @@ -1,6 +1,7 @@ #include #include "reactor-cpp/dependency_graph.hh" +#include "reactor-cpp/port.hh" #include "reactor-cpp/reaction.hh" #include "reactor-cpp/reactor.hh" #include @@ -22,8 +23,8 @@ void populate_graph_with_reactions(DependencyGraph& graph, ReactionToVertexMap& } } -void populate_graph_with_prioty_edges(DependencyGraph& graph, const ReactionToVertexMap& vertex_map, - const Reactor* reactor) { +void populate_graph_with_priority_edges(DependencyGraph& graph, const ReactionToVertexMap& vertex_map, + const Reactor* reactor) { const auto& reactions = reactor->reactions(); if (reactions.size() > 1) { @@ -37,7 +38,26 @@ void populate_graph_with_prioty_edges(DependencyGraph& graph, const ReactionToVe } for (auto* sub_reactor : reactor->reactors()) { - populate_graph_with_prioty_edges(graph, vertex_map, sub_reactor); + populate_graph_with_priority_edges(graph, vertex_map, sub_reactor); + } +} + +void populate_graph_with_dependency_edges(DependencyGraph& graph, const ReactionToVertexMap& vertex_map, + const Reactor* reactor) { + for (auto* reaction : reactor->reactions()) { + for (auto* dependency : reaction->dependencies()) { + auto* source = dependency; + while (source->has_inward_binding()) { + source = source->inward_binding(); + } + for (auto* antidependency : source->antidependencies()) { + graph.add_edge(vertex_map.at(antidependency), vertex_map.at(reaction)); + } + } + } + + for (auto* sub_reactor : reactor->reactors()) { + populate_graph_with_dependency_edges(graph, vertex_map, sub_reactor); } } @@ -48,7 +68,8 @@ void generate_dependency_graph(std::set& top_level_reactors) { for (auto* reactor : top_level_reactors) { populate_graph_with_reactions(graph, vertex_map, reactor); - populate_graph_with_prioty_edges(graph, vertex_map, reactor); + populate_graph_with_priority_edges(graph, vertex_map, reactor); + populate_graph_with_dependency_edges(graph, vertex_map, reactor); } std::ofstream dot_file("test.dot"); From d8e3a8bd24e1cd18b4a1edad09226c5c09829760 Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Mon, 4 Jul 2022 12:38:37 +0200 Subject: [PATCH 07/49] annotate and color the graph --- lib/dependency_graph.cc | 47 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 43 insertions(+), 4 deletions(-) diff --git a/lib/dependency_graph.cc b/lib/dependency_graph.cc index 24ac5b03..53535923 100644 --- a/lib/dependency_graph.cc +++ b/lib/dependency_graph.cc @@ -1,17 +1,28 @@ -#include - #include "reactor-cpp/dependency_graph.hh" #include "reactor-cpp/port.hh" #include "reactor-cpp/reaction.hh" #include "reactor-cpp/reactor.hh" + #include #include +#include +#include +#include + +#include +#include +#include using namespace boost; namespace reactor { -using DependencyGraph = directed_graph<>; +struct reaction_info_t { + using kind = vertex_property_tag; +}; + +using ReactionProperty = property; +using DependencyGraph = directed_graph; using ReactionToVertexMap = std::map; void populate_graph_with_reactions(DependencyGraph& graph, ReactionToVertexMap& vertex_map, const Reactor* reactor) { @@ -72,8 +83,36 @@ void generate_dependency_graph(std::set& top_level_reactors) { populate_graph_with_dependency_edges(graph, vertex_map, reactor); } + property_map::type reaction_proprty_map = get(reaction_info_t(), graph); + for (auto entry : vertex_map) { + put(reaction_proprty_map, entry.second, entry.first); + } + + dynamic_properties dp; + dp.property("node_id", get(boost::vertex_index, graph)); + dp.property("label", make_function_property_map( + [&reaction_proprty_map](DependencyGraph::vertex_descriptor vertex) { + return boost::get(reaction_proprty_map, vertex)->name(); + })); + dp.property("tooltip", make_function_property_map( + [&reaction_proprty_map](DependencyGraph::vertex_descriptor vertex) { + return "container: " + boost::get(reaction_proprty_map, vertex)->container()->fqn(); + })); + dp.property("fillcolor", + make_function_property_map( + [&reaction_proprty_map](DependencyGraph::vertex_descriptor vertex) { + auto hash = std::hash{}(boost::get(reaction_proprty_map, vertex)->container()->fqn()); + auto red = (hash & 0xff0000) >> 16; // NOLINT + auto green = (hash & 0x00ff00) >> 9; // NOLINT + auto blue = (hash & 0x0000ff); // NOLINT + std::stringstream ss; + ss << "#" << std::setfill('0') << std::setw(2) << std::hex << red << green << blue; + return ss.str(); + })); + dp.property("style", make_constant_property("filled")); + std::ofstream dot_file("test.dot"); - write_graphviz(dot_file, graph); + write_graphviz_dp(dot_file, graph, dp); } } // namespace reactor From f7a12efa7f2e15400cde7b6fb0a899c250cdf9c0 Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Mon, 4 Jul 2022 14:35:54 +0200 Subject: [PATCH 08/49] remove transitive edges --- lib/dependency_graph.cc | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/lib/dependency_graph.cc b/lib/dependency_graph.cc index 53535923..b8cb6a2c 100644 --- a/lib/dependency_graph.cc +++ b/lib/dependency_graph.cc @@ -3,14 +3,21 @@ #include "reactor-cpp/reaction.hh" #include "reactor-cpp/reactor.hh" +#include #include #include #include +#include #include #include +#include +#include +#include +#include #include #include +#include #include using namespace boost; @@ -88,6 +95,19 @@ void generate_dependency_graph(std::set& top_level_reactors) { put(reaction_proprty_map, entry.second, entry.first); } + DependencyGraph reduced_graph{}; + std::map graph_to_reduced_graph{}; + std::map id_map{}; + size_t id{0}; + for (DependencyGraph::vertex_descriptor vd : boost::make_iterator_range(vertices(graph))) { + id_map[vd] = id++; + } + + transitive_reduction(graph, reduced_graph, make_assoc_property_map(graph_to_reduced_graph), make_assoc_property_map(id_map)); + for (DependencyGraph::vertex_descriptor vd : boost::make_iterator_range(vertices(graph))) { + put(reaction_proprty_map, graph_to_reduced_graph[vd], get(reaction_proprty_map, vd)); + } + dynamic_properties dp; dp.property("node_id", get(boost::vertex_index, graph)); dp.property("label", make_function_property_map( @@ -102,17 +122,20 @@ void generate_dependency_graph(std::set& top_level_reactors) { make_function_property_map( [&reaction_proprty_map](DependencyGraph::vertex_descriptor vertex) { auto hash = std::hash{}(boost::get(reaction_proprty_map, vertex)->container()->fqn()); - auto red = (hash & 0xff0000) >> 16; // NOLINT + auto red = (hash & 0xff0000) >> 16; // NOLINT auto green = (hash & 0x00ff00) >> 9; // NOLINT - auto blue = (hash & 0x0000ff); // NOLINT + auto blue = (hash & 0x0000ff); // NOLINT std::stringstream ss; ss << "#" << std::setfill('0') << std::setw(2) << std::hex << red << green << blue; return ss.str(); })); dp.property("style", make_constant_property("filled")); - std::ofstream dot_file("test.dot"); + std::ofstream dot_file("graph.dot"); write_graphviz_dp(dot_file, graph, dp); + + std::ofstream reduced_dot_file("reduced_graph.dot"); + write_graphviz_dp(reduced_dot_file, reduced_graph, dp); } } // namespace reactor From a05e0150c1fc3d502bf1f489495d29ee4d651c4b Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Mon, 4 Jul 2022 17:14:55 +0200 Subject: [PATCH 09/49] restructure the code and introduce an actual class --- include/reactor-cpp/dependency_graph.hh | 33 ++++++++- lib/dependency_graph.cc | 89 ++++++++++++------------- lib/environment.cc | 7 +- 3 files changed, 80 insertions(+), 49 deletions(-) diff --git a/include/reactor-cpp/dependency_graph.hh b/include/reactor-cpp/dependency_graph.hh index eec8070f..9d0e528c 100644 --- a/include/reactor-cpp/dependency_graph.hh +++ b/include/reactor-cpp/dependency_graph.hh @@ -3,9 +3,40 @@ #include "reactor-cpp/reactor.hh" +#include + namespace reactor { -void generate_dependency_graph(std::set& top_level_reactors); +class ReactionDependencyGraph { + +private: + struct reaction_info_t { + using kind = boost::vertex_property_tag; + }; + using ReactionProperty = boost::property; + using ReactionGraph = boost::directed_graph; + using ReactionToVertexMap = std::map; + using ReactionPropertyMap = boost::property_map::type; + + ReactionGraph graph{}; + ReactionToVertexMap vertex_map{}; + + // helper functions + void populate_graph_with_reactions(const Reactor* reactor); + void populate_graph_with_priority_edges(const Reactor* reactor); + void populate_graph_with_dependency_edges(const Reactor* reactor); + + [[nodiscard]] auto get_reaction_property_map() -> ReactionPropertyMap { return boost::get(reaction_info_t{}, graph); } + + ReactionDependencyGraph() = default; + +public: + ReactionDependencyGraph(const std::set& top_level_reactors); + + void export_graphviz(const std::string& file_name); + + [[nodiscard]] auto transitive_reduction() const -> ReactionDependencyGraph; +}; } // namespace reactor diff --git a/lib/dependency_graph.cc b/lib/dependency_graph.cc index b8cb6a2c..85eaf2d9 100644 --- a/lib/dependency_graph.cc +++ b/lib/dependency_graph.cc @@ -3,16 +3,12 @@ #include "reactor-cpp/reaction.hh" #include "reactor-cpp/reactor.hh" -#include #include #include #include #include -#include #include -#include -#include #include #include #include @@ -24,25 +20,16 @@ using namespace boost; namespace reactor { -struct reaction_info_t { - using kind = vertex_property_tag; -}; - -using ReactionProperty = property; -using DependencyGraph = directed_graph; -using ReactionToVertexMap = std::map; - -void populate_graph_with_reactions(DependencyGraph& graph, ReactionToVertexMap& vertex_map, const Reactor* reactor) { +void ReactionDependencyGraph::populate_graph_with_reactions(const Reactor* reactor) { for (auto* reaction : reactor->reactions()) { vertex_map[reaction] = graph.add_vertex(); } for (auto* sub_reactor : reactor->reactors()) { - populate_graph_with_reactions(graph, vertex_map, sub_reactor); + populate_graph_with_reactions(sub_reactor); } } -void populate_graph_with_priority_edges(DependencyGraph& graph, const ReactionToVertexMap& vertex_map, - const Reactor* reactor) { +void ReactionDependencyGraph::populate_graph_with_priority_edges(const Reactor* reactor) { const auto& reactions = reactor->reactions(); if (reactions.size() > 1) { @@ -56,12 +43,11 @@ void populate_graph_with_priority_edges(DependencyGraph& graph, const ReactionTo } for (auto* sub_reactor : reactor->reactors()) { - populate_graph_with_priority_edges(graph, vertex_map, sub_reactor); + populate_graph_with_priority_edges(sub_reactor); } } -void populate_graph_with_dependency_edges(DependencyGraph& graph, const ReactionToVertexMap& vertex_map, - const Reactor* reactor) { +void ReactionDependencyGraph::populate_graph_with_dependency_edges(const Reactor* reactor) { for (auto* reaction : reactor->reactions()) { for (auto* dependency : reaction->dependencies()) { auto* source = dependency; @@ -75,52 +61,64 @@ void populate_graph_with_dependency_edges(DependencyGraph& graph, const Reaction } for (auto* sub_reactor : reactor->reactors()) { - populate_graph_with_dependency_edges(graph, vertex_map, sub_reactor); + populate_graph_with_dependency_edges(sub_reactor); } } -void generate_dependency_graph(std::set& top_level_reactors) { - - DependencyGraph graph{}; - ReactionToVertexMap vertex_map{}; - +ReactionDependencyGraph::ReactionDependencyGraph(const std::set& top_level_reactors) { for (auto* reactor : top_level_reactors) { - populate_graph_with_reactions(graph, vertex_map, reactor); - populate_graph_with_priority_edges(graph, vertex_map, reactor); - populate_graph_with_dependency_edges(graph, vertex_map, reactor); + populate_graph_with_reactions(reactor); + populate_graph_with_priority_edges(reactor); + populate_graph_with_dependency_edges(reactor); } - property_map::type reaction_proprty_map = get(reaction_info_t(), graph); + // annotate edge vertex with a property that points to the reaction it corresponds to + auto reaction_proprty_map = get_reaction_property_map(); for (auto entry : vertex_map) { put(reaction_proprty_map, entry.second, entry.first); } +} - DependencyGraph reduced_graph{}; - std::map graph_to_reduced_graph{}; - std::map id_map{}; +auto ReactionDependencyGraph::transitive_reduction() const -> ReactionDependencyGraph { + ReactionDependencyGraph reduced{}; + + // transitive_reduction uses this to populate a mapping from original vertices to new vertices in the reduced graph + std::map graph_to_reduced_graph{}; + // transitive reduction needs this mapping of vertices to integers + std::map id_map{}; size_t id{0}; - for (DependencyGraph::vertex_descriptor vd : boost::make_iterator_range(vertices(graph))) { + for (ReactionGraph::vertex_descriptor vd : boost::make_iterator_range(vertices(graph))) { id_map[vd] = id++; } - transitive_reduction(graph, reduced_graph, make_assoc_property_map(graph_to_reduced_graph), make_assoc_property_map(id_map)); - for (DependencyGraph::vertex_descriptor vd : boost::make_iterator_range(vertices(graph))) { - put(reaction_proprty_map, graph_to_reduced_graph[vd], get(reaction_proprty_map, vd)); + // perform the actual reduction + boost::transitive_reduction(graph, reduced.graph, make_assoc_property_map(graph_to_reduced_graph), + make_assoc_property_map(id_map)); + + // update the mapping of reactions to vertices + auto reaction_property_map = reduced.get_reaction_property_map(); + for (ReactionGraph::vertex_descriptor vd : boost::make_iterator_range(vertices(graph))) { + put(reaction_property_map, graph_to_reduced_graph[vd], get(reaction_property_map, vd)); } + return reduced; +} + +void ReactionDependencyGraph::export_graphviz(const std::string& file_name) { + auto reaction_proprty_map = get_reaction_property_map(); dynamic_properties dp; dp.property("node_id", get(boost::vertex_index, graph)); - dp.property("label", make_function_property_map( - [&reaction_proprty_map](DependencyGraph::vertex_descriptor vertex) { + dp.property("label", make_function_property_map( + [&reaction_proprty_map](ReactionGraph::vertex_descriptor vertex) { return boost::get(reaction_proprty_map, vertex)->name(); })); - dp.property("tooltip", make_function_property_map( - [&reaction_proprty_map](DependencyGraph::vertex_descriptor vertex) { + dp.property("tooltip", make_function_property_map( + [&reaction_proprty_map](ReactionGraph::vertex_descriptor vertex) { return "container: " + boost::get(reaction_proprty_map, vertex)->container()->fqn(); })); dp.property("fillcolor", - make_function_property_map( - [&reaction_proprty_map](DependencyGraph::vertex_descriptor vertex) { + make_function_property_map( + [&reaction_proprty_map](ReactionGraph::vertex_descriptor vertex) { auto hash = std::hash{}(boost::get(reaction_proprty_map, vertex)->container()->fqn()); auto red = (hash & 0xff0000) >> 16; // NOLINT auto green = (hash & 0x00ff00) >> 9; // NOLINT @@ -129,13 +127,10 @@ void generate_dependency_graph(std::set& top_level_reactors) { ss << "#" << std::setfill('0') << std::setw(2) << std::hex << red << green << blue; return ss.str(); })); - dp.property("style", make_constant_property("filled")); + dp.property("style", make_constant_property("filled")); - std::ofstream dot_file("graph.dot"); + std::ofstream dot_file(file_name); write_graphviz_dp(dot_file, graph, dp); - - std::ofstream reduced_dot_file("reduced_graph.dot"); - write_graphviz_dp(reduced_dot_file, reduced_graph, dp); } } // namespace reactor diff --git a/lib/environment.cc b/lib/environment.cc index 7b974021..e6ec5330 100644 --- a/lib/environment.cc +++ b/lib/environment.cc @@ -48,7 +48,12 @@ void Environment::assemble() { } calculate_indexes(); - generate_dependency_graph(top_level_reactors_); + // Testbed for the new graph structure + ReactionDependencyGraph graph{top_level_reactors_}; + ReactionDependencyGraph reduced_graph = graph.transitive_reduction(); + + graph.export_graphviz("graph.dot"); + reduced_graph.export_graphviz("reduced_graph.dot"); } void Environment::build_dependency_graph(Reactor* reactor) { // NOLINT From 76f70e73e10081bffa218d7bff5fb57ab4f70a2a Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Mon, 4 Jul 2022 17:33:47 +0200 Subject: [PATCH 10/49] fix priority sorting --- lib/reactor.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/reactor.cc b/lib/reactor.cc index 88e252a2..6ee7f010 100644 --- a/lib/reactor.cc +++ b/lib/reactor.cc @@ -174,7 +174,7 @@ auto Reactor::get_elapsed_physical_time() const noexcept -> Duration { } inline auto Reactor::compare_reaction_priority(const Reaction* a, const Reaction* b) -> bool { - return a->priority() < b->priority(); + return a->priority() > b->priority(); } } // namespace reactor From 14310f1db609d7ca86683af3e6a241b8ae6e6614 Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Mon, 4 Jul 2022 18:32:01 +0200 Subject: [PATCH 11/49] first implementation of a grouped dependency graph --- include/reactor-cpp/dependency_graph.hh | 36 +++++++++++++++- lib/dependency_graph.cc | 55 +++++++++++++++++++++++++ lib/environment.cc | 2 + 3 files changed, 91 insertions(+), 2 deletions(-) diff --git a/include/reactor-cpp/dependency_graph.hh b/include/reactor-cpp/dependency_graph.hh index 9d0e528c..b77de4ba 100644 --- a/include/reactor-cpp/dependency_graph.hh +++ b/include/reactor-cpp/dependency_graph.hh @@ -1,14 +1,17 @@ #ifndef REACTOR_CPP_DEPENDENCY_GRAPH_HH #define REACTOR_CPP_DEPENDENCY_GRAPH_HH +#include "reactor-cpp/reaction.hh" #include "reactor-cpp/reactor.hh" #include +#include namespace reactor { -class ReactionDependencyGraph { +class GroupedDependencyGraph; +class ReactionDependencyGraph { private: struct reaction_info_t { using kind = boost::vertex_property_tag; @@ -18,7 +21,7 @@ private: using ReactionToVertexMap = std::map; using ReactionPropertyMap = boost::property_map::type; - ReactionGraph graph{}; + ReactionGraph graph{}; ReactionToVertexMap vertex_map{}; // helper functions @@ -36,6 +39,35 @@ public: void export_graphviz(const std::string& file_name); [[nodiscard]] auto transitive_reduction() const -> ReactionDependencyGraph; + + friend GroupedDependencyGraph; +}; + +class GroupedDependencyGraph { + +private: + struct group_info_t { + using kind = boost::vertex_property_tag; + }; + using Group = std::vector; + using GroupProperty = boost::property; + using GroupGraph = boost::directed_graph; + using ReactionToVertexMap = std::map; + using GroupPropertyMap = boost::property_map::type; + + GroupGraph graph{}; + ReactionToVertexMap vertex_map{}; + + [[nodiscard]] auto get_group_property_map() -> GroupPropertyMap { return boost::get(group_info_t{}, graph); } + +public: + // TODO: This should be a const reference, but I don't know how to get immutable access to the reaction graph + // properties... + GroupedDependencyGraph(ReactionDependencyGraph& reactionGraph); + + void export_graphviz(const std::string& file_name); + + [[nodiscard]] auto transitive_reduction() const -> GroupedDependencyGraph; }; } // namespace reactor diff --git a/lib/dependency_graph.cc b/lib/dependency_graph.cc index 85eaf2d9..1416e369 100644 --- a/lib/dependency_graph.cc +++ b/lib/dependency_graph.cc @@ -3,8 +3,10 @@ #include "reactor-cpp/reaction.hh" #include "reactor-cpp/reactor.hh" +#include #include #include +#include #include #include #include @@ -133,4 +135,57 @@ void ReactionDependencyGraph::export_graphviz(const std::string& file_name) { write_graphviz_dp(dot_file, graph, dp); } +GroupedDependencyGraph::GroupedDependencyGraph(ReactionDependencyGraph& reactionGraph) { + // the lambda below also has the side-effect of updating vertex map + copy_graph(reactionGraph.graph, graph, + vertex_copy([this, &reactionGraph](ReactionDependencyGraph::ReactionGraph::vertex_descriptor in, + GroupGraph::vertex_descriptor out) { + Group group{}; + group.push_back(boost::get(reactionGraph.get_reaction_property_map(), in)); + boost::put(get_group_property_map(), out, group); + vertex_map[group[0]] = out; + })); +} + + void GroupedDependencyGraph::export_graphviz(const std::string& file_name) { + auto group_proprty_map = get_group_property_map(); + dynamic_properties dp; + dp.property("node_id", get(boost::vertex_index, graph)); + dp.property("label", make_function_property_map( + [&group_proprty_map](GroupGraph::vertex_descriptor vertex) { + std::stringstream ss; + for (const auto* reaction : boost::get(group_proprty_map, vertex)) { + auto sep = ss.tellp()==0 ? '{' : ','; + ss << sep << reaction->name(); + } + ss << '}'; + return ss.str(); + })); + dp.property("tooltip", make_function_property_map( + [&group_proprty_map](GroupGraph::vertex_descriptor vertex) { + std::stringstream ss; + for (const auto* reaction : boost::get(group_proprty_map, vertex)) { + ss << "reactions: \n"; + ss << " - " << reaction->fqn(); + } + ss << '}'; + return ss.str(); + })); + dp.property("fillcolor", + make_function_property_map( + [&group_proprty_map](GroupGraph::vertex_descriptor vertex) { + auto hash = std::hash{}(boost::get(group_proprty_map, vertex)[0]->container()->fqn()); + auto red = (hash & 0xff0000) >> 16; // NOLINT + auto green = (hash & 0x00ff00) >> 9; // NOLINT + auto blue = (hash & 0x0000ff); // NOLINT + std::stringstream ss; + ss << "#" << std::setfill('0') << std::setw(2) << std::hex << red << green << blue; + return ss.str(); + })); + dp.property("style", make_constant_property("filled")); + + std::ofstream dot_file(file_name); + write_graphviz_dp(dot_file, graph, dp); +} + } // namespace reactor diff --git a/lib/environment.cc b/lib/environment.cc index e6ec5330..49a5e876 100644 --- a/lib/environment.cc +++ b/lib/environment.cc @@ -51,9 +51,11 @@ void Environment::assemble() { // Testbed for the new graph structure ReactionDependencyGraph graph{top_level_reactors_}; ReactionDependencyGraph reduced_graph = graph.transitive_reduction(); + GroupedDependencyGraph grouped_graph{reduced_graph}; graph.export_graphviz("graph.dot"); reduced_graph.export_graphviz("reduced_graph.dot"); + grouped_graph.export_graphviz("grouped_graph.dot"); } void Environment::build_dependency_graph(Reactor* reactor) { // NOLINT From 8eb1ffafbb414f7088038392b6b09f08f0fd58a4 Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Tue, 5 Jul 2022 14:01:42 +0200 Subject: [PATCH 12/49] group reactions within the same reactor --- include/reactor-cpp/dependency_graph.hh | 28 ++++++++ lib/dependency_graph.cc | 87 ++++++++++++++++++++++++- lib/environment.cc | 4 +- 3 files changed, 116 insertions(+), 3 deletions(-) diff --git a/include/reactor-cpp/dependency_graph.hh b/include/reactor-cpp/dependency_graph.hh index b77de4ba..612351ff 100644 --- a/include/reactor-cpp/dependency_graph.hh +++ b/include/reactor-cpp/dependency_graph.hh @@ -4,6 +4,7 @@ #include "reactor-cpp/reaction.hh" #include "reactor-cpp/reactor.hh" +#include #include #include @@ -60,6 +61,27 @@ private: [[nodiscard]] auto get_group_property_map() -> GroupPropertyMap { return boost::get(group_info_t{}, graph); } + struct ReachabilityVisitor : public boost::default_bfs_visitor { + private: + GroupGraph::vertex_descriptor to; + + public: + ReachabilityVisitor(GroupGraph::vertex_descriptor to) + : to{to} {} + + // this exception is used to stop the BFS early if a matching vertex is found + struct PathExists : public std::exception {}; + + // this throws PathExists on success + void discover_vertex(GroupGraph::vertex_descriptor u, [[maybe_unused]] const GroupGraph& g) const { + if (u == to) { + throw PathExists(); + } + } + }; + + void group_reactions_by_container_helper(const Reactor* reactor); + public: // TODO: This should be a const reference, but I don't know how to get immutable access to the reaction graph // properties... @@ -67,6 +89,12 @@ public: void export_graphviz(const std::string& file_name); + void try_contract_edge(const Reaction* a, const Reaction* b); + + void group_reactions_by_container(const std::set& top_level_reactors); + + auto has_path(const Reaction* a, const Reaction* b) const -> bool; + [[nodiscard]] auto transitive_reduction() const -> GroupedDependencyGraph; }; diff --git a/lib/dependency_graph.cc b/lib/dependency_graph.cc index 1416e369..60641d9e 100644 --- a/lib/dependency_graph.cc +++ b/lib/dependency_graph.cc @@ -3,13 +3,17 @@ #include "reactor-cpp/reaction.hh" #include "reactor-cpp/reactor.hh" +#include #include #include +#include #include #include #include +#include #include #include +#include #include #include @@ -147,7 +151,7 @@ GroupedDependencyGraph::GroupedDependencyGraph(ReactionDependencyGraph& reaction })); } - void GroupedDependencyGraph::export_graphviz(const std::string& file_name) { +void GroupedDependencyGraph::export_graphviz(const std::string& file_name) { auto group_proprty_map = get_group_property_map(); dynamic_properties dp; dp.property("node_id", get(boost::vertex_index, graph)); @@ -155,7 +159,7 @@ GroupedDependencyGraph::GroupedDependencyGraph(ReactionDependencyGraph& reaction [&group_proprty_map](GroupGraph::vertex_descriptor vertex) { std::stringstream ss; for (const auto* reaction : boost::get(group_proprty_map, vertex)) { - auto sep = ss.tellp()==0 ? '{' : ','; + auto sep = ss.tellp() == 0 ? '{' : ','; ss << sep << reaction->name(); } ss << '}'; @@ -188,4 +192,83 @@ GroupedDependencyGraph::GroupedDependencyGraph(ReactionDependencyGraph& reaction write_graphviz_dp(dot_file, graph, dp); } +void GroupedDependencyGraph::try_contract_edge(const Reaction* a, const Reaction* b) { + GroupGraph::vertex_descriptor va = vertex_map[a]; + GroupGraph::vertex_descriptor vb = vertex_map[b]; + + // if both vertexes are the same we can abort... + if (va == vb) { + return; + } + + // we remove the edge that we want to contract + remove_edge(va, vb, graph); + // and then check if there is another path from a to b + if (has_path(a, b)) { + // If there is another path from a to b, contracting would introduce a cycle into the graph. Thus we abort here and + // simply add the edge back that we removed earlier. + add_edge(va, vb, graph); + } else { + // it is safe to contract. + + // route all edges in/out of vb to va + for (auto edge : make_iterator_range(out_edges(vb, graph))) { + add_edge(va, target(edge, graph), graph); + } + for (auto edge : make_iterator_range(in_edges(vb, graph))) { + add_edge(source(edge, graph), va, graph); + } + + // update va's properties + auto& va_reactions = boost::get(get_group_property_map(), va); + auto& vb_reactions = boost::get(get_group_property_map(), vb); + va_reactions.insert(va_reactions.end(), vb_reactions.begin(), vb_reactions.end()); + + // update the vertex mapping + for (const auto* reaction : vb_reactions) { + vertex_map[reaction] = va; + } + + // delete vb + clear_vertex(vb, graph); + remove_vertex(vb, graph); + } +} + +auto GroupedDependencyGraph::has_path(const Reaction* a, const Reaction* b) const -> bool { + GroupGraph::vertex_descriptor va = vertex_map.at(a); + GroupGraph::vertex_descriptor vb = vertex_map.at(b); + + try { + // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDelete) [prevents a warning triggered from the boost headers] + breadth_first_search(graph, va, visitor(ReachabilityVisitor(vb))); + } catch (const ReachabilityVisitor::PathExists& e) { + return true; + } + return false; +} + +void GroupedDependencyGraph::group_reactions_by_container(const std::set& top_level_reactors) { + for (const auto* reactor : top_level_reactors) { + group_reactions_by_container_helper(reactor); + } +} + +void GroupedDependencyGraph::group_reactions_by_container_helper(const Reactor* reactor) { + const auto& reactions = reactor->reactions(); + + if (reactions.size() > 1) { + auto it = reactions.begin(); + auto next = std::next(it); + while (next != reactions.end()) { + try_contract_edge(*it, *next); + it = next; + next = std::next(it); + } + } + + for (const auto* r : reactor->reactors()) { + group_reactions_by_container_helper(r); + } +} } // namespace reactor diff --git a/lib/environment.cc b/lib/environment.cc index 49a5e876..5d309cfb 100644 --- a/lib/environment.cc +++ b/lib/environment.cc @@ -51,10 +51,12 @@ void Environment::assemble() { // Testbed for the new graph structure ReactionDependencyGraph graph{top_level_reactors_}; ReactionDependencyGraph reduced_graph = graph.transitive_reduction(); - GroupedDependencyGraph grouped_graph{reduced_graph}; graph.export_graphviz("graph.dot"); reduced_graph.export_graphviz("reduced_graph.dot"); + + GroupedDependencyGraph grouped_graph{reduced_graph}; + grouped_graph.group_reactions_by_container(top_level_reactors_); grouped_graph.export_graphviz("grouped_graph.dot"); } From a713d90f54ee6af54245fb98b8d8be0dc99e4022 Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Tue, 5 Jul 2022 14:37:47 +0200 Subject: [PATCH 13/49] simplify contraction algorithm --- include/reactor-cpp/dependency_graph.hh | 2 - lib/dependency_graph.cc | 62 +++++++++---------------- 2 files changed, 23 insertions(+), 41 deletions(-) diff --git a/include/reactor-cpp/dependency_graph.hh b/include/reactor-cpp/dependency_graph.hh index 612351ff..3ba1e72a 100644 --- a/include/reactor-cpp/dependency_graph.hh +++ b/include/reactor-cpp/dependency_graph.hh @@ -93,8 +93,6 @@ public: void group_reactions_by_container(const std::set& top_level_reactors); - auto has_path(const Reaction* a, const Reaction* b) const -> bool; - [[nodiscard]] auto transitive_reduction() const -> GroupedDependencyGraph; }; diff --git a/lib/dependency_graph.cc b/lib/dependency_graph.cc index 60641d9e..73f93baa 100644 --- a/lib/dependency_graph.cc +++ b/lib/dependency_graph.cc @@ -201,51 +201,35 @@ void GroupedDependencyGraph::try_contract_edge(const Reaction* a, const Reaction return; } - // we remove the edge that we want to contract - remove_edge(va, vb, graph); - // and then check if there is another path from a to b - if (has_path(a, b)) { - // If there is another path from a to b, contracting would introduce a cycle into the graph. Thus we abort here and - // simply add the edge back that we removed earlier. - add_edge(va, vb, graph); - } else { - // it is safe to contract. - - // route all edges in/out of vb to va - for (auto edge : make_iterator_range(out_edges(vb, graph))) { - add_edge(va, target(edge, graph), graph); - } - for (auto edge : make_iterator_range(in_edges(vb, graph))) { - add_edge(source(edge, graph), va, graph); - } - - // update va's properties - auto& va_reactions = boost::get(get_group_property_map(), va); - auto& vb_reactions = boost::get(get_group_property_map(), vb); - va_reactions.insert(va_reactions.end(), vb_reactions.begin(), vb_reactions.end()); + // Abort if there is no direct edge between the vertexes + if (!edge(va, vb, graph).second) { + return; + } - // update the vertex mapping - for (const auto* reaction : vb_reactions) { - vertex_map[reaction] = va; - } + // remove the direct edge between va and vb + remove_edge(va, vb, graph); - // delete vb - clear_vertex(vb, graph); - remove_vertex(vb, graph); + // route all edges in/out of vb to va + for (auto edge : make_iterator_range(out_edges(vb, graph))) { + add_edge(va, target(edge, graph), graph); + } + for (auto edge : make_iterator_range(in_edges(vb, graph))) { + add_edge(source(edge, graph), va, graph); } -} -auto GroupedDependencyGraph::has_path(const Reaction* a, const Reaction* b) const -> bool { - GroupGraph::vertex_descriptor va = vertex_map.at(a); - GroupGraph::vertex_descriptor vb = vertex_map.at(b); + // update va's properties + auto& va_reactions = boost::get(get_group_property_map(), va); + auto& vb_reactions = boost::get(get_group_property_map(), vb); + va_reactions.insert(va_reactions.end(), vb_reactions.begin(), vb_reactions.end()); - try { - // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDelete) [prevents a warning triggered from the boost headers] - breadth_first_search(graph, va, visitor(ReachabilityVisitor(vb))); - } catch (const ReachabilityVisitor::PathExists& e) { - return true; + // update the vertex mapping + for (const auto* reaction : vb_reactions) { + vertex_map[reaction] = va; } - return false; + + // delete vb + clear_vertex(vb, graph); + remove_vertex(vb, graph); } void GroupedDependencyGraph::group_reactions_by_container(const std::set& top_level_reactors) { From ca41810afb11d017a7e3a8ea9577d4460bd676a3 Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Tue, 5 Jul 2022 14:38:07 +0200 Subject: [PATCH 14/49] reduce the grouped graph --- include/reactor-cpp/dependency_graph.hh | 2 ++ lib/dependency_graph.cc | 26 +++++++++++++++++++++++++ lib/environment.cc | 3 +++ 3 files changed, 31 insertions(+) diff --git a/include/reactor-cpp/dependency_graph.hh b/include/reactor-cpp/dependency_graph.hh index 3ba1e72a..3a450ff3 100644 --- a/include/reactor-cpp/dependency_graph.hh +++ b/include/reactor-cpp/dependency_graph.hh @@ -82,6 +82,8 @@ private: void group_reactions_by_container_helper(const Reactor* reactor); + GroupedDependencyGraph() = default; + public: // TODO: This should be a const reference, but I don't know how to get immutable access to the reaction graph // properties... diff --git a/lib/dependency_graph.cc b/lib/dependency_graph.cc index 73f93baa..2077781c 100644 --- a/lib/dependency_graph.cc +++ b/lib/dependency_graph.cc @@ -255,4 +255,30 @@ void GroupedDependencyGraph::group_reactions_by_container_helper(const Reactor* group_reactions_by_container_helper(r); } } + +auto GroupedDependencyGraph::transitive_reduction() const -> GroupedDependencyGraph { + GroupedDependencyGraph reduced{}; + + // transitive_reduction uses this to populate a mapping from original vertices to new vertices in the reduced graph + std::map graph_to_reduced_graph{}; + // transitive reduction needs this mapping of vertices to integers + std::map id_map{}; + size_t id{0}; + for (GroupGraph::vertex_descriptor vd : boost::make_iterator_range(vertices(graph))) { + id_map[vd] = id++; + } + + // perform the actual reduction + boost::transitive_reduction(graph, reduced.graph, make_assoc_property_map(graph_to_reduced_graph), + make_assoc_property_map(id_map)); + + // update the mapping of reactions to vertices + auto group_property_map = reduced.get_group_property_map(); + for (GroupGraph::vertex_descriptor vd : boost::make_iterator_range(vertices(graph))) { + put(group_property_map, graph_to_reduced_graph[vd], get(group_property_map, vd)); + } + + return reduced; +} + } // namespace reactor diff --git a/lib/environment.cc b/lib/environment.cc index 5d309cfb..4f69ba31 100644 --- a/lib/environment.cc +++ b/lib/environment.cc @@ -58,6 +58,9 @@ void Environment::assemble() { GroupedDependencyGraph grouped_graph{reduced_graph}; grouped_graph.group_reactions_by_container(top_level_reactors_); grouped_graph.export_graphviz("grouped_graph.dot"); + + GroupedDependencyGraph reduced_grouped_graph = grouped_graph.transitive_reduction(); + reduced_grouped_graph.export_graphviz("reduced_grouped_graph.dot"); } void Environment::build_dependency_graph(Reactor* reactor) { // NOLINT From febe19f50802e2a890252d7310aa1e309058e952 Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Wed, 6 Jul 2022 15:17:38 +0200 Subject: [PATCH 15/49] fix segfaults removing vertices from the graph invalidates all iterators and descriptors if the underlying data structure is a vector. This fix implements a better strategy for deleting vertices. --- include/reactor-cpp/dependency_graph.hh | 16 +++++++ lib/dependency_graph.cc | 58 +++++++++++++++++++------ 2 files changed, 61 insertions(+), 13 deletions(-) diff --git a/include/reactor-cpp/dependency_graph.hh b/include/reactor-cpp/dependency_graph.hh index 3a450ff3..19c3b8da 100644 --- a/include/reactor-cpp/dependency_graph.hh +++ b/include/reactor-cpp/dependency_graph.hh @@ -6,6 +6,8 @@ #include #include +#include +#include #include namespace reactor { @@ -80,8 +82,22 @@ private: } }; + class NonEmptyGoupFilter { + private: + GroupPropertyMap property_map; + + public: + NonEmptyGoupFilter() = default; + NonEmptyGoupFilter(const GroupPropertyMap& map) + : property_map{map} {} + + auto operator()(GroupGraph::vertex_descriptor vertex) const -> bool { return !boost::get(property_map, vertex).empty(); } + }; + void group_reactions_by_container_helper(const Reactor* reactor); + void clear_all_empty_vertices(); + GroupedDependencyGraph() = default; public: diff --git a/lib/dependency_graph.cc b/lib/dependency_graph.cc index 2077781c..c7eef9b6 100644 --- a/lib/dependency_graph.cc +++ b/lib/dependency_graph.cc @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -175,17 +176,20 @@ void GroupedDependencyGraph::export_graphviz(const std::string& file_name) { ss << '}'; return ss.str(); })); - dp.property("fillcolor", - make_function_property_map( - [&group_proprty_map](GroupGraph::vertex_descriptor vertex) { - auto hash = std::hash{}(boost::get(group_proprty_map, vertex)[0]->container()->fqn()); - auto red = (hash & 0xff0000) >> 16; // NOLINT - auto green = (hash & 0x00ff00) >> 9; // NOLINT - auto blue = (hash & 0x0000ff); // NOLINT - std::stringstream ss; - ss << "#" << std::setfill('0') << std::setw(2) << std::hex << red << green << blue; - return ss.str(); - })); + dp.property("fillcolor", make_function_property_map( + [&group_proprty_map](GroupGraph::vertex_descriptor vertex) { + auto& reactions = boost::get(group_proprty_map, vertex); + if (!reactions.empty()) { + auto hash = std::hash{}(reactions[0]->container()->fqn()); + auto red = (hash & 0xff0000) >> 16; // NOLINT + auto green = (hash & 0x00ff00) >> 9; // NOLINT + auto blue = (hash & 0x0000ff); // NOLINT + std::stringstream ss; + ss << "#" << std::setfill('0') << std::setw(2) << std::hex << red << green << blue; + return ss.str(); + } + return std::string{"#ffffff"}; + })); dp.property("style", make_constant_property("filled")); std::ofstream dot_file(file_name); @@ -227,15 +231,43 @@ void GroupedDependencyGraph::try_contract_edge(const Reaction* a, const Reaction vertex_map[reaction] = va; } - // delete vb + // and remove all reactions listed in vb making it empty + vb_reactions.clear(); + + // remove all the edges from/to vb, but don't remove vb itself yet. Removing it would invalidate all the vertex + // descriptors and iterators. Its better to later remove all empty vertices in a single go. clear_vertex(vb, graph); - remove_vertex(vb, graph); } void GroupedDependencyGraph::group_reactions_by_container(const std::set& top_level_reactors) { for (const auto* reactor : top_level_reactors) { group_reactions_by_container_helper(reactor); } + + clear_all_empty_vertices(); +} + +void GroupedDependencyGraph::clear_all_empty_vertices() { + // First, get a filtered view of the graph only showing the non-empty vertices + NonEmptyGoupFilter filter{get_group_property_map()}; + filtered_graph filtered_graph{graph, keep_all(), filter}; + + // Create a new graph and copy the contents from the filtered view. + GroupGraph new_graph{}; + copy_graph(filtered_graph, new_graph); + + // Replace our current internal graph and clear the old one + graph.swap(new_graph); + new_graph.clear(); + + // update the vertex map with the new vertex descriptors + vertex_map.clear(); + for (GroupGraph::vertex_descriptor vd : boost::make_iterator_range(vertices(graph))) { + const auto& reactions = get(get_group_property_map(), vd); + for (const auto* reaction : reactions) { + vertex_map[reaction] = vd; + } + } } void GroupedDependencyGraph::group_reactions_by_container_helper(const Reactor* reactor) { From 442fed3acdb35a08d2aab6df8eaf3a93a6deff17 Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Wed, 6 Jul 2022 15:27:53 +0200 Subject: [PATCH 16/49] fix a potential bug --- include/reactor-cpp/dependency_graph.hh | 6 ++++-- lib/dependency_graph.cc | 12 ++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/include/reactor-cpp/dependency_graph.hh b/include/reactor-cpp/dependency_graph.hh index 19c3b8da..35ebdd0a 100644 --- a/include/reactor-cpp/dependency_graph.hh +++ b/include/reactor-cpp/dependency_graph.hh @@ -41,7 +41,8 @@ public: void export_graphviz(const std::string& file_name); - [[nodiscard]] auto transitive_reduction() const -> ReactionDependencyGraph; + // TODO: This should be const, but I don't know how to get immutable access to the reaction graph properties... + [[nodiscard]] auto transitive_reduction() -> ReactionDependencyGraph; friend GroupedDependencyGraph; }; @@ -111,7 +112,8 @@ public: void group_reactions_by_container(const std::set& top_level_reactors); - [[nodiscard]] auto transitive_reduction() const -> GroupedDependencyGraph; + // TODO: This should be const, but I don't know how to get immutable access to the reaction graph properties... + [[nodiscard]] auto transitive_reduction() -> GroupedDependencyGraph; }; } // namespace reactor diff --git a/lib/dependency_graph.cc b/lib/dependency_graph.cc index c7eef9b6..1c18fcad 100644 --- a/lib/dependency_graph.cc +++ b/lib/dependency_graph.cc @@ -86,7 +86,7 @@ ReactionDependencyGraph::ReactionDependencyGraph(const std::set& top_l } } -auto ReactionDependencyGraph::transitive_reduction() const -> ReactionDependencyGraph { +auto ReactionDependencyGraph::transitive_reduction() -> ReactionDependencyGraph { ReactionDependencyGraph reduced{}; // transitive_reduction uses this to populate a mapping from original vertices to new vertices in the reduced graph @@ -103,9 +103,8 @@ auto ReactionDependencyGraph::transitive_reduction() const -> ReactionDependency make_assoc_property_map(id_map)); // update the mapping of reactions to vertices - auto reaction_property_map = reduced.get_reaction_property_map(); for (ReactionGraph::vertex_descriptor vd : boost::make_iterator_range(vertices(graph))) { - put(reaction_property_map, graph_to_reduced_graph[vd], get(reaction_property_map, vd)); + put(reduced.get_reaction_property_map(), graph_to_reduced_graph[vd], get(get_reaction_property_map(), vd)); } return reduced; @@ -288,7 +287,7 @@ void GroupedDependencyGraph::group_reactions_by_container_helper(const Reactor* } } -auto GroupedDependencyGraph::transitive_reduction() const -> GroupedDependencyGraph { +auto GroupedDependencyGraph::transitive_reduction() -> GroupedDependencyGraph { GroupedDependencyGraph reduced{}; // transitive_reduction uses this to populate a mapping from original vertices to new vertices in the reduced graph @@ -305,9 +304,10 @@ auto GroupedDependencyGraph::transitive_reduction() const -> GroupedDependencyGr make_assoc_property_map(id_map)); // update the mapping of reactions to vertices - auto group_property_map = reduced.get_group_property_map(); for (GroupGraph::vertex_descriptor vd : boost::make_iterator_range(vertices(graph))) { - put(group_property_map, graph_to_reduced_graph[vd], get(group_property_map, vd)); + auto& reactions = boost::get(get_group_property_map(), vd); + auto& reduced_reactions = boost::get(reduced.get_group_property_map(), graph_to_reduced_graph[vd]); + reduced_reactions.insert(reduced_reactions.end(), reactions.begin(), reactions.end()); } return reduced; From 2003b589730fb979f9a8b0d605790fe92770a0d8 Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Wed, 6 Jul 2022 15:55:13 +0200 Subject: [PATCH 17/49] bugfix in graphviz output --- lib/dependency_graph.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/dependency_graph.cc b/lib/dependency_graph.cc index 1c18fcad..67e4e413 100644 --- a/lib/dependency_graph.cc +++ b/lib/dependency_graph.cc @@ -168,9 +168,9 @@ void GroupedDependencyGraph::export_graphviz(const std::string& file_name) { dp.property("tooltip", make_function_property_map( [&group_proprty_map](GroupGraph::vertex_descriptor vertex) { std::stringstream ss; + ss << "reactions: \n"; for (const auto* reaction : boost::get(group_proprty_map, vertex)) { - ss << "reactions: \n"; - ss << " - " << reaction->fqn(); + ss << " - " << reaction->fqn() << '\n'; } ss << '}'; return ss.str(); From 192d5b0f59db4f1de840f6a4594c90cb3a5571bf Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Wed, 6 Jul 2022 16:28:53 +0200 Subject: [PATCH 18/49] finally group chains --- include/reactor-cpp/dependency_graph.hh | 4 ++- lib/dependency_graph.cc | 39 ++++++++++++++++++++----- lib/environment.cc | 3 ++ 3 files changed, 37 insertions(+), 9 deletions(-) diff --git a/include/reactor-cpp/dependency_graph.hh b/include/reactor-cpp/dependency_graph.hh index 35ebdd0a..b25cc41d 100644 --- a/include/reactor-cpp/dependency_graph.hh +++ b/include/reactor-cpp/dependency_graph.hh @@ -108,10 +108,12 @@ public: void export_graphviz(const std::string& file_name); - void try_contract_edge(const Reaction* a, const Reaction* b); + void try_contract_edge(GroupGraph::vertex_descriptor va, GroupGraph::vertex_descriptor vb); void group_reactions_by_container(const std::set& top_level_reactors); + void group_chains(); + // TODO: This should be const, but I don't know how to get immutable access to the reaction graph properties... [[nodiscard]] auto transitive_reduction() -> GroupedDependencyGraph; }; diff --git a/lib/dependency_graph.cc b/lib/dependency_graph.cc index 67e4e413..fdde0b24 100644 --- a/lib/dependency_graph.cc +++ b/lib/dependency_graph.cc @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -16,6 +17,7 @@ #include #include +#include #include #include #include @@ -158,9 +160,19 @@ void GroupedDependencyGraph::export_graphviz(const std::string& file_name) { dp.property("label", make_function_property_map( [&group_proprty_map](GroupGraph::vertex_descriptor vertex) { std::stringstream ss; - for (const auto* reaction : boost::get(group_proprty_map, vertex)) { - auto sep = ss.tellp() == 0 ? '{' : ','; - ss << sep << reaction->name(); + std::size_t count{0}; + const auto& reactions = boost::get(group_proprty_map, vertex); + + ss << '{'; + for (const auto* reaction : reactions) { + count++; + ss << reaction->name(); + if (count != reactions.size()) { + ss << ','; + if (count % 4 == 0) { + ss << '\n'; + } + } } ss << '}'; return ss.str(); @@ -195,10 +207,7 @@ void GroupedDependencyGraph::export_graphviz(const std::string& file_name) { write_graphviz_dp(dot_file, graph, dp); } -void GroupedDependencyGraph::try_contract_edge(const Reaction* a, const Reaction* b) { - GroupGraph::vertex_descriptor va = vertex_map[a]; - GroupGraph::vertex_descriptor vb = vertex_map[b]; - +void GroupedDependencyGraph::try_contract_edge(GroupGraph::vertex_descriptor va, GroupGraph::vertex_descriptor vb) { // if both vertexes are the same we can abort... if (va == vb) { return; @@ -276,7 +285,7 @@ void GroupedDependencyGraph::group_reactions_by_container_helper(const Reactor* auto it = reactions.begin(); auto next = std::next(it); while (next != reactions.end()) { - try_contract_edge(*it, *next); + try_contract_edge(vertex_map.at(*it), vertex_map.at(*next)); it = next; next = std::next(it); } @@ -313,4 +322,18 @@ auto GroupedDependencyGraph::transitive_reduction() -> GroupedDependencyGraph { return reduced; } +void GroupedDependencyGraph::group_chains() { + for (GroupGraph::vertex_descriptor vd : boost::make_iterator_range(vertices(graph))) { + if (in_degree(vd, graph) == 1) { + auto edge_iterator = in_edges(vd, graph).first; + GroupGraph::vertex_descriptor vs = source(*edge_iterator, graph); + if (out_degree(vs, graph) == 1) { + try_contract_edge(vs, vd); + } + } + } + + clear_all_empty_vertices(); +} + } // namespace reactor diff --git a/lib/environment.cc b/lib/environment.cc index 4f69ba31..b8e232d6 100644 --- a/lib/environment.cc +++ b/lib/environment.cc @@ -61,6 +61,9 @@ void Environment::assemble() { GroupedDependencyGraph reduced_grouped_graph = grouped_graph.transitive_reduction(); reduced_grouped_graph.export_graphviz("reduced_grouped_graph.dot"); + + reduced_grouped_graph.group_chains(); + reduced_grouped_graph.export_graphviz("grouped_chains_graph.dot"); } void Environment::build_dependency_graph(Reactor* reactor) { // NOLINT From 43aec941f61c3f9565489c7f90f43efa6deb5fe8 Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Fri, 8 Jul 2022 13:36:45 +0200 Subject: [PATCH 19/49] check if a contraction would introduce a cycle --- include/reactor-cpp/dependency_graph.hh | 6 +++++- lib/dependency_graph.cc | 23 +++++++++++++++++++++-- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/include/reactor-cpp/dependency_graph.hh b/include/reactor-cpp/dependency_graph.hh index b25cc41d..b125f77c 100644 --- a/include/reactor-cpp/dependency_graph.hh +++ b/include/reactor-cpp/dependency_graph.hh @@ -92,7 +92,9 @@ private: NonEmptyGoupFilter(const GroupPropertyMap& map) : property_map{map} {} - auto operator()(GroupGraph::vertex_descriptor vertex) const -> bool { return !boost::get(property_map, vertex).empty(); } + auto operator()(GroupGraph::vertex_descriptor vertex) const -> bool { + return !boost::get(property_map, vertex).empty(); + } }; void group_reactions_by_container_helper(const Reactor* reactor); @@ -108,6 +110,8 @@ public: void export_graphviz(const std::string& file_name); + auto has_path(GroupGraph::vertex_descriptor va, GroupGraph::vertex_descriptor vb) const -> bool; + void try_contract_edge(GroupGraph::vertex_descriptor va, GroupGraph::vertex_descriptor vb); void group_reactions_by_container(const std::set& top_level_reactors); diff --git a/lib/dependency_graph.cc b/lib/dependency_graph.cc index fdde0b24..1616f90a 100644 --- a/lib/dependency_graph.cc +++ b/lib/dependency_graph.cc @@ -221,6 +221,14 @@ void GroupedDependencyGraph::try_contract_edge(GroupGraph::vertex_descriptor va, // remove the direct edge between va and vb remove_edge(va, vb, graph); + // check if there is still a path from va to vb; abort in this case as we would introduce a cycle when contracting the + // edge + if (has_path(va, vb)) { + // add the original edge back and abort + add_edge(va, vb, graph); + return; + } + // route all edges in/out of vb to va for (auto edge : make_iterator_range(out_edges(vb, graph))) { add_edge(va, target(edge, graph), graph); @@ -325,8 +333,7 @@ auto GroupedDependencyGraph::transitive_reduction() -> GroupedDependencyGraph { void GroupedDependencyGraph::group_chains() { for (GroupGraph::vertex_descriptor vd : boost::make_iterator_range(vertices(graph))) { if (in_degree(vd, graph) == 1) { - auto edge_iterator = in_edges(vd, graph).first; - GroupGraph::vertex_descriptor vs = source(*edge_iterator, graph); + GroupGraph::vertex_descriptor vs = source(*in_edges(vd, graph).first, graph); if (out_degree(vs, graph) == 1) { try_contract_edge(vs, vd); } @@ -336,4 +343,16 @@ void GroupedDependencyGraph::group_chains() { clear_all_empty_vertices(); } +#ifndef __clang_analyzer__ // exclude this from static analysis as breadth_first_search appears to include a bug... +auto GroupedDependencyGraph::has_path(GroupGraph::vertex_descriptor va, GroupGraph::vertex_descriptor vb) const + -> bool { + try { + breadth_first_search(graph, va, visitor(ReachabilityVisitor(vb))); + } catch (const ReachabilityVisitor::PathExists& e) { + return true; + } + return false; +} +#endif + } // namespace reactor From 97f9eb12f160b4a1b6e592f0212b2178f7afcfed Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Tue, 12 Jul 2022 14:25:03 +0200 Subject: [PATCH 20/49] annotate dependency types in basic graph --- include/reactor-cpp/dependency_graph.hh | 13 +++++++++- lib/dependency_graph.cc | 32 ++++++++++++++++++++++--- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/include/reactor-cpp/dependency_graph.hh b/include/reactor-cpp/dependency_graph.hh index b125f77c..aa700499 100644 --- a/include/reactor-cpp/dependency_graph.hh +++ b/include/reactor-cpp/dependency_graph.hh @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -14,15 +15,22 @@ namespace reactor { class GroupedDependencyGraph; +enum class DependencyType { Undefined, Priority, Trigger, Effect }; + class ReactionDependencyGraph { private: struct reaction_info_t { using kind = boost::vertex_property_tag; }; + struct dependency_info_t { + using kind = boost::edge_property_tag; + }; using ReactionProperty = boost::property; - using ReactionGraph = boost::directed_graph; + using DependencyProperty = boost::property; + using ReactionGraph = boost::directed_graph; using ReactionToVertexMap = std::map; using ReactionPropertyMap = boost::property_map::type; + using DependencyPropertyMap = boost::property_map::type; ReactionGraph graph{}; ReactionToVertexMap vertex_map{}; @@ -33,6 +41,9 @@ private: void populate_graph_with_dependency_edges(const Reactor* reactor); [[nodiscard]] auto get_reaction_property_map() -> ReactionPropertyMap { return boost::get(reaction_info_t{}, graph); } + [[nodiscard]] auto get_dependency_property_map() -> DependencyPropertyMap { + return boost::get(dependency_info_t{}, graph); + } ReactionDependencyGraph() = default; diff --git a/lib/dependency_graph.cc b/lib/dependency_graph.cc index 1616f90a..9b5d425a 100644 --- a/lib/dependency_graph.cc +++ b/lib/dependency_graph.cc @@ -45,7 +45,8 @@ void ReactionDependencyGraph::populate_graph_with_priority_edges(const Reactor* auto iterator = reactions.begin(); auto next = std::next(iterator); while (next != reactions.end()) { - graph.add_edge(vertex_map.at(*iterator), vertex_map.at(*next)); + auto edge = graph.add_edge(vertex_map.at(*iterator), vertex_map.at(*next)).first; + put(get_dependency_property_map(), edge, DependencyType::Priority); iterator = next; next = std::next(iterator); } @@ -64,7 +65,12 @@ void ReactionDependencyGraph::populate_graph_with_dependency_edges(const Reactor source = source->inward_binding(); } for (auto* antidependency : source->antidependencies()) { - graph.add_edge(vertex_map.at(antidependency), vertex_map.at(reaction)); + auto edge = graph.add_edge(vertex_map.at(antidependency), vertex_map.at(reaction)).first; + if (reaction->port_triggers().count(dependency) == 0) { + put(get_dependency_property_map(), edge, DependencyType::Effect); + } else { + put(get_dependency_property_map(), edge, DependencyType::Trigger); + } } } } @@ -114,6 +120,7 @@ auto ReactionDependencyGraph::transitive_reduction() -> ReactionDependencyGraph void ReactionDependencyGraph::export_graphviz(const std::string& file_name) { auto reaction_proprty_map = get_reaction_property_map(); + auto dependency_proprty_map = get_dependency_property_map(); dynamic_properties dp; dp.property("node_id", get(boost::vertex_index, graph)); dp.property("label", make_function_property_map( @@ -136,6 +143,21 @@ void ReactionDependencyGraph::export_graphviz(const std::string& file_name) { return ss.str(); })); dp.property("style", make_constant_property("filled")); + dp.property("style", make_function_property_map( + [&dependency_proprty_map](ReactionGraph::edge_descriptor edge) { + auto dependency_type = boost::get(dependency_proprty_map, edge); + switch (dependency_type) { + case DependencyType::Undefined: + return "solid"; + case DependencyType::Priority: + return "dotted"; + case DependencyType::Effect: + return "dashed"; + case DependencyType::Trigger: + return "bold"; + } + return "invis"; + })); std::ofstream dot_file(file_name); write_graphviz_dp(dot_file, graph, dp); @@ -150,7 +172,11 @@ GroupedDependencyGraph::GroupedDependencyGraph(ReactionDependencyGraph& reaction group.push_back(boost::get(reactionGraph.get_reaction_property_map(), in)); boost::put(get_group_property_map(), out, group); vertex_map[group[0]] = out; - })); + }) + .edge_copy([]([[maybe_unused]] ReactionDependencyGraph::ReactionGraph::edge_descriptor in, + [[maybe_unused]] GroupGraph::edge_descriptor out) { + // do nothing; simply drop the edge descriptors + })); } void GroupedDependencyGraph::export_graphviz(const std::string& file_name) { From 16b4708564044e461b8069e71a2448de573936ea Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Tue, 12 Jul 2022 14:32:11 +0200 Subject: [PATCH 21/49] expose the grouped dependency graph from the environment --- include/reactor-cpp/dependency_graph.hh | 3 +-- include/reactor-cpp/environment.hh | 5 +++++ lib/environment.cc | 2 ++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/include/reactor-cpp/dependency_graph.hh b/include/reactor-cpp/dependency_graph.hh index aa700499..88034671 100644 --- a/include/reactor-cpp/dependency_graph.hh +++ b/include/reactor-cpp/dependency_graph.hh @@ -112,12 +112,11 @@ private: void clear_all_empty_vertices(); - GroupedDependencyGraph() = default; - public: // TODO: This should be a const reference, but I don't know how to get immutable access to the reaction graph // properties... GroupedDependencyGraph(ReactionDependencyGraph& reactionGraph); + GroupedDependencyGraph() = default; void export_graphviz(const std::string& file_name); diff --git a/include/reactor-cpp/environment.hh b/include/reactor-cpp/environment.hh index 8679f26f..f5fe7bc8 100644 --- a/include/reactor-cpp/environment.hh +++ b/include/reactor-cpp/environment.hh @@ -13,6 +13,7 @@ #include #include +#include "reactor-cpp/dependency_graph.hh" #include "reactor.hh" #include "scheduler.hh" @@ -42,6 +43,8 @@ private: Phase phase_{Phase::Construction}; TimePoint start_time_{}; + GroupedDependencyGraph grouped_graph_{}; + void build_dependency_graph(Reactor* reactor); void calculate_indexes(); @@ -83,6 +86,8 @@ public: [[nodiscard]] auto fast_fwd_execution() const noexcept -> bool { return fast_fwd_execution_; } [[nodiscard]] auto run_forever() const noexcept -> bool { return run_forever_; } [[nodiscard]] auto max_reaction_index() const noexcept -> unsigned int { return max_reaction_index_; } + + [[nodiscard]] auto grouped_graph() const noexcept -> const GroupedDependencyGraph& { return grouped_graph_; } }; } // namespace reactor diff --git a/lib/environment.cc b/lib/environment.cc index b8e232d6..f48d70d5 100644 --- a/lib/environment.cc +++ b/lib/environment.cc @@ -64,6 +64,8 @@ void Environment::assemble() { reduced_grouped_graph.group_chains(); reduced_grouped_graph.export_graphviz("grouped_chains_graph.dot"); + + this->grouped_graph_ = reduced_grouped_graph; } void Environment::build_dependency_graph(Reactor* reactor) { // NOLINT From 58e6ffeac306c9c93de8532666a1ea8cbcbe8822 Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Tue, 12 Jul 2022 14:56:28 +0200 Subject: [PATCH 22/49] copy the original scheduler to grouped_scheduler --- include/reactor-cpp/action.hh | 1 + include/reactor-cpp/environment.hh | 11 +- include/reactor-cpp/fwd.hh | 1 + include/reactor-cpp/group_scheduler.hh | 145 +++++++++ include/reactor-cpp/port.hh | 1 + lib/CMakeLists.txt | 1 + lib/group_scheduler.cc | 398 +++++++++++++++++++++++++ 7 files changed, 553 insertions(+), 5 deletions(-) create mode 100644 include/reactor-cpp/group_scheduler.hh create mode 100644 lib/group_scheduler.cc diff --git a/include/reactor-cpp/action.hh b/include/reactor-cpp/action.hh index 129f6526..44055bb2 100644 --- a/include/reactor-cpp/action.hh +++ b/include/reactor-cpp/action.hh @@ -45,6 +45,7 @@ public: friend class Reaction; friend class Scheduler; + friend class GroupScheduler; }; template class Action : public BaseAction { diff --git a/include/reactor-cpp/environment.hh b/include/reactor-cpp/environment.hh index f5fe7bc8..0c70489c 100644 --- a/include/reactor-cpp/environment.hh +++ b/include/reactor-cpp/environment.hh @@ -11,11 +11,12 @@ #include #include +#include #include -#include "reactor-cpp/dependency_graph.hh" +#include "dependency_graph.hh" +#include "group_scheduler.hh" #include "reactor.hh" -#include "scheduler.hh" namespace reactor { @@ -39,7 +40,7 @@ private: std::set reactions_{}; std::vector dependencies_{}; - Scheduler scheduler_; + GroupScheduler scheduler_; Phase phase_{Phase::Construction}; TimePoint start_time_{}; @@ -73,9 +74,9 @@ public: [[nodiscard]] auto top_level_reactors() const noexcept -> const auto& { return top_level_reactors_; } [[nodiscard]] auto phase() const noexcept -> Phase { return phase_; } - [[nodiscard]] auto scheduler() const noexcept -> const Scheduler* { return &scheduler_; } + [[nodiscard]] auto scheduler() const noexcept -> const GroupScheduler* { return &scheduler_; } - auto scheduler() noexcept -> Scheduler* { return &scheduler_; } + auto scheduler() noexcept -> GroupScheduler* { return &scheduler_; } [[nodiscard]] auto logical_time() const noexcept -> const LogicalTime& { return scheduler_.logical_time(); } [[nodiscard]] auto start_time() const noexcept -> const TimePoint& { return start_time_; } diff --git a/include/reactor-cpp/fwd.hh b/include/reactor-cpp/fwd.hh index 52edced4..a82053d0 100644 --- a/include/reactor-cpp/fwd.hh +++ b/include/reactor-cpp/fwd.hh @@ -17,6 +17,7 @@ class Environment; class Reaction; class Reactor; class Scheduler; +class GroupScheduler; class Tag; template class Action; diff --git a/include/reactor-cpp/group_scheduler.hh b/include/reactor-cpp/group_scheduler.hh new file mode 100644 index 00000000..38a53f60 --- /dev/null +++ b/include/reactor-cpp/group_scheduler.hh @@ -0,0 +1,145 @@ +/* + * Copyright (C) 2019 TU Dresden + * All rights reserved. + * + * Authors: + * Christian Menard + */ + +#ifndef REACTOR_CPP_GROUP_SCHEDULER_HH +#define REACTOR_CPP_GROUP_SCHEDULER_HH + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "fwd.hh" +#include "logical_time.hh" +#include "semaphore.hh" + +namespace reactor { + +// forward declarations +class GroupScheduler; +class GroupWorker; + +class GroupWorker { // NOLINT +public: + GroupScheduler& scheduler_; + const unsigned int identity_{0}; + std::thread thread_{}; + + // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) + static thread_local const GroupWorker* current_worker; + + void work() const; + void execute_reaction(Reaction* reaction) const; + + GroupWorker(GroupScheduler& scheduler, unsigned int identity) + : scheduler_{scheduler} + , identity_{identity} {} + GroupWorker(GroupWorker&& worker); // NOLINT(performance-noexcept-move-constructor) + GroupWorker(const GroupWorker& worker) = delete; + + void start_thread() { thread_ = std::thread(&GroupWorker::work, this); } + void join_thread() { thread_.join(); } + + static auto current_worker_id() -> unsigned { return current_worker->identity_; } +}; + +class GroupReadyQueue { +private: + std::vector queue_{}; + std::atomic size_{0}; + Semaphore sem_{0}; + std::ptrdiff_t waiting_workers_{0}; + const unsigned int num_workers_; + +public: + explicit GroupReadyQueue(unsigned num_workers) + : num_workers_(num_workers) {} + + /** + * Retrieve a ready reaction from the queue. + * + * This method may be called concurrently. In case the queue is empty, the + * method blocks and waits until a ready reaction becomes available. + */ + auto pop() -> Reaction*; + + /** + * Fill the queue up with ready reactions. + * + * This method assumes that the internal queue is empty. It moves all + * reactions from the provided `ready_reactions` vector to the internal + * queue, leaving `ready_reactions` empty. + * + * Note that this method is not thread-safe. The caller needs to ensure that + * no other thread will try to read from the queue during this operation. + */ + void fill_up(std::vector& ready_reactions); +}; + +using EventMap = std::map>; + +class GroupScheduler { // NOLINT +private: + const bool using_workers_; + LogicalTime logical_time_{}; + + Environment* environment_; + std::vector workers_{}; + + std::mutex scheduling_mutex_; + std::unique_lock scheduling_lock_{scheduling_mutex_, std::defer_lock}; + std::condition_variable cv_schedule_; + + std::mutex lock_event_queue_; + std::map event_queue_; + + std::vector> set_ports_; + std::vector> triggered_reactions_; + + std::vector> reaction_queue_; + unsigned int reaction_queue_pos_{std::numeric_limits::max()}; + + GroupReadyQueue ready_queue_; + std::atomic reactions_to_process_{0}; // NOLINT + + std::atomic stop_{false}; + bool continue_execution_{true}; + + void schedule() noexcept; + auto schedule_ready_reactions() -> bool; + void next(); + void terminate_all_workers(); + void set_port_helper(BasePort* port); + +public: + explicit GroupScheduler(Environment* env); + ~GroupScheduler(); + + void schedule_sync(const Tag& tag, BaseAction* action, std::function pre_handler); + void schedule_async(const Tag& tag, BaseAction* action, std::function pre_handler); + + void inline lock() noexcept { scheduling_lock_.lock(); } + void inline unlock() noexcept { scheduling_lock_.unlock(); } + + void set_port(BasePort* port); + + [[nodiscard]] inline auto logical_time() const noexcept -> const auto& { return logical_time_; } + + void start(); + void stop(); + + friend GroupWorker; +}; + +} // namespace reactor + +#endif // REACTOR_CPP_GROUP_SCHEDULER_HH diff --git a/include/reactor-cpp/port.hh b/include/reactor-cpp/port.hh index f2345532..bf031927 100644 --- a/include/reactor-cpp/port.hh +++ b/include/reactor-cpp/port.hh @@ -57,6 +57,7 @@ public: friend class Reaction; friend class Scheduler; + friend class GroupScheduler; }; template class Port : public BasePort { diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index 95f25892..36365f22 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -3,6 +3,7 @@ set(SOURCE_FILES assert.cc dependency_graph.cc environment.cc + group_scheduler.cc logical_time.cc port.cc reaction.cc diff --git a/lib/group_scheduler.cc b/lib/group_scheduler.cc new file mode 100644 index 00000000..0ef2a9fd --- /dev/null +++ b/lib/group_scheduler.cc @@ -0,0 +1,398 @@ +/* + * Copyright (C) 2019 TU Dresden + * All rights reserved. + * + * Authors: + * Christian Menard + */ + +#include + +#include "reactor-cpp/group_scheduler.hh" + +#include "reactor-cpp/action.hh" +#include "reactor-cpp/assert.hh" +#include "reactor-cpp/environment.hh" +#include "reactor-cpp/logging.hh" +#include "reactor-cpp/port.hh" +#include "reactor-cpp/reaction.hh" +#include "reactor-cpp/trace.hh" + +namespace reactor { + +// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) +thread_local const GroupWorker* GroupWorker::current_worker = nullptr; + +GroupWorker::GroupWorker(GroupWorker&& work) // NOLINT(performance-noexcept-move-constructor) + : scheduler_{work.scheduler_} + , identity_{work.identity_} { + // Need to provide the move constructor in order to organize workers in a + // std::vector. However, moving is not save if the thread is already running, + // thus we throw an exception here if the worker is moved but the + // internal thread is already running. + + if (work.thread_.joinable()) { + throw std::runtime_error{"Running workers cannot be moved!"}; + } +} + +void GroupWorker::work() const { + // initialize the current worker thread local variable + current_worker = this; + + log::Debug() << "(GroupWorker " << this->identity_ << ") Starting"; + + if (identity_ == 0) { + log::Debug() << "(GroupWorker 0) do the initial scheduling"; + scheduler_.schedule(); + } + + while (true) { + // wait for a ready reaction + auto* reaction = scheduler_.ready_queue_.pop(); + + // receiving a nullptr indicates that the worker should terminate + if (reaction == nullptr) { + break; + } + + // execute the reaction + execute_reaction(reaction); + + // was this the very last reaction? + if (scheduler_.reactions_to_process_.fetch_sub(1, std::memory_order_acq_rel) == 1) { + // Yes, then schedule. The atomic decrement above ensures that only one + // thread enters this block. + scheduler_.schedule(); + } + // continue otherwise + } + + log::Debug() << "(GroupWorker " << identity_ << ") terminates"; +} + +void GroupWorker::execute_reaction(Reaction* reaction) const { + log::Debug() << "(GroupWorker " << identity_ << ") " + << "execute reaction " << reaction->fqn(); + + tracepoint(reactor_cpp, reaction_execution_starts, id, reaction->fqn(), scheduler.logical_time()); + reaction->trigger(); + tracepoint(reactor_cpp, reaction_execution_finishes, id, reaction->fqn(), scheduler.logical_time()); +} + +void GroupScheduler::schedule() noexcept { + bool found_ready_reactions = schedule_ready_reactions(); + + while (!found_ready_reactions) { + log::Debug() << "(GroupScheduler) call next()"; + next(); + reaction_queue_pos_ = 0; + + found_ready_reactions = schedule_ready_reactions(); + + if (!continue_execution_ && !found_ready_reactions) { + // let all workers know that they should terminate + terminate_all_workers(); + break; + } + } +} + +auto GroupReadyQueue::pop() -> Reaction* { + auto old_size = size_.fetch_sub(1, std::memory_order_acq_rel); + + // If there is no ready reaction available, wait until there is one. + while (old_size <= 0) { + log::Debug() << "(GroupWorker " << GroupWorker::current_worker_id() << ") Wait for work"; + sem_.acquire(); + log::Debug() << "(GroupWorker " << GroupWorker::current_worker_id() << ") Waking up"; + old_size = size_.fetch_sub(1, std::memory_order_acq_rel); + // FIXME: Protect against underflow? + } + + auto pos = old_size - 1; + return queue_[pos]; +} + +void GroupReadyQueue::fill_up(std::vector& ready_reactions) { + // clear the internal queue and swap contents + queue_.clear(); + queue_.swap(ready_reactions); + + // update the atomic size counter and release the semaphore to wake up + // waiting worker threads + auto new_size = static_cast(queue_.size()); + auto old_size = size_.exchange(new_size, std::memory_order_acq_rel); + + // calculate how many workers to wake up. -old_size indicates the number of + // workers who started waiting since the last update. + // We want to wake up at most all the waiting workers. If we would release + // more, other workers that are out of work would not block when acquiring + // the semaphore. + // Also, we do not want to wake up more workers than there is work. new_size + // indicates the number of ready reactions. Since there is always at least + // one worker running running, new_size - running_workers indicates the + // number of additional workers needed to process all reactions. + waiting_workers_ += -old_size; + auto running_workers = num_workers_ - waiting_workers_; + auto workers_to_wakeup = std::min(waiting_workers_, new_size - running_workers); + + // wakeup other workers_ + if (workers_to_wakeup > 0) { + waiting_workers_ -= workers_to_wakeup; + log::Debug() << "Wakeup " << workers_to_wakeup << " workers"; + sem_.release(static_cast(workers_to_wakeup)); + } +} + +void GroupScheduler::terminate_all_workers() { + log::Debug() << "(GroupScheduler) Send termination signal to all workers"; + auto num_workers = environment_->num_workers(); + std::vector null_reactions{num_workers, nullptr}; + log::Debug() << null_reactions.size(); + ready_queue_.fill_up(null_reactions); +} + +auto GroupScheduler::schedule_ready_reactions() -> bool { + // insert any triggered reactions_ into the reaction queue + for (auto& vec_reaction : triggered_reactions_) { + for (auto* reaction : vec_reaction) { + reaction_queue_[reaction->index()].push_back(reaction); + } + vec_reaction.clear(); + } + + log::Debug() << "(GroupScheduler) Scanning the reaction queue for ready reactions"; + + // continue iterating over the reaction queue + for (; reaction_queue_pos_ < reaction_queue_.size(); reaction_queue_pos_++) { + auto& reactions = reaction_queue_[reaction_queue_pos_]; + + // any ready reactions of current priority? + if (!reactions.empty()) { + log::Debug() << "(GroupScheduler) Process reactions of priority " << reaction_queue_pos_; + + // Make sure that any reaction is only executed once even if it + // was triggered multiple times. + std::sort(reactions.begin(), reactions.end()); + reactions.erase(std::unique(reactions.begin(), reactions.end()), reactions.end()); + + if constexpr (log::debug_enabled || tracing_enabled) { // NOLINT + for (auto* reaction : reactions) { + log::Debug() << "(GroupScheduler) Reaction " << reaction->fqn() << " is ready for execution"; + tracepoint(reactor_cpp, trigger_reaction, reaction->container()->fqn(), reaction->name(), logical_time_); + } + } + + reactions_to_process_.store(static_cast(reactions.size()), std::memory_order_release); + ready_queue_.fill_up(reactions); + + // break out of the loop and return + return true; + } + } + + log::Debug() << "(GroupScheduler) Reached end of reaction queue"; + return false; +} + +void GroupScheduler::start() { + log::Debug() << "Starting the scheduler..."; + + auto num_workers = environment_->num_workers(); + // initialize the reaction queue, set ports vector, and triggered reactions + // vector + reaction_queue_.resize(environment_->max_reaction_index() + 1); + set_ports_.resize(num_workers); + triggered_reactions_.resize(num_workers); + + // Initialize and start the workers. By resizing the workers vector first, + // we make sure that there is sufficient space for all the workers and non of + // them needs to be moved. This is important because a running worker may not + // be moved. + workers_.reserve(num_workers); + for (unsigned i = 0; i < num_workers; i++) { + workers_.emplace_back(*this, i); + workers_.back().start_thread(); + } + + // join all worker threads + for (auto& worker : workers_) { + worker.join_thread(); + } +} + +void GroupScheduler::next() { // NOLINT + static EventMap events{}; + + // clean up before scheduling any new events + if (!events.empty()) { + // cleanup all triggered actions + for (auto& vec_ports : events) { + vec_ports.first->cleanup(); + } + // cleanup all set ports + for (auto& vec_ports : set_ports_) { + for (auto& port : vec_ports) { + port->cleanup(); + } + vec_ports.clear(); + } + events.clear(); + } + + { + std::unique_lock lock{scheduling_mutex_}; + + // shutdown if there are no more events in the queue + if (event_queue_.empty() && !stop_) { + if (environment_->run_forever()) { + // wait for a new asynchronous event + cv_schedule_.wait(lock, [this]() { return !event_queue_.empty() || stop_; }); + } else { + log::Debug() << "No more events in queue_. -> Terminate!"; + environment_->sync_shutdown(); + } + } + + while (events.empty()) { + if (stop_) { + continue_execution_ = false; + log::Debug() << "Shutting down the scheduler"; + Tag t_next = Tag::from_logical_time(logical_time_).delay(); + if (t_next == event_queue_.begin()->first) { + log::Debug() << "Schedule the last round of reactions including all " + "termination reactions"; + events = std::move(event_queue_.begin()->second); + event_queue_.erase(event_queue_.begin()); + log::Debug() << "advance logical time to tag [" << t_next.time_point() << ", " << t_next.micro_step() << "]"; + logical_time_.advance_to(t_next); + } else { + return; + } + } else { + // collect events of the next tag + auto t_next = event_queue_.begin()->first; + + // synchronize with physical time if not in fast forward mode + if (!environment_->fast_fwd_execution()) { + // keep track of the current physical time in a static variable + static auto physical_time = TimePoint::min(); + + // If physical time is smaller than the next logical time point, + // then update the physical time. This step is small optimization to + // avoid calling get_physical_time() in every iteration as this + // would add a significant overhead. + if (physical_time < t_next.time_point()) { + physical_time = get_physical_time(); + } + + // If physical time is still smaller than the next logical time + // point, then wait until the next tag or until a new event is + // inserted asynchronously into the queue + if (physical_time < t_next.time_point()) { + auto status = cv_schedule_.wait_until(lock, t_next.time_point()); + // Start over if the event queue was modified + if (status == std::cv_status::no_timeout) { + continue; + } + // update physical time and continue otherwise + physical_time = t_next.time_point(); + } + } + + // retrieve all events with tag equal to current logical time from the + // queue + events = std::move(event_queue_.begin()->second); + event_queue_.erase(event_queue_.begin()); + + // advance logical time + log::Debug() << "advance logical time to tag [" << t_next.time_point() << ", " << t_next.micro_step() << "]"; + logical_time_.advance_to(t_next); + } + } + } // mutex schedule_ + + // execute all setup functions; this sets the values of the corresponding + // actions + for (auto& vec_reactor : events) { + auto& setup = vec_reactor.second; + if (setup != nullptr) { + setup(); + } + } + + log::Debug() << "events: " << events.size(); + for (auto& vec_reactor : events) { + log::Debug() << "Action " << vec_reactor.first->fqn(); + for (auto* reaction : vec_reactor.first->triggers()) { + // There is no need to acquire the mutex. At this point the scheduler + // should be the only thread accessing the reaction queue as none of the + // workers_ are running + log::Debug() << "insert reaction " << reaction->fqn() << " with index " << reaction->index(); + reaction_queue_[reaction->index()].push_back(reaction); + } + } +} + +GroupScheduler::GroupScheduler(Environment* env) + : using_workers_(env->num_workers() > 1) + , environment_(env) + , ready_queue_(env->num_workers()) {} + +GroupScheduler::~GroupScheduler() = default; + +void GroupScheduler::schedule_sync(const Tag& tag, BaseAction* action, std::function pre_handler) { + reactor_assert(logical_time_ < tag); + // TODO verify that the action is indeed allowed to be scheduled by the + // current reaction + log::Debug() << "Schedule action " << action->fqn() << (action->is_logical() ? " synchronously " : " asynchronously ") + << " with tag [" << tag.time_point() << ", " << tag.micro_step() << "]"; + { + auto unique_lock = + using_workers_ ? std::unique_lock(lock_event_queue_) : std::unique_lock(); + + tracepoint(reactor_cpp, schedule_action, action->container()->fqn(), action->name(), tag); // NOLINT + + // create a new event map or retrieve the existing one + auto emplace_result = event_queue_.try_emplace(tag, EventMap()); + auto& event_map = emplace_result.first->second; + + // insert the new event + event_map[action] = std::move(pre_handler); + } +} + +void GroupScheduler::schedule_async(const Tag& tag, BaseAction* action, std::function pre_handler) { + std::lock_guard lock_guard(scheduling_mutex_); + schedule_sync(tag, action, std::move(pre_handler)); + cv_schedule_.notify_one(); +} + +void GroupScheduler::set_port(BasePort* port) { + log::Debug() << "Set port " << port->fqn(); + + // We do not check here if port is already in the list. This means clean() + // could be called multiple times for a single port. However, calling + // clean() multiple time is not harmful and more efficient then checking if + set_ports_[GroupWorker::current_worker_id()].push_back(port); + + // recursively search for triggered reactions + set_port_helper(port); +} + +void GroupScheduler::set_port_helper(BasePort* port) { + for (auto* reaction : port->triggers()) { + triggered_reactions_[GroupWorker::current_worker_id()].push_back(reaction); + } + for (auto* binding : port->outward_bindings()) { + set_port_helper(binding); + } +} + +void GroupScheduler::stop() { + stop_ = true; + cv_schedule_.notify_one(); +} + +} // namespace reactor From 88cfadc87db2413f91a4daced1a7c7b54ca3639d Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Tue, 19 Jul 2022 15:40:30 +0200 Subject: [PATCH 23/49] first step at introducing a scheduling policy This makes the worker implementation plolicy based --- include/reactor-cpp/impl/scheduler_impl.hh | 58 ++++++++++++++++++++++ include/reactor-cpp/scheduler.hh | 51 +++++++++++++++---- lib/scheduler.cc | 47 +++--------------- 3 files changed, 107 insertions(+), 49 deletions(-) create mode 100644 include/reactor-cpp/impl/scheduler_impl.hh diff --git a/include/reactor-cpp/impl/scheduler_impl.hh b/include/reactor-cpp/impl/scheduler_impl.hh new file mode 100644 index 00000000..0c3da4be --- /dev/null +++ b/include/reactor-cpp/impl/scheduler_impl.hh @@ -0,0 +1,58 @@ +/* + * Copyright (C) 2022 TU Dresden + * All rights reserved. + * + * Authors: + * Christian Menard + */ + +#ifndef REACTOR_CPP_IMPL_SCHEDULER_IMPL_HH +#define REACTOR_CPP_IMPL_SCHEDULER_IMPL_HH + +#include "reactor-cpp/trace.hh" +#include +#include + +namespace reactor { + +template +// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) +thread_local const Worker* Worker::current_worker = nullptr; + +template +Worker::Worker(Worker&& worker) // NOLINT(performance-noexcept-move-constructor) + : policy_{worker.policy_} + , identity_{worker.identity_} { + // Need to provide the move constructor in order to organize workers in a + // std::vector. However, moving is not save if the thread is already running, + // thus we throw an exception here if the worker is moved but the + // internal thread is already running. + + if (worker.thread_.joinable()) { + throw std::runtime_error{"Running workers cannot be moved!"}; + } +} + +template void Worker::work() const { + // initialize the current worker thread local variable + current_worker = this; + + log::Debug() << "(Worker " << identity_ << ") Starting"; + + policy_.worker_function(*this); + + log::Debug() << "(Worker " << identity_ << ") terminates"; +} + +template void Worker::execute_reaction(Reaction* reaction) const { + log::Debug() << "(Worker " << identity_ << ") " + << "execute reaction " << reaction->fqn(); + + tracepoint(reactor_cpp, reaction_execution_starts, id, reaction->fqn(), scheduler.logical_time()); + reaction->trigger(); + tracepoint(reactor_cpp, reaction_execution_finishes, id, reaction->fqn(), scheduler.logical_time()); +} + +} // namespace reactor + +#endif // REACTOR_CPP_IMPL_SCHEDULER_IMPL_HH diff --git a/include/reactor-cpp/scheduler.hh b/include/reactor-cpp/scheduler.hh index 652afb6b..2fadd40f 100644 --- a/include/reactor-cpp/scheduler.hh +++ b/include/reactor-cpp/scheduler.hh @@ -10,6 +10,7 @@ #define REACTOR_CPP_SCHEDULER_HH #include +#include #include #include #include @@ -26,12 +27,11 @@ namespace reactor { // forward declarations class Scheduler; -class Worker; -class Worker { // NOLINT -public: - Scheduler& scheduler_; - const unsigned int identity_{0}; +template class Worker { +private: + SchedulingPolicy& policy_; + const std::size_t identity_{0}; std::thread thread_{}; // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) @@ -40,20 +40,44 @@ public: void work() const; void execute_reaction(Reaction* reaction) const; - Worker(Scheduler& scheduler, unsigned int identity) - : scheduler_{scheduler} +public: + Worker(SchedulingPolicy& policy, std::size_t identity) + : policy_{policy} , identity_{identity} {} Worker(Worker&& worker); // NOLINT(performance-noexcept-move-constructor) Worker(const Worker& worker) = delete; + ~Worker() = default; + + auto operator=(const Worker& worker) -> Worker& = delete; + auto operator=(Worker&& worker) -> Worker& = delete; - void start_thread() { thread_ = std::thread(&Worker::work, this); } - void join_thread() { thread_.join(); } + void start() { thread_ = std::thread(&Worker::work, this); } + void join() { thread_.join(); } + + [[nodiscard]] auto id() const -> std::size_t { return identity_; } static auto current_worker_id() -> unsigned { return current_worker->identity_; } + + friend SchedulingPolicy; +}; + +class DefaultSchedulingPolicy { + Scheduler& scheduler_; + std::size_t identity_counter{0}; + +public: + DefaultSchedulingPolicy(Scheduler& scheduler) + : scheduler_(scheduler) {} + + void worker_function(const Worker& worker) const; + + auto create_worker() -> Worker { return {*this, identity_counter++}; } }; class ReadyQueue { private: + using Worker = Worker; + std::vector queue_{}; std::atomic size_{0}; Semaphore sem_{0}; @@ -89,6 +113,10 @@ using EventMap = std::map>; class Scheduler { // NOLINT private: + using Worker = Worker; + + DefaultSchedulingPolicy policy_; + const bool using_workers_; LogicalTime logical_time_{}; @@ -137,9 +165,12 @@ public: void start(); void stop(); - friend Worker; + // FIXME: this needs to be removed in the final version, as we cannot make all policies friends... + friend DefaultSchedulingPolicy; }; } // namespace reactor +#include "impl/scheduler_impl.hh" + #endif // REACTOR_CPP_SCHEDULER_HH diff --git a/lib/scheduler.cc b/lib/scheduler.cc index 1e9b541e..75a8a9fc 100644 --- a/lib/scheduler.cc +++ b/lib/scheduler.cc @@ -20,29 +20,8 @@ namespace reactor { -// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) -thread_local const Worker* Worker::current_worker = nullptr; - -Worker::Worker(Worker&& work) // NOLINT(performance-noexcept-move-constructor) - : scheduler_{work.scheduler_} - , identity_{work.identity_} { - // Need to provide the move constructor in order to organize workers in a - // std::vector. However, moving is not save if the thread is already running, - // thus we throw an exception here if the worker is moved but the - // internal thread is already running. - - if (work.thread_.joinable()) { - throw std::runtime_error{"Running workers cannot be moved!"}; - } -} - -void Worker::work() const { - // initialize the current worker thread local variable - current_worker = this; - - log::Debug() << "(Worker " << this->identity_ << ") Starting"; - - if (identity_ == 0) { +void DefaultSchedulingPolicy::worker_function(const Worker& worker) const { + if (worker.id() == 0) { log::Debug() << "(Worker 0) do the initial scheduling"; scheduler_.schedule(); } @@ -57,7 +36,7 @@ void Worker::work() const { } // execute the reaction - execute_reaction(reaction); + worker.execute_reaction(reaction); // was this the very last reaction? if (scheduler_.reactions_to_process_.fetch_sub(1, std::memory_order_acq_rel) == 1) { @@ -67,17 +46,6 @@ void Worker::work() const { } // continue otherwise } - - log::Debug() << "(Worker " << identity_ << ") terminates"; -} - -void Worker::execute_reaction(Reaction* reaction) const { - log::Debug() << "(Worker " << identity_ << ") " - << "execute reaction " << reaction->fqn(); - - tracepoint(reactor_cpp, reaction_execution_starts, id, reaction->fqn(), scheduler.logical_time()); - reaction->trigger(); - tracepoint(reactor_cpp, reaction_execution_finishes, id, reaction->fqn(), scheduler.logical_time()); } void Scheduler::schedule() noexcept { @@ -212,13 +180,13 @@ void Scheduler::start() { // be moved. workers_.reserve(num_workers); for (unsigned i = 0; i < num_workers; i++) { - workers_.emplace_back(*this, i); - workers_.back().start_thread(); + workers_.emplace_back(policy_.create_worker()); + workers_.back().start(); } // join all worker threads for (auto& worker : workers_) { - worker.join_thread(); + worker.join(); } } @@ -336,7 +304,8 @@ void Scheduler::next() { // NOLINT } Scheduler::Scheduler(Environment* env) - : using_workers_(env->num_workers() > 1) + : policy_(*this) + , using_workers_(env->num_workers() > 1) , environment_(env) , ready_queue_(env->num_workers()) {} From 878809075dc55f83334e882f5a4c9a1c09b2a869 Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Tue, 19 Jul 2022 16:42:34 +0200 Subject: [PATCH 24/49] fix Worker name --- include/reactor-cpp/scheduler.hh | 8 +++----- lib/scheduler.cc | 8 ++++---- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/include/reactor-cpp/scheduler.hh b/include/reactor-cpp/scheduler.hh index 2fadd40f..288c6a2a 100644 --- a/include/reactor-cpp/scheduler.hh +++ b/include/reactor-cpp/scheduler.hh @@ -74,10 +74,10 @@ public: auto create_worker() -> Worker { return {*this, identity_counter++}; } }; +using DefaultWorker = Worker; + class ReadyQueue { private: - using Worker = Worker; - std::vector queue_{}; std::atomic size_{0}; Semaphore sem_{0}; @@ -113,15 +113,13 @@ using EventMap = std::map>; class Scheduler { // NOLINT private: - using Worker = Worker; - DefaultSchedulingPolicy policy_; const bool using_workers_; LogicalTime logical_time_{}; Environment* environment_; - std::vector workers_{}; + std::vector workers_{}; std::mutex scheduling_mutex_; std::unique_lock scheduling_lock_{scheduling_mutex_, std::defer_lock}; diff --git a/lib/scheduler.cc b/lib/scheduler.cc index 75a8a9fc..a7288de3 100644 --- a/lib/scheduler.cc +++ b/lib/scheduler.cc @@ -71,9 +71,9 @@ auto ReadyQueue::pop() -> Reaction* { // If there is no ready reaction available, wait until there is one. while (old_size <= 0) { - log::Debug() << "(Worker " << Worker::current_worker_id() << ") Wait for work"; + log::Debug() << "(Worker " << DefaultWorker::current_worker_id() << ") Wait for work"; sem_.acquire(); - log::Debug() << "(Worker " << Worker::current_worker_id() << ") Waking up"; + log::Debug() << "(Worker " << DefaultWorker::current_worker_id() << ") Waking up"; old_size = size_.fetch_sub(1, std::memory_order_acq_rel); // FIXME: Protect against underflow? } @@ -344,7 +344,7 @@ void Scheduler::set_port(BasePort* port) { // We do not check here if port is already in the list. This means clean() // could be called multiple times for a single port. However, calling // clean() multiple time is not harmful and more efficient then checking if - set_ports_[Worker::current_worker_id()].push_back(port); + set_ports_[DefaultWorker::current_worker_id()].push_back(port); // recursively search for triggered reactions set_port_helper(port); @@ -352,7 +352,7 @@ void Scheduler::set_port(BasePort* port) { void Scheduler::set_port_helper(BasePort* port) { for (auto* reaction : port->triggers()) { - triggered_reactions_[Worker::current_worker_id()].push_back(reaction); + triggered_reactions_[DefaultWorker::current_worker_id()].push_back(reaction); } for (auto* binding : port->outward_bindings()) { set_port_helper(binding); From 87a2f365b97eee58fe61122e61d6fae66f23a7cc Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Tue, 19 Jul 2022 15:55:09 +0200 Subject: [PATCH 25/49] install boost in CI --- .github/workflows/cpp-tests.yml | 5 +++++ .github/workflows/main.yaml | 9 +++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cpp-tests.yml b/.github/workflows/cpp-tests.yml index 9a523ace..3ec79d90 100644 --- a/.github/workflows/cpp-tests.yml +++ b/.github/workflows/cpp-tests.yml @@ -29,6 +29,11 @@ jobs: repository: lf-lang/lingua-franca submodules: true ref: ${{ inputs.compiler-ref }} + - name: Install boost on Ubuntu + run: | + sudo apt-get update + sudo apt-get install -y boost boost-dev + if: matrix.platform == 'ubuntu-latest' - name: Set cpp runtime version run: | echo ${{ inputs.runtime-ref }} > org.lflang/src/org/lflang/generator/cpp/cpp-runtime-version.txt diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index ba68ca42..190e7456 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -14,7 +14,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest, macos-latest] + os: [ubuntu-latest] steps: - uses: actions/checkout@v1 @@ -23,6 +23,11 @@ jobs: sudo apt-get update sudo apt-get install -y clang-tidy if: matrix.os == 'ubuntu-latest' + - name: Install boost on Ubuntu + run: | + sudo apt-get update + sudo apt-get install -y libboost-all-dev + if: matrix.os == 'ubuntu-latest' - name: configure run: | mkdir build @@ -35,7 +40,7 @@ jobs: cmake --build build --target examples lf-tests-pull-request: - uses: lf-lang/lingua-franca/.github/workflows/cpp-tests.yml@master + uses: lf-lang/lingua-franca/.github/workflows/cpp-tests.yml@cpp-boost with: runtime-ref: ${{github.ref}} compiler-ref: master From 4220ab95151b6923ac44139b15dba445afb99dba Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Tue, 19 Jul 2022 16:31:05 +0200 Subject: [PATCH 26/49] set minimal boost version to 1.71 --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 227c887b..8c1f0e89 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -26,7 +26,7 @@ if (NOT DEFINED LF_REACTOR_CPP_SUFFIX) endif() find_package (Threads) -find_package (Boost 1.79 COMPONENTS graph REQUIRED) +find_package (Boost 1.71 COMPONENTS graph REQUIRED) set(DEFAULT_BUILD_TYPE "Release") if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) From 24acace2d0a3d1b0ac5ce1c94a1af5212ccc7531 Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Tue, 19 Jul 2022 17:28:07 +0200 Subject: [PATCH 27/49] place the ready queue within the default policy --- include/reactor-cpp/scheduler.hh | 78 +++++++++++++++++--------------- lib/scheduler.cc | 13 +++--- 2 files changed, 47 insertions(+), 44 deletions(-) diff --git a/include/reactor-cpp/scheduler.hh b/include/reactor-cpp/scheduler.hh index 288c6a2a..295a7a25 100644 --- a/include/reactor-cpp/scheduler.hh +++ b/include/reactor-cpp/scheduler.hh @@ -65,50 +65,55 @@ class DefaultSchedulingPolicy { Scheduler& scheduler_; std::size_t identity_counter{0}; + class ReadyQueue { + private: + std::vector queue_{}; + std::atomic size_{0}; + Semaphore sem_{0}; + std::ptrdiff_t waiting_workers_{0}; + const unsigned int num_workers_; + + public: + explicit ReadyQueue(unsigned num_workers) + : num_workers_(num_workers) {} + + /** + * Retrieve a ready reaction from the queue. + * + * This method may be called concurrently. In case the queue is empty, the + * method blocks and waits until a ready reaction becomes available. + */ + auto pop() -> Reaction*; + + /** + * Fill the queue up with ready reactions. + * + * This method assumes that the internal queue is empty. It moves all + * reactions from the provided `ready_reactions` vector to the internal + * queue, leaving `ready_reactions` empty. + * + * Note that this method is not thread-safe. The caller needs to ensure that + * no other thread will try to read from the queue during this operation. + */ + void fill_up(std::vector& ready_reactions); + }; + + ReadyQueue ready_queue_; + public: - DefaultSchedulingPolicy(Scheduler& scheduler) - : scheduler_(scheduler) {} + // FIXME better to pass the environment and use num_workers from there, but for this we need to first fully separate + // the scheduler implementation from the policy. + DefaultSchedulingPolicy(Scheduler& scheduler, std::size_t num_workers) + : scheduler_(scheduler) + , ready_queue_(num_workers) {} - void worker_function(const Worker& worker) const; + void worker_function(const Worker& worker); auto create_worker() -> Worker { return {*this, identity_counter++}; } }; using DefaultWorker = Worker; -class ReadyQueue { -private: - std::vector queue_{}; - std::atomic size_{0}; - Semaphore sem_{0}; - std::ptrdiff_t waiting_workers_{0}; - const unsigned int num_workers_; - -public: - explicit ReadyQueue(unsigned num_workers) - : num_workers_(num_workers) {} - - /** - * Retrieve a ready reaction from the queue. - * - * This method may be called concurrently. In case the queue is empty, the - * method blocks and waits until a ready reaction becomes available. - */ - auto pop() -> Reaction*; - - /** - * Fill the queue up with ready reactions. - * - * This method assumes that the internal queue is empty. It moves all - * reactions from the provided `ready_reactions` vector to the internal - * queue, leaving `ready_reactions` empty. - * - * Note that this method is not thread-safe. The caller needs to ensure that - * no other thread will try to read from the queue during this operation. - */ - void fill_up(std::vector& ready_reactions); -}; - using EventMap = std::map>; class Scheduler { // NOLINT @@ -134,7 +139,6 @@ private: std::vector> reaction_queue_; unsigned int reaction_queue_pos_{std::numeric_limits::max()}; - ReadyQueue ready_queue_; std::atomic reactions_to_process_{0}; // NOLINT std::atomic stop_{false}; diff --git a/lib/scheduler.cc b/lib/scheduler.cc index a7288de3..f7da4027 100644 --- a/lib/scheduler.cc +++ b/lib/scheduler.cc @@ -20,7 +20,7 @@ namespace reactor { -void DefaultSchedulingPolicy::worker_function(const Worker& worker) const { +void DefaultSchedulingPolicy::worker_function(const Worker& worker) { if (worker.id() == 0) { log::Debug() << "(Worker 0) do the initial scheduling"; scheduler_.schedule(); @@ -28,7 +28,7 @@ void DefaultSchedulingPolicy::worker_function(const Worker Reaction* { +auto DefaultSchedulingPolicy::ReadyQueue::pop() -> Reaction* { auto old_size = size_.fetch_sub(1, std::memory_order_acq_rel); // If there is no ready reaction available, wait until there is one. @@ -82,7 +82,7 @@ auto ReadyQueue::pop() -> Reaction* { return queue_[pos]; } -void ReadyQueue::fill_up(std::vector& ready_reactions) { +void DefaultSchedulingPolicy::ReadyQueue::fill_up(std::vector& ready_reactions) { // clear the internal queue and swap contents queue_.clear(); queue_.swap(ready_reactions); @@ -304,10 +304,9 @@ void Scheduler::next() { // NOLINT } Scheduler::Scheduler(Environment* env) - : policy_(*this) + : policy_(*this, env->num_workers()) , using_workers_(env->num_workers() > 1) - , environment_(env) - , ready_queue_(env->num_workers()) {} + , environment_(env) {} Scheduler::~Scheduler() = default; From 99a8db6bda1f14c729275d6f1764380c97a705ac Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Tue, 19 Jul 2022 18:23:28 +0200 Subject: [PATCH 28/49] implement the default policy in its own file --- include/reactor-cpp/fwd.hh | 3 ++ include/reactor-cpp/scheduler.hh | 51 ---------------------- lib/CMakeLists.txt | 1 + lib/scheduler.cc | 75 -------------------------------- 4 files changed, 4 insertions(+), 126 deletions(-) diff --git a/include/reactor-cpp/fwd.hh b/include/reactor-cpp/fwd.hh index 52edced4..4d26792d 100644 --- a/include/reactor-cpp/fwd.hh +++ b/include/reactor-cpp/fwd.hh @@ -19,6 +19,9 @@ class Reactor; class Scheduler; class Tag; +class DefaultSchedulingPolicy; +template class Worker; + template class Action; template class Port; diff --git a/include/reactor-cpp/scheduler.hh b/include/reactor-cpp/scheduler.hh index 295a7a25..8faffd33 100644 --- a/include/reactor-cpp/scheduler.hh +++ b/include/reactor-cpp/scheduler.hh @@ -21,7 +21,6 @@ #include "fwd.hh" #include "logical_time.hh" -#include "semaphore.hh" namespace reactor { @@ -61,56 +60,6 @@ public: friend SchedulingPolicy; }; -class DefaultSchedulingPolicy { - Scheduler& scheduler_; - std::size_t identity_counter{0}; - - class ReadyQueue { - private: - std::vector queue_{}; - std::atomic size_{0}; - Semaphore sem_{0}; - std::ptrdiff_t waiting_workers_{0}; - const unsigned int num_workers_; - - public: - explicit ReadyQueue(unsigned num_workers) - : num_workers_(num_workers) {} - - /** - * Retrieve a ready reaction from the queue. - * - * This method may be called concurrently. In case the queue is empty, the - * method blocks and waits until a ready reaction becomes available. - */ - auto pop() -> Reaction*; - - /** - * Fill the queue up with ready reactions. - * - * This method assumes that the internal queue is empty. It moves all - * reactions from the provided `ready_reactions` vector to the internal - * queue, leaving `ready_reactions` empty. - * - * Note that this method is not thread-safe. The caller needs to ensure that - * no other thread will try to read from the queue during this operation. - */ - void fill_up(std::vector& ready_reactions); - }; - - ReadyQueue ready_queue_; - -public: - // FIXME better to pass the environment and use num_workers from there, but for this we need to first fully separate - // the scheduler implementation from the policy. - DefaultSchedulingPolicy(Scheduler& scheduler, std::size_t num_workers) - : scheduler_(scheduler) - , ready_queue_(num_workers) {} - - void worker_function(const Worker& worker); - - auto create_worker() -> Worker { return {*this, identity_counter++}; } -}; using DefaultWorker = Worker; diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index f8b6c9a1..b1bfce4b 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -1,6 +1,7 @@ set(SOURCE_FILES action.cc assert.cc + default_scheduling_policy.cc environment.cc logical_time.cc port.cc diff --git a/lib/scheduler.cc b/lib/scheduler.cc index f7da4027..adabf944 100644 --- a/lib/scheduler.cc +++ b/lib/scheduler.cc @@ -20,34 +20,6 @@ namespace reactor { -void DefaultSchedulingPolicy::worker_function(const Worker& worker) { - if (worker.id() == 0) { - log::Debug() << "(Worker 0) do the initial scheduling"; - scheduler_.schedule(); - } - - while (true) { - // wait for a ready reaction - auto* reaction = ready_queue_.pop(); - - // receiving a nullptr indicates that the worker should terminate - if (reaction == nullptr) { - break; - } - - // execute the reaction - worker.execute_reaction(reaction); - - // was this the very last reaction? - if (scheduler_.reactions_to_process_.fetch_sub(1, std::memory_order_acq_rel) == 1) { - // Yes, then schedule. The atomic decrement above ensures that only one - // thread enters this block. - scheduler_.schedule(); - } - // continue otherwise - } -} - void Scheduler::schedule() noexcept { bool found_ready_reactions = schedule_ready_reactions(); @@ -66,53 +38,6 @@ void Scheduler::schedule() noexcept { } } -auto DefaultSchedulingPolicy::ReadyQueue::pop() -> Reaction* { - auto old_size = size_.fetch_sub(1, std::memory_order_acq_rel); - - // If there is no ready reaction available, wait until there is one. - while (old_size <= 0) { - log::Debug() << "(Worker " << DefaultWorker::current_worker_id() << ") Wait for work"; - sem_.acquire(); - log::Debug() << "(Worker " << DefaultWorker::current_worker_id() << ") Waking up"; - old_size = size_.fetch_sub(1, std::memory_order_acq_rel); - // FIXME: Protect against underflow? - } - - auto pos = old_size - 1; - return queue_[pos]; -} - -void DefaultSchedulingPolicy::ReadyQueue::fill_up(std::vector& ready_reactions) { - // clear the internal queue and swap contents - queue_.clear(); - queue_.swap(ready_reactions); - - // update the atomic size counter and release the semaphore to wake up - // waiting worker threads - auto new_size = static_cast(queue_.size()); - auto old_size = size_.exchange(new_size, std::memory_order_acq_rel); - - // calculate how many workers to wake up. -old_size indicates the number of - // workers who started waiting since the last update. - // We want to wake up at most all the waiting workers. If we would release - // more, other workers that are out of work would not block when acquiring - // the semaphore. - // Also, we do not want to wake up more workers than there is work. new_size - // indicates the number of ready reactions. Since there is always at least - // one worker running running, new_size - running_workers indicates the - // number of additional workers needed to process all reactions. - waiting_workers_ += -old_size; - auto running_workers = num_workers_ - waiting_workers_; - auto workers_to_wakeup = std::min(waiting_workers_, new_size - running_workers); - - // wakeup other workers_ - if (workers_to_wakeup > 0) { - waiting_workers_ -= workers_to_wakeup; - log::Debug() << "Wakeup " << workers_to_wakeup << " workers"; - sem_.release(static_cast(workers_to_wakeup)); - } -} - void Scheduler::terminate_all_workers() { log::Debug() << "(Scheduler) Send termination signal to all workers"; auto num_workers = environment_->num_workers(); From 2a3bec2f1b40e092c2c071e1f8c9923f090b3c68 Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Wed, 20 Jul 2022 17:45:51 +0200 Subject: [PATCH 29/49] almost complete implementation of the scheduling policy --- include/reactor-cpp/action.hh | 4 +- .../reactor-cpp/default_scheduling_policy.hh | 86 ++++++ include/reactor-cpp/environment.hh | 12 +- include/reactor-cpp/fwd.hh | 2 +- include/reactor-cpp/impl/port_impl.hh | 4 +- include/reactor-cpp/impl/scheduler_impl.hh | 224 +++++++++++++- include/reactor-cpp/port.hh | 4 +- include/reactor-cpp/reactor-cpp.hh | 6 + include/reactor-cpp/scheduler.hh | 43 +-- lib/CMakeLists.txt | 1 - lib/default_scheduling_policy.cc | 183 +++++++++++ lib/scheduler.cc | 291 ------------------ 12 files changed, 523 insertions(+), 337 deletions(-) create mode 100644 include/reactor-cpp/default_scheduling_policy.hh create mode 100644 lib/default_scheduling_policy.cc delete mode 100644 lib/scheduler.cc diff --git a/include/reactor-cpp/action.hh b/include/reactor-cpp/action.hh index 129f6526..7b8c7341 100644 --- a/include/reactor-cpp/action.hh +++ b/include/reactor-cpp/action.hh @@ -44,7 +44,7 @@ public: [[nodiscard]] auto inline min_delay() const noexcept -> Duration { return min_delay_; } friend class Reaction; - friend class Scheduler; + template friend class Scheduler; }; template class Action : public BaseAction { @@ -154,6 +154,4 @@ public: } // namespace reactor -#include "impl/action_impl.hh" - #endif // REACTOR_CPP_ACTION_HH diff --git a/include/reactor-cpp/default_scheduling_policy.hh b/include/reactor-cpp/default_scheduling_policy.hh new file mode 100644 index 00000000..e1ecc5cf --- /dev/null +++ b/include/reactor-cpp/default_scheduling_policy.hh @@ -0,0 +1,86 @@ +/* + * Copyright (C) 2022 TU Dresden + * All rights reserved. + * + * Authors: + * Christian Menard + */ + +#ifndef REACTOR_CPP_DEFAULT_SCHEDULING_POLICY_HH +#define REACTOR_CPP_DEFAULT_SCHEDULING_POLICY_HH + +#include +#include + +#include "reactor-cpp/fwd.hh" +#include "reactor-cpp/reaction.hh" +#include "reactor-cpp/semaphore.hh" + +namespace reactor { + +class DefaultSchedulingPolicy { + Scheduler& scheduler_; + Environment& environment_; + std::size_t identity_counter{0}; + + class ReadyQueue { + private: + std::vector queue_{}; + std::atomic size_{0}; + Semaphore sem_{0}; + std::ptrdiff_t waiting_workers_{0}; + const unsigned int num_workers_; + + public: + explicit ReadyQueue(unsigned num_workers) + : num_workers_(num_workers) {} + + /** + * Retrieve a ready reaction from the queue. + * + * This method may be called concurrently. In case the queue is empty, the + * method blocks and waits until a ready reaction becomes available. + */ + auto pop() -> Reaction*; + + /** + * Fill the queue up with ready reactions. + * + * This method assumes that the internal queue is empty. It moves all + * reactions from the provided `ready_reactions` vector to the internal + * queue, leaving `ready_reactions` empty. + * + * Note that this method is not thread-safe. The caller needs to ensure that + * no other thread will try to read from the queue during this operation. + */ + void fill_up(std::vector& ready_reactions); + }; + + ReadyQueue ready_queue_; + + std::vector> reaction_queue_; + unsigned int reaction_queue_pos_{std::numeric_limits::max()}; + + std::atomic reactions_to_process_{0}; + std::vector> triggered_reactions_; + + bool continue_execution_{true}; + + void schedule() noexcept; + void terminate_all_workers(); + auto schedule_ready_reactions() -> bool; + +public: + DefaultSchedulingPolicy(Scheduler& scheduler, Environment& env); + + void init(); + auto create_worker() -> Worker; + void worker_function(const Worker& worker); + + void trigger_reaction_from_next(Reaction* reaction); + void trigger_reaction_from_set_port(Reaction* reaction); +}; + +} // namespace reactor + +#endif // REACTOR_CPP_DEFAULT_SCHEDULING_POLICY_HH diff --git a/include/reactor-cpp/environment.hh b/include/reactor-cpp/environment.hh index 8679f26f..4e2322af 100644 --- a/include/reactor-cpp/environment.hh +++ b/include/reactor-cpp/environment.hh @@ -13,8 +13,10 @@ #include #include -#include "reactor.hh" -#include "scheduler.hh" +#include "reactor-cpp/default_scheduling_policy.hh" +#include "reactor-cpp/fwd.hh" +#include "reactor-cpp/reactor.hh" +#include "reactor-cpp/scheduler.hh" namespace reactor { @@ -38,7 +40,7 @@ private: std::set reactions_{}; std::vector dependencies_{}; - Scheduler scheduler_; + Scheduler scheduler_; Phase phase_{Phase::Construction}; TimePoint start_time_{}; @@ -70,9 +72,9 @@ public: [[nodiscard]] auto top_level_reactors() const noexcept -> const auto& { return top_level_reactors_; } [[nodiscard]] auto phase() const noexcept -> Phase { return phase_; } - [[nodiscard]] auto scheduler() const noexcept -> const Scheduler* { return &scheduler_; } + [[nodiscard]] auto scheduler() const noexcept -> const Scheduler* { return &scheduler_; } - auto scheduler() noexcept -> Scheduler* { return &scheduler_; } + auto scheduler() noexcept -> Scheduler* { return &scheduler_; } [[nodiscard]] auto logical_time() const noexcept -> const LogicalTime& { return scheduler_.logical_time(); } [[nodiscard]] auto start_time() const noexcept -> const TimePoint& { return start_time_; } diff --git a/include/reactor-cpp/fwd.hh b/include/reactor-cpp/fwd.hh index 4d26792d..2177f92b 100644 --- a/include/reactor-cpp/fwd.hh +++ b/include/reactor-cpp/fwd.hh @@ -16,11 +16,11 @@ class BasePort; class Environment; class Reaction; class Reactor; -class Scheduler; class Tag; class DefaultSchedulingPolicy; template class Worker; +template class Scheduler; template class Action; template class Port; diff --git a/include/reactor-cpp/impl/port_impl.hh b/include/reactor-cpp/impl/port_impl.hh index 0755aeea..64943a99 100644 --- a/include/reactor-cpp/impl/port_impl.hh +++ b/include/reactor-cpp/impl/port_impl.hh @@ -9,8 +9,8 @@ #ifndef REACTOR_CPP_IMPL_PORT_IMPL_HH #define REACTOR_CPP_IMPL_PORT_IMPL_HH -#include "../assert.hh" -#include "../environment.hh" +#include "reactor-cpp/assert.hh" +#include "reactor-cpp/environment.hh" namespace reactor { diff --git a/include/reactor-cpp/impl/scheduler_impl.hh b/include/reactor-cpp/impl/scheduler_impl.hh index 0c3da4be..10c01350 100644 --- a/include/reactor-cpp/impl/scheduler_impl.hh +++ b/include/reactor-cpp/impl/scheduler_impl.hh @@ -9,15 +9,25 @@ #ifndef REACTOR_CPP_IMPL_SCHEDULER_IMPL_HH #define REACTOR_CPP_IMPL_SCHEDULER_IMPL_HH +#include "reactor-cpp/action.hh" +#include "reactor-cpp/assert.hh" +#include "reactor-cpp/environment.hh" +#include "reactor-cpp/fwd.hh" +#include "reactor-cpp/logging.hh" +#include "reactor-cpp/port.hh" +#include "reactor-cpp/reaction.hh" #include "reactor-cpp/trace.hh" -#include -#include +#include namespace reactor { template // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) -thread_local const Worker* Worker::current_worker = nullptr; +thread_local std::size_t Worker::current_worker_id_{0}; + +template auto Worker::current_worker_id() -> unsigned { + return current_worker_id_; +} template Worker::Worker(Worker&& worker) // NOLINT(performance-noexcept-move-constructor) @@ -35,7 +45,7 @@ Worker::Worker(Worker&& worker) // NOLINT(pe template void Worker::work() const { // initialize the current worker thread local variable - current_worker = this; + current_worker_id_ = identity_; log::Debug() << "(Worker " << identity_ << ") Starting"; @@ -53,6 +63,212 @@ template void Worker::execute_reactio tracepoint(reactor_cpp, reaction_execution_finishes, id, reaction->fqn(), scheduler.logical_time()); } +template +Scheduler::Scheduler(Environment* env) + : policy_(*this, *env) + , using_workers_(env->num_workers() > 1) + , environment_(env) {} + +template void Scheduler::start() { + log::Debug() << "Starting the scheduler..."; + + auto num_workers = environment_->num_workers(); + // initialize the reaction queue, set ports vector, and triggered reactions + // vector + set_ports_.resize(num_workers); + policy_.init(); + + // Initialize and start the workers. By resizing the workers vector first, + // we make sure that there is sufficient space for all the workers and non of + // them needs to be moved. This is important because a running worker may not + // be moved. + workers_.reserve(num_workers); + for (unsigned i = 0; i < num_workers; i++) { + workers_.emplace_back(policy_.create_worker()); + workers_.back().start(); + } + + // join all worker threads + for (auto& worker : workers_) { + worker.join(); + } +} + +// FIXME: Reduce complexity of this function +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +template auto Scheduler::next() -> bool { + static EventMap events{}; + bool continue_execution{true}; + + // clean up before scheduling any new events + if (!events.empty()) { + // cleanup all triggered actions + for (auto& vec_ports : events) { + vec_ports.first->cleanup(); + } + // cleanup all set ports + for (auto& vec_ports : set_ports_) { + for (auto& port : vec_ports) { + port->cleanup(); + } + vec_ports.clear(); + } + events.clear(); + } + + { + std::unique_lock lock{scheduling_mutex_}; + + // shutdown if there are no more events in the queue + if (event_queue_.empty() && !stop_) { + if (environment_->run_forever()) { + // wait for a new asynchronous event + cv_schedule_.wait(lock, [this]() { return !event_queue_.empty() || stop_; }); + } else { + log::Debug() << "No more events in queue_. -> Terminate!"; + environment_->sync_shutdown(); + } + } + + while (events.empty()) { + if (stop_) { + continue_execution = false; + log::Debug() << "Shutting down the scheduler"; + Tag t_next = Tag::from_logical_time(logical_time_).delay(); + if (t_next == event_queue_.begin()->first) { + log::Debug() << "Schedule the last round of reactions including all " + "termination reactions"; + events = std::move(event_queue_.begin()->second); + event_queue_.erase(event_queue_.begin()); + log::Debug() << "advance logical time to tag [" << t_next.time_point() << ", " << t_next.micro_step() << "]"; + logical_time_.advance_to(t_next); + } else { + return continue_execution; + } + } else { + // collect events of the next tag + auto t_next = event_queue_.begin()->first; + + // synchronize with physical time if not in fast forward mode + if (!environment_->fast_fwd_execution()) { + // keep track of the current physical time in a static variable + static auto physical_time = TimePoint::min(); + + // If physical time is smaller than the next logical time point, + // then update the physical time. This step is small optimization to + // avoid calling get_physical_time() in every iteration as this + // would add a significant overhead. + if (physical_time < t_next.time_point()) { + physical_time = get_physical_time(); + } + + // If physical time is still smaller than the next logical time + // point, then wait until the next tag or until a new event is + // inserted asynchronously into the queue + if (physical_time < t_next.time_point()) { + auto status = cv_schedule_.wait_until(lock, t_next.time_point()); + // Start over if the event queue was modified + if (status == std::cv_status::no_timeout) { + continue; + } + // update physical time and continue otherwise + physical_time = t_next.time_point(); + } + } + + // retrieve all events with tag equal to current logical time from the + // queue + events = std::move(event_queue_.begin()->second); + event_queue_.erase(event_queue_.begin()); + + // advance logical time + log::Debug() << "advance logical time to tag [" << t_next.time_point() << ", " << t_next.micro_step() << "]"; + logical_time_.advance_to(t_next); + } + } + } // mutex schedule_ + + // execute all setup functions; this sets the values of the corresponding + // actions + for (auto& vec_reactor : events) { + auto& setup = vec_reactor.second; + if (setup != nullptr) { + setup(); + } + } + + log::Debug() << "events: " << events.size(); + for (auto& vec_reactor : events) { + log::Debug() << "Action " << vec_reactor.first->fqn(); + for (auto* reaction : vec_reactor.first->triggers()) { + // There is no need to acquire the mutex. At this point the scheduler + // should be the only thread accessing the reaction queue as none of the + // workers_ are running + log::Debug() << "(Scheduler) trigger reaction " << reaction->fqn() << " with index " << reaction->index(); + policy_.trigger_reaction_from_next(reaction); + } + } + + return continue_execution; +} + +template +void Scheduler::schedule_sync(const Tag& tag, BaseAction* action, + std::function pre_handler) { + reactor_assert(logical_time_ < tag); + // TODO verify that the action is indeed allowed to be scheduled by the + // current reaction + log::Debug() << "Schedule action " << action->fqn() << (action->is_logical() ? " synchronously " : " asynchronously ") + << " with tag [" << tag.time_point() << ", " << tag.micro_step() << "]"; + { + auto unique_lock = + using_workers_ ? std::unique_lock(lock_event_queue_) : std::unique_lock(); + + tracepoint(reactor_cpp, schedule_action, action->container()->fqn(), action->name(), tag); // NOLINT + + // create a new event map or retrieve the existing one + auto emplace_result = event_queue_.try_emplace(tag, EventMap()); + auto& event_map = emplace_result.first->second; + + // insert the new event + event_map[action] = std::move(pre_handler); + } +} + +template +void Scheduler::schedule_async(const Tag& tag, BaseAction* action, + std::function pre_handler) { + std::lock_guard lock_guard(scheduling_mutex_); + schedule_sync(tag, action, std::move(pre_handler)); + cv_schedule_.notify_one(); +} + +template void Scheduler::set_port(BasePort* port) { + log::Debug() << "Set port " << port->fqn(); + + // We do not check here if port is already in the list. This means clean() + // could be called multiple times for a single port. However, calling + // clean() multiple time is not harmful and more efficient then checking if + set_ports_[Worker::current_worker_id()].push_back(port); + + // recursively search for triggered reactions + set_port_helper(port); +} + +template void Scheduler::set_port_helper(BasePort* port) { + for (auto* reaction : port->triggers()) { + policy_.trigger_reaction_from_set_port(reaction); + } + for (auto* binding : port->outward_bindings()) { + set_port_helper(binding); + } +} + +template void Scheduler::stop() { + stop_ = true; + cv_schedule_.notify_one(); +} + } // namespace reactor #endif // REACTOR_CPP_IMPL_SCHEDULER_IMPL_HH diff --git a/include/reactor-cpp/port.hh b/include/reactor-cpp/port.hh index f2345532..7e8c6afc 100644 --- a/include/reactor-cpp/port.hh +++ b/include/reactor-cpp/port.hh @@ -56,7 +56,7 @@ public: [[nodiscard]] inline auto antidependencies() const noexcept -> const auto& { return antidependencies_; } friend class Reaction; - friend class Scheduler; + template friend class Scheduler; }; template class Port : public BasePort { @@ -127,6 +127,4 @@ public: } // namespace reactor -#include "impl/port_impl.hh" - #endif // REACTOR_CPP_PORT_HH diff --git a/include/reactor-cpp/reactor-cpp.hh b/include/reactor-cpp/reactor-cpp.hh index 024597c0..8f15d968 100644 --- a/include/reactor-cpp/reactor-cpp.hh +++ b/include/reactor-cpp/reactor-cpp.hh @@ -19,4 +19,10 @@ #include "reactor.hh" #include "time.hh" +// include implementation header files + +#include "impl/action_impl.hh" +#include "impl/port_impl.hh" +#include "impl/scheduler_impl.hh" + #endif // REACTOR_CPP_REACTOR_CPP_HH diff --git a/include/reactor-cpp/scheduler.hh b/include/reactor-cpp/scheduler.hh index 8faffd33..c413691e 100644 --- a/include/reactor-cpp/scheduler.hh +++ b/include/reactor-cpp/scheduler.hh @@ -24,9 +24,6 @@ namespace reactor { -// forward declarations -class Scheduler; - template class Worker { private: SchedulingPolicy& policy_; @@ -34,7 +31,7 @@ private: std::thread thread_{}; // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) - static thread_local const Worker* current_worker; + static thread_local std::size_t current_worker_id_; void work() const; void execute_reaction(Reaction* reaction) const; @@ -55,25 +52,24 @@ public: [[nodiscard]] auto id() const -> std::size_t { return identity_; } - static auto current_worker_id() -> unsigned { return current_worker->identity_; } + static auto current_worker_id() -> unsigned; friend SchedulingPolicy; }; - -using DefaultWorker = Worker; - using EventMap = std::map>; -class Scheduler { // NOLINT +template class Scheduler { private: - DefaultSchedulingPolicy policy_; + using worker_t = Worker; + + SchedulingPolicy policy_; const bool using_workers_; LogicalTime logical_time_{}; Environment* environment_; - std::vector workers_{}; + std::vector workers_{}; std::mutex scheduling_mutex_; std::unique_lock scheduling_lock_{scheduling_mutex_, std::defer_lock}; @@ -83,25 +79,21 @@ private: std::map event_queue_; std::vector> set_ports_; - std::vector> triggered_reactions_; - - std::vector> reaction_queue_; - unsigned int reaction_queue_pos_{std::numeric_limits::max()}; - - std::atomic reactions_to_process_{0}; // NOLINT std::atomic stop_{false}; - bool continue_execution_{true}; - void schedule() noexcept; - auto schedule_ready_reactions() -> bool; - void next(); - void terminate_all_workers(); + std::vector> triggered_reactions_; + + auto next() -> bool; void set_port_helper(BasePort* port); public: explicit Scheduler(Environment* env); - ~Scheduler(); + Scheduler(Scheduler&&) = delete; + Scheduler(const Scheduler&) = delete; + ~Scheduler() = default; + auto operator=(Scheduler&&) -> Scheduler = delete; + auto operator=(const Scheduler&) -> Scheduler = delete; void schedule_sync(const Tag& tag, BaseAction* action, std::function pre_handler); void schedule_async(const Tag& tag, BaseAction* action, std::function pre_handler); @@ -116,12 +108,9 @@ public: void start(); void stop(); - // FIXME: this needs to be removed in the final version, as we cannot make all policies friends... - friend DefaultSchedulingPolicy; + friend SchedulingPolicy; }; } // namespace reactor -#include "impl/scheduler_impl.hh" - #endif // REACTOR_CPP_SCHEDULER_HH diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index b1bfce4b..d0b7c071 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -7,7 +7,6 @@ set(SOURCE_FILES port.cc reaction.cc reactor.cc - scheduler.cc time.cc ) diff --git a/lib/default_scheduling_policy.cc b/lib/default_scheduling_policy.cc new file mode 100644 index 00000000..601211bb --- /dev/null +++ b/lib/default_scheduling_policy.cc @@ -0,0 +1,183 @@ +/* + * Copyright (C) 2022 TU Dresden + * All rights reserved. + * + * Authors: + * Christian Menard + */ + +#include "reactor-cpp/default_scheduling_policy.hh" + +#include "reactor-cpp/environment.hh" +#include "reactor-cpp/logging.hh" +#include "reactor-cpp/reaction.hh" +#include "reactor-cpp/scheduler.hh" +#include "reactor-cpp/trace.hh" + +namespace reactor { + +DefaultSchedulingPolicy::DefaultSchedulingPolicy(Scheduler& scheduler, Environment& env) + : scheduler_(scheduler) + , environment_(env) + , ready_queue_(env.num_workers()) {} + +void DefaultSchedulingPolicy::init() { + reaction_queue_.resize(environment_.max_reaction_index() + 1); + triggered_reactions_.resize(environment_.num_workers()); +} + +void DefaultSchedulingPolicy::worker_function(const Worker& worker) { + if (worker.id() == 0) { + log::Debug() << "(Worker 0) do the initial scheduling"; + schedule(); + } + + while (true) { + // wait for a ready reaction + auto* reaction = ready_queue_.pop(); + + // receiving a nullptr indicates that the worker should terminate + if (reaction == nullptr) { + break; + } + + // execute the reaction + worker.execute_reaction(reaction); + + // was this the very last reaction? + if (reactions_to_process_.fetch_sub(1, std::memory_order_acq_rel) == 1) { + // Yes, then schedule. The atomic decrement above ensures that only one + // thread enters this block. + schedule(); + } + // continue otherwise + } +} + +auto DefaultSchedulingPolicy::create_worker() -> Worker { return {*this, identity_counter++}; } + +void DefaultSchedulingPolicy::schedule() noexcept { + bool found_ready_reactions = schedule_ready_reactions(); + + while (!found_ready_reactions) { + log::Debug() << "(DefaultSchedulingPolicy) call next()"; + continue_execution_ = scheduler_.next(); + reaction_queue_pos_ = 0; + + found_ready_reactions = schedule_ready_reactions(); + + if (!continue_execution_ && !found_ready_reactions) { + // let all workers know that they should terminate + terminate_all_workers(); + break; + } + } +} + +auto DefaultSchedulingPolicy::schedule_ready_reactions() -> bool { + // insert any triggered reactions_ into the reaction queue + for (auto& vec_reaction : triggered_reactions_) { + for (auto* reaction : vec_reaction) { + reaction_queue_[reaction->index()].push_back(reaction); + } + vec_reaction.clear(); + } + + log::Debug() << "(DefaultSchedulingPolicy) Scanning the reaction queue for ready reactions"; + + // continue iterating over the reaction queue + for (; reaction_queue_pos_ < reaction_queue_.size(); reaction_queue_pos_++) { + auto& reactions = reaction_queue_[reaction_queue_pos_]; + + // any ready reactions of current priority? + if (!reactions.empty()) { + log::Debug() << "(DefaultSchedulingPolicy) Process reactions of priority " << reaction_queue_pos_; + + // Make sure that any reaction is only executed once even if it + // was triggered multiple times. + std::sort(reactions.begin(), reactions.end()); + reactions.erase(std::unique(reactions.begin(), reactions.end()), reactions.end()); + + if constexpr (log::debug_enabled || tracing_enabled) { // NOLINT + for (auto* reaction : reactions) { + log::Debug() << "(DefaultSchedulingPolicy) Reaction " << reaction->fqn() << " is ready for execution"; + tracepoint(reactor_cpp, trigger_reaction, reaction->container()->fqn(), reaction->name(), logical_time_); + } + } + + reactions_to_process_.store(static_cast(reactions.size()), std::memory_order_release); + ready_queue_.fill_up(reactions); + + // break out of the loop and return + return true; + } + } + + log::Debug() << "(DefaultSchedulingPolicy) Reached end of reaction queue"; + return false; +} + +void DefaultSchedulingPolicy::terminate_all_workers() { + log::Debug() << "(DefaultSchedulingPolicy) Send termination signal to all workers"; + auto num_workers = environment_.num_workers(); + std::vector null_reactions{num_workers, nullptr}; + log::Debug() << null_reactions.size(); + ready_queue_.fill_up(null_reactions); +} + +void DefaultSchedulingPolicy::trigger_reaction_from_next(Reaction* reaction) { + reaction_queue_[reaction->index()].push_back(reaction); +} + +void DefaultSchedulingPolicy::trigger_reaction_from_set_port(Reaction* reaction) { + triggered_reactions_[Worker::current_worker_id()].push_back(reaction); +} + +auto DefaultSchedulingPolicy::ReadyQueue::pop() -> Reaction* { + auto old_size = size_.fetch_sub(1, std::memory_order_acq_rel); + + // If there is no ready reaction available, wait until there is one. + while (old_size <= 0) { + log::Debug() << "(Worker " << Worker::current_worker_id() << ") Wait for work"; + sem_.acquire(); + log::Debug() << "(Worker " << Worker::current_worker_id() << ") Waking up"; + old_size = size_.fetch_sub(1, std::memory_order_acq_rel); + // FIXME: Protect against underflow? + } + + auto pos = old_size - 1; + return queue_[pos]; +} + +void DefaultSchedulingPolicy::ReadyQueue::fill_up(std::vector& ready_reactions) { + // clear the internal queue and swap contents + queue_.clear(); + queue_.swap(ready_reactions); + + // update the atomic size counter and release the semaphore to wake up + // waiting worker threads + auto new_size = static_cast(queue_.size()); + auto old_size = size_.exchange(new_size, std::memory_order_acq_rel); + + // calculate how many workers to wake up. -old_size indicates the number of + // workers who started waiting since the last update. + // We want to wake up at most all the waiting workers. If we would release + // more, other workers that are out of work would not block when acquiring + // the semaphore. + // Also, we do not want to wake up more workers than there is work. new_size + // indicates the number of ready reactions. Since there is always at least + // one worker running running, new_size - running_workers indicates the + // number of additional workers needed to process all reactions. + waiting_workers_ += -old_size; + auto running_workers = num_workers_ - waiting_workers_; + auto workers_to_wakeup = std::min(waiting_workers_, new_size - running_workers); + + // wakeup other workers_ + if (workers_to_wakeup > 0) { + waiting_workers_ -= workers_to_wakeup; + log::Debug() << "Wakeup " << workers_to_wakeup << " workers"; + sem_.release(static_cast(workers_to_wakeup)); + } +} + +} // namespace reactor diff --git a/lib/scheduler.cc b/lib/scheduler.cc deleted file mode 100644 index adabf944..00000000 --- a/lib/scheduler.cc +++ /dev/null @@ -1,291 +0,0 @@ -/* - * Copyright (C) 2019 TU Dresden - * All rights reserved. - * - * Authors: - * Christian Menard - */ - -#include - -#include "reactor-cpp/scheduler.hh" - -#include "reactor-cpp/action.hh" -#include "reactor-cpp/assert.hh" -#include "reactor-cpp/environment.hh" -#include "reactor-cpp/logging.hh" -#include "reactor-cpp/port.hh" -#include "reactor-cpp/reaction.hh" -#include "reactor-cpp/trace.hh" - -namespace reactor { - -void Scheduler::schedule() noexcept { - bool found_ready_reactions = schedule_ready_reactions(); - - while (!found_ready_reactions) { - log::Debug() << "(Scheduler) call next()"; - next(); - reaction_queue_pos_ = 0; - - found_ready_reactions = schedule_ready_reactions(); - - if (!continue_execution_ && !found_ready_reactions) { - // let all workers know that they should terminate - terminate_all_workers(); - break; - } - } -} - -void Scheduler::terminate_all_workers() { - log::Debug() << "(Scheduler) Send termination signal to all workers"; - auto num_workers = environment_->num_workers(); - std::vector null_reactions{num_workers, nullptr}; - log::Debug() << null_reactions.size(); - ready_queue_.fill_up(null_reactions); -} - -auto Scheduler::schedule_ready_reactions() -> bool { - // insert any triggered reactions_ into the reaction queue - for (auto& vec_reaction : triggered_reactions_) { - for (auto* reaction : vec_reaction) { - reaction_queue_[reaction->index()].push_back(reaction); - } - vec_reaction.clear(); - } - - log::Debug() << "(Scheduler) Scanning the reaction queue for ready reactions"; - - // continue iterating over the reaction queue - for (; reaction_queue_pos_ < reaction_queue_.size(); reaction_queue_pos_++) { - auto& reactions = reaction_queue_[reaction_queue_pos_]; - - // any ready reactions of current priority? - if (!reactions.empty()) { - log::Debug() << "(Scheduler) Process reactions of priority " << reaction_queue_pos_; - - // Make sure that any reaction is only executed once even if it - // was triggered multiple times. - std::sort(reactions.begin(), reactions.end()); - reactions.erase(std::unique(reactions.begin(), reactions.end()), reactions.end()); - - if constexpr (log::debug_enabled || tracing_enabled) { // NOLINT - for (auto* reaction : reactions) { - log::Debug() << "(Scheduler) Reaction " << reaction->fqn() << " is ready for execution"; - tracepoint(reactor_cpp, trigger_reaction, reaction->container()->fqn(), reaction->name(), logical_time_); - } - } - - reactions_to_process_.store(static_cast(reactions.size()), std::memory_order_release); - ready_queue_.fill_up(reactions); - - // break out of the loop and return - return true; - } - } - - log::Debug() << "(Scheduler) Reached end of reaction queue"; - return false; -} - -void Scheduler::start() { - log::Debug() << "Starting the scheduler..."; - - auto num_workers = environment_->num_workers(); - // initialize the reaction queue, set ports vector, and triggered reactions - // vector - reaction_queue_.resize(environment_->max_reaction_index() + 1); - set_ports_.resize(num_workers); - triggered_reactions_.resize(num_workers); - - // Initialize and start the workers. By resizing the workers vector first, - // we make sure that there is sufficient space for all the workers and non of - // them needs to be moved. This is important because a running worker may not - // be moved. - workers_.reserve(num_workers); - for (unsigned i = 0; i < num_workers; i++) { - workers_.emplace_back(policy_.create_worker()); - workers_.back().start(); - } - - // join all worker threads - for (auto& worker : workers_) { - worker.join(); - } -} - -void Scheduler::next() { // NOLINT - static EventMap events{}; - - // clean up before scheduling any new events - if (!events.empty()) { - // cleanup all triggered actions - for (auto& vec_ports : events) { - vec_ports.first->cleanup(); - } - // cleanup all set ports - for (auto& vec_ports : set_ports_) { - for (auto& port : vec_ports) { - port->cleanup(); - } - vec_ports.clear(); - } - events.clear(); - } - - { - std::unique_lock lock{scheduling_mutex_}; - - // shutdown if there are no more events in the queue - if (event_queue_.empty() && !stop_) { - if (environment_->run_forever()) { - // wait for a new asynchronous event - cv_schedule_.wait(lock, [this]() { return !event_queue_.empty() || stop_; }); - } else { - log::Debug() << "No more events in queue_. -> Terminate!"; - environment_->sync_shutdown(); - } - } - - while (events.empty()) { - if (stop_) { - continue_execution_ = false; - log::Debug() << "Shutting down the scheduler"; - Tag t_next = Tag::from_logical_time(logical_time_).delay(); - if (t_next == event_queue_.begin()->first) { - log::Debug() << "Schedule the last round of reactions including all " - "termination reactions"; - events = std::move(event_queue_.begin()->second); - event_queue_.erase(event_queue_.begin()); - log::Debug() << "advance logical time to tag [" << t_next.time_point() << ", " << t_next.micro_step() << "]"; - logical_time_.advance_to(t_next); - } else { - return; - } - } else { - // collect events of the next tag - auto t_next = event_queue_.begin()->first; - - // synchronize with physical time if not in fast forward mode - if (!environment_->fast_fwd_execution()) { - // keep track of the current physical time in a static variable - static auto physical_time = TimePoint::min(); - - // If physical time is smaller than the next logical time point, - // then update the physical time. This step is small optimization to - // avoid calling get_physical_time() in every iteration as this - // would add a significant overhead. - if (physical_time < t_next.time_point()) { - physical_time = get_physical_time(); - } - - // If physical time is still smaller than the next logical time - // point, then wait until the next tag or until a new event is - // inserted asynchronously into the queue - if (physical_time < t_next.time_point()) { - auto status = cv_schedule_.wait_until(lock, t_next.time_point()); - // Start over if the event queue was modified - if (status == std::cv_status::no_timeout) { - continue; - } - // update physical time and continue otherwise - physical_time = t_next.time_point(); - } - } - - // retrieve all events with tag equal to current logical time from the - // queue - events = std::move(event_queue_.begin()->second); - event_queue_.erase(event_queue_.begin()); - - // advance logical time - log::Debug() << "advance logical time to tag [" << t_next.time_point() << ", " << t_next.micro_step() << "]"; - logical_time_.advance_to(t_next); - } - } - } // mutex schedule_ - - // execute all setup functions; this sets the values of the corresponding - // actions - for (auto& vec_reactor : events) { - auto& setup = vec_reactor.second; - if (setup != nullptr) { - setup(); - } - } - - log::Debug() << "events: " << events.size(); - for (auto& vec_reactor : events) { - log::Debug() << "Action " << vec_reactor.first->fqn(); - for (auto* reaction : vec_reactor.first->triggers()) { - // There is no need to acquire the mutex. At this point the scheduler - // should be the only thread accessing the reaction queue as none of the - // workers_ are running - log::Debug() << "insert reaction " << reaction->fqn() << " with index " << reaction->index(); - reaction_queue_[reaction->index()].push_back(reaction); - } - } -} - -Scheduler::Scheduler(Environment* env) - : policy_(*this, env->num_workers()) - , using_workers_(env->num_workers() > 1) - , environment_(env) {} - -Scheduler::~Scheduler() = default; - -void Scheduler::schedule_sync(const Tag& tag, BaseAction* action, std::function pre_handler) { - reactor_assert(logical_time_ < tag); - // TODO verify that the action is indeed allowed to be scheduled by the - // current reaction - log::Debug() << "Schedule action " << action->fqn() << (action->is_logical() ? " synchronously " : " asynchronously ") - << " with tag [" << tag.time_point() << ", " << tag.micro_step() << "]"; - { - auto unique_lock = - using_workers_ ? std::unique_lock(lock_event_queue_) : std::unique_lock(); - - tracepoint(reactor_cpp, schedule_action, action->container()->fqn(), action->name(), tag); // NOLINT - - // create a new event map or retrieve the existing one - auto emplace_result = event_queue_.try_emplace(tag, EventMap()); - auto& event_map = emplace_result.first->second; - - // insert the new event - event_map[action] = std::move(pre_handler); - } -} - -void Scheduler::schedule_async(const Tag& tag, BaseAction* action, std::function pre_handler) { - std::lock_guard lock_guard(scheduling_mutex_); - schedule_sync(tag, action, std::move(pre_handler)); - cv_schedule_.notify_one(); -} - -void Scheduler::set_port(BasePort* port) { - log::Debug() << "Set port " << port->fqn(); - - // We do not check here if port is already in the list. This means clean() - // could be called multiple times for a single port. However, calling - // clean() multiple time is not harmful and more efficient then checking if - set_ports_[DefaultWorker::current_worker_id()].push_back(port); - - // recursively search for triggered reactions - set_port_helper(port); -} - -void Scheduler::set_port_helper(BasePort* port) { - for (auto* reaction : port->triggers()) { - triggered_reactions_[DefaultWorker::current_worker_id()].push_back(reaction); - } - for (auto* binding : port->outward_bindings()) { - set_port_helper(binding); - } -} - -void Scheduler::stop() { - stop_ = true; - cv_schedule_.notify_one(); -} - -} // namespace reactor From 09aceceab07f3cfebd43e8a0719c6d3eabaad67b Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Fri, 22 Jul 2022 15:45:50 +0200 Subject: [PATCH 30/49] factor out a base scheduler interface --- include/reactor-cpp/action.hh | 2 + include/reactor-cpp/assert.hh | 1 + include/reactor-cpp/base_scheduler.hh | 60 ++++++++++++++++++++++ include/reactor-cpp/environment.hh | 21 ++++---- include/reactor-cpp/fwd.hh | 1 + include/reactor-cpp/impl/action_impl.hh | 33 ++++++------ include/reactor-cpp/impl/port_impl.hh | 7 ++- include/reactor-cpp/impl/scheduler_impl.hh | 38 ++------------ include/reactor-cpp/logging.hh | 3 +- include/reactor-cpp/port.hh | 2 + include/reactor-cpp/reactor-cpp.hh | 7 +-- include/reactor-cpp/scheduler.hh | 42 ++++----------- lib/CMakeLists.txt | 1 + lib/action.cc | 9 ++-- lib/base_scheduler.cc | 52 +++++++++++++++++++ lib/environment.cc | 23 +++++++-- lib/port.cc | 7 +-- lib/reactor.cc | 9 ++-- 18 files changed, 200 insertions(+), 118 deletions(-) create mode 100644 include/reactor-cpp/base_scheduler.hh create mode 100644 lib/base_scheduler.cc diff --git a/include/reactor-cpp/action.hh b/include/reactor-cpp/action.hh index 7b8c7341..0793df7c 100644 --- a/include/reactor-cpp/action.hh +++ b/include/reactor-cpp/action.hh @@ -154,4 +154,6 @@ public: } // namespace reactor +#include "reactor-cpp/impl/action_impl.hh" + #endif // REACTOR_CPP_ACTION_HH diff --git a/include/reactor-cpp/assert.hh b/include/reactor-cpp/assert.hh index 424ebfdf..03529a38 100644 --- a/include/reactor-cpp/assert.hh +++ b/include/reactor-cpp/assert.hh @@ -24,6 +24,7 @@ constexpr bool runtime_assertion = true; #include "environment.hh" #include +#include #include #include #include diff --git a/include/reactor-cpp/base_scheduler.hh b/include/reactor-cpp/base_scheduler.hh new file mode 100644 index 00000000..0567a29c --- /dev/null +++ b/include/reactor-cpp/base_scheduler.hh @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2022 TU Dresden + * All rights reserved. + * + * Authors: + * Christian Menard + */ + +#ifndef REACTOR_CPP_BASE_SCHEDULER_HH +#define REACTOR_CPP_BASE_SCHEDULER_HH + +#include +#include +#include +#include +#include + +#include "reactor-cpp/environment.hh" +#include "reactor-cpp/fwd.hh" +#include "reactor-cpp/logical_time.hh" + +namespace reactor { + +using EventMap = std::map>; + +class BaseScheduler { +protected: + Environment* environment_; // NOLINT + + LogicalTime logical_time_{}; // NOLINT + const bool using_workers_; // NOLINT + + std::mutex scheduling_mutex_; // NOLINT + std::unique_lock scheduling_lock_{scheduling_mutex_, std::defer_lock}; // NOLINT + std::condition_variable cv_schedule_; // NOLINT + + std::mutex lock_event_queue_; // NOLINT + std::map event_queue_; // NOLINT + +public: + BaseScheduler(Environment* env); + virtual ~BaseScheduler() = default; + BaseScheduler(const BaseScheduler&) = delete; + BaseScheduler(BaseScheduler&&) = delete; + auto operator=(const BaseScheduler&) -> BaseScheduler& = delete; + auto operator=(BaseScheduler&&) -> BaseScheduler& = delete; + + [[nodiscard]] inline auto logical_time() const noexcept -> const auto& { return logical_time_; } + void schedule_sync(const Tag& tag, BaseAction* action, std::function pre_handler); + void schedule_async(const Tag& tag, BaseAction* action, std::function pre_handler); + + virtual void set_port(BasePort* port) = 0; + + void inline lock() noexcept { scheduling_lock_.lock(); } + void inline unlock() noexcept { scheduling_lock_.unlock(); } +}; + +} // namespace reactor + +#endif // REACTOR_CPP_BASE_SCHEDULER_HH diff --git a/include/reactor-cpp/environment.hh b/include/reactor-cpp/environment.hh index 4e2322af..8973b5c3 100644 --- a/include/reactor-cpp/environment.hh +++ b/include/reactor-cpp/environment.hh @@ -9,14 +9,16 @@ #ifndef REACTOR_CPP_ENVIRONMENT_HH #define REACTOR_CPP_ENVIRONMENT_HH +#include #include #include +#include +#include #include #include "reactor-cpp/default_scheduling_policy.hh" #include "reactor-cpp/fwd.hh" #include "reactor-cpp/reactor.hh" -#include "reactor-cpp/scheduler.hh" namespace reactor { @@ -40,7 +42,7 @@ private: std::set reactions_{}; std::vector dependencies_{}; - Scheduler scheduler_; + std::unique_ptr> scheduler_; Phase phase_{Phase::Construction}; TimePoint start_time_{}; @@ -49,11 +51,7 @@ private: public: explicit Environment(unsigned int num_workers, bool run_forever = default_run_forever, - bool fast_fwd_execution = default_fast_fwd_execution) - : num_workers_(num_workers) - , run_forever_(run_forever) - , fast_fwd_execution_(fast_fwd_execution) - , scheduler_(this) {} + bool fast_fwd_execution = default_fast_fwd_execution); void register_reactor(Reactor* reactor); void assemble(); @@ -72,14 +70,13 @@ public: [[nodiscard]] auto top_level_reactors() const noexcept -> const auto& { return top_level_reactors_; } [[nodiscard]] auto phase() const noexcept -> Phase { return phase_; } - [[nodiscard]] auto scheduler() const noexcept -> const Scheduler* { return &scheduler_; } + [[nodiscard]] auto scheduler() const noexcept -> const BaseScheduler&; + [[nodiscard]] auto scheduler() noexcept -> BaseScheduler&; - auto scheduler() noexcept -> Scheduler* { return &scheduler_; } - - [[nodiscard]] auto logical_time() const noexcept -> const LogicalTime& { return scheduler_.logical_time(); } + [[nodiscard]] auto logical_time() const noexcept -> const LogicalTime&; [[nodiscard]] auto start_time() const noexcept -> const TimePoint& { return start_time_; } - static auto physical_time() noexcept -> TimePoint { return get_physical_time(); } + [[nodiscard]] static auto physical_time() noexcept -> TimePoint { return get_physical_time(); } [[nodiscard]] auto num_workers() const noexcept -> unsigned int { return num_workers_; } [[nodiscard]] auto fast_fwd_execution() const noexcept -> bool { return fast_fwd_execution_; } diff --git a/include/reactor-cpp/fwd.hh b/include/reactor-cpp/fwd.hh index 2177f92b..7494ac02 100644 --- a/include/reactor-cpp/fwd.hh +++ b/include/reactor-cpp/fwd.hh @@ -18,6 +18,7 @@ class Reaction; class Reactor; class Tag; +class BaseScheduler; class DefaultSchedulingPolicy; template class Worker; template class Scheduler; diff --git a/include/reactor-cpp/impl/action_impl.hh b/include/reactor-cpp/impl/action_impl.hh index 3ed4fb12..c2dca9bd 100644 --- a/include/reactor-cpp/impl/action_impl.hh +++ b/include/reactor-cpp/impl/action_impl.hh @@ -9,40 +9,41 @@ #ifndef REACTOR_CPP_IMPL_ACTION_IMPL_HH #define REACTOR_CPP_IMPL_ACTION_IMPL_HH -#include "../assert.hh" -#include "../environment.hh" +#include "reactor-cpp/assert.hh" +#include "reactor-cpp/base_scheduler.hh" +#include "reactor-cpp/environment.hh" namespace reactor { template template void Action::schedule(const ImmutableValuePtr& value_ptr, Dur delay) { - Duration time_delay = std::chrono::duration_cast(delay); // NOLINT + Duration time_delay = std::chrono::duration_cast(delay); reactor::validate(time_delay >= Duration::zero(), "Schedule cannot be called with a negative delay!"); reactor::validate(value_ptr != nullptr, "Actions may not be scheduled with a nullptr value!"); - auto* scheduler = environment()->scheduler(); // NOLINT - auto setup = [value_ptr, this]() { this->value_ptr_ = std::move(value_ptr); }; // NOLINT + auto& scheduler = environment()->scheduler(); + auto setup = [value_ptr, this]() { this->value_ptr_ = std::move(value_ptr); }; if (is_logical()) { time_delay += this->min_delay(); - auto tag = Tag::from_logical_time(scheduler->logical_time()).delay(time_delay); // NOLINT - scheduler->schedule_sync(tag, this, setup); + auto tag = Tag::from_logical_time(scheduler.logical_time()).delay(time_delay); + scheduler.schedule_sync(tag, this, setup); } else { - auto tag = Tag::from_physical_time(get_physical_time() + time_delay); // NOLINT - scheduler->schedule_async(tag, this, setup); + auto tag = Tag::from_physical_time(get_physical_time() + time_delay); + scheduler.schedule_async(tag, this, setup); } } template void Action::schedule(Dur delay) { - auto time_delay = std::chrono::duration_cast(delay); // NOLINT + auto time_delay = std::chrono::duration_cast(delay); reactor::validate(time_delay >= Duration::zero(), "Schedule cannot be called with a negative delay!"); - auto* scheduler = environment()->scheduler(); // NOLINT - auto setup = [this]() { this->present_ = true; }; // NOLINT + auto& scheduler = environment()->scheduler(); + auto setup = [this]() { this->present_ = true; }; if (is_logical()) { time_delay += this->min_delay(); - auto tag = Tag::from_logical_time(scheduler->logical_time()).delay(time_delay); // NOLINT - scheduler->schedule_sync(tag, this, setup); + auto tag = Tag::from_logical_time(scheduler.logical_time()).delay(time_delay); + scheduler.schedule_sync(tag, this, setup); } else { // physical action - auto tag = Tag::from_physical_time(get_physical_time() + time_delay); // NOLINT - scheduler->schedule_async(tag, this, setup); + auto tag = Tag::from_physical_time(get_physical_time() + time_delay); + scheduler.schedule_async(tag, this, setup); } } diff --git a/include/reactor-cpp/impl/port_impl.hh b/include/reactor-cpp/impl/port_impl.hh index 64943a99..6fac48cd 100644 --- a/include/reactor-cpp/impl/port_impl.hh +++ b/include/reactor-cpp/impl/port_impl.hh @@ -9,7 +9,10 @@ #ifndef REACTOR_CPP_IMPL_PORT_IMPL_HH #define REACTOR_CPP_IMPL_PORT_IMPL_HH +#include "reactor-cpp/port.hh" + #include "reactor-cpp/assert.hh" +#include "reactor-cpp/base_scheduler.hh" #include "reactor-cpp/environment.hh" namespace reactor { @@ -31,9 +34,9 @@ template void Port::set(const ImmutableValuePtr& value_ptr) { reactor::validate(!has_inward_binding(), "set() may only be called on ports that do not have an inward " "binding!"); reactor::validate(value_ptr != nullptr, "Ports may not be set to nullptr!"); - auto scheduler = environment()->scheduler(); + auto& scheduler = environment()->scheduler(); this->value_ptr_ = std::move(value_ptr); - scheduler->set_port(this); + scheduler.set_port(this); } template auto Port::get() const noexcept -> const ImmutableValuePtr& { diff --git a/include/reactor-cpp/impl/scheduler_impl.hh b/include/reactor-cpp/impl/scheduler_impl.hh index 10c01350..ebfd01e4 100644 --- a/include/reactor-cpp/impl/scheduler_impl.hh +++ b/include/reactor-cpp/impl/scheduler_impl.hh @@ -11,12 +11,12 @@ #include "reactor-cpp/action.hh" #include "reactor-cpp/assert.hh" -#include "reactor-cpp/environment.hh" #include "reactor-cpp/fwd.hh" #include "reactor-cpp/logging.hh" #include "reactor-cpp/port.hh" #include "reactor-cpp/reaction.hh" #include "reactor-cpp/trace.hh" + #include namespace reactor { @@ -65,9 +65,8 @@ template void Worker::execute_reactio template Scheduler::Scheduler(Environment* env) - : policy_(*this, *env) - , using_workers_(env->num_workers() > 1) - , environment_(env) {} + : BaseScheduler(env) + , policy_(*this, *env) {} template void Scheduler::start() { log::Debug() << "Starting the scheduler..."; @@ -212,37 +211,6 @@ template auto Scheduler::next() -> bo return continue_execution; } -template -void Scheduler::schedule_sync(const Tag& tag, BaseAction* action, - std::function pre_handler) { - reactor_assert(logical_time_ < tag); - // TODO verify that the action is indeed allowed to be scheduled by the - // current reaction - log::Debug() << "Schedule action " << action->fqn() << (action->is_logical() ? " synchronously " : " asynchronously ") - << " with tag [" << tag.time_point() << ", " << tag.micro_step() << "]"; - { - auto unique_lock = - using_workers_ ? std::unique_lock(lock_event_queue_) : std::unique_lock(); - - tracepoint(reactor_cpp, schedule_action, action->container()->fqn(), action->name(), tag); // NOLINT - - // create a new event map or retrieve the existing one - auto emplace_result = event_queue_.try_emplace(tag, EventMap()); - auto& event_map = emplace_result.first->second; - - // insert the new event - event_map[action] = std::move(pre_handler); - } -} - -template -void Scheduler::schedule_async(const Tag& tag, BaseAction* action, - std::function pre_handler) { - std::lock_guard lock_guard(scheduling_mutex_); - schedule_sync(tag, action, std::move(pre_handler)); - cv_schedule_.notify_one(); -} - template void Scheduler::set_port(BasePort* port) { log::Debug() << "Set port " << port->fqn(); diff --git a/include/reactor-cpp/logging.hh b/include/reactor-cpp/logging.hh index e27863fc..9c738238 100644 --- a/include/reactor-cpp/logging.hh +++ b/include/reactor-cpp/logging.hh @@ -9,8 +9,9 @@ #ifndef REACTOR_CPP_LOGGING_HH #define REACTOR_CPP_LOGGING_HH -#include "reactor-cpp/config.hh" //NOLINT +#include "reactor-cpp/config.hh" #include "reactor-cpp/time.hh" + #include #include #include diff --git a/include/reactor-cpp/port.hh b/include/reactor-cpp/port.hh index 7e8c6afc..b8dd8d8e 100644 --- a/include/reactor-cpp/port.hh +++ b/include/reactor-cpp/port.hh @@ -127,4 +127,6 @@ public: } // namespace reactor +#include "reactor-cpp/impl/port_impl.hh" + #endif // REACTOR_CPP_PORT_HH diff --git a/include/reactor-cpp/reactor-cpp.hh b/include/reactor-cpp/reactor-cpp.hh index 8f15d968..56155a3c 100644 --- a/include/reactor-cpp/reactor-cpp.hh +++ b/include/reactor-cpp/reactor-cpp.hh @@ -17,12 +17,7 @@ #include "port.hh" #include "reaction.hh" #include "reactor.hh" +#include "scheduler.hh" #include "time.hh" -// include implementation header files - -#include "impl/action_impl.hh" -#include "impl/port_impl.hh" -#include "impl/scheduler_impl.hh" - #endif // REACTOR_CPP_REACTOR_CPP_HH diff --git a/include/reactor-cpp/scheduler.hh b/include/reactor-cpp/scheduler.hh index c413691e..76184e06 100644 --- a/include/reactor-cpp/scheduler.hh +++ b/include/reactor-cpp/scheduler.hh @@ -19,8 +19,9 @@ #include #include -#include "fwd.hh" -#include "logical_time.hh" +#include "reactor-cpp/base_scheduler.hh" +#include "reactor-cpp/fwd.hh" +#include "reactor-cpp/logical_time.hh" namespace reactor { @@ -57,33 +58,18 @@ public: friend SchedulingPolicy; }; -using EventMap = std::map>; - -template class Scheduler { +template class Scheduler : public BaseScheduler { private: using worker_t = Worker; SchedulingPolicy policy_; - const bool using_workers_; - LogicalTime logical_time_{}; - - Environment* environment_; std::vector workers_{}; - std::mutex scheduling_mutex_; - std::unique_lock scheduling_lock_{scheduling_mutex_, std::defer_lock}; - std::condition_variable cv_schedule_; - - std::mutex lock_event_queue_; - std::map event_queue_; - std::vector> set_ports_; std::atomic stop_{false}; - std::vector> triggered_reactions_; - auto next() -> bool; void set_port_helper(BasePort* port); @@ -91,26 +77,20 @@ public: explicit Scheduler(Environment* env); Scheduler(Scheduler&&) = delete; Scheduler(const Scheduler&) = delete; - ~Scheduler() = default; - auto operator=(Scheduler&&) -> Scheduler = delete; - auto operator=(const Scheduler&) -> Scheduler = delete; - - void schedule_sync(const Tag& tag, BaseAction* action, std::function pre_handler); - void schedule_async(const Tag& tag, BaseAction* action, std::function pre_handler); - - void inline lock() noexcept { scheduling_lock_.lock(); } - void inline unlock() noexcept { scheduling_lock_.unlock(); } - - void set_port(BasePort* port); - - [[nodiscard]] inline auto logical_time() const noexcept -> const auto& { return logical_time_; } + ~Scheduler() override = default; + auto operator=(Scheduler&&) -> Scheduler& = delete; + auto operator=(const Scheduler&) -> Scheduler& = delete; void start(); void stop(); + void set_port(BasePort* port) override; + friend SchedulingPolicy; }; } // namespace reactor +#include "reactor-cpp/impl/scheduler_impl.hh" + #endif // REACTOR_CPP_SCHEDULER_HH diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index d0b7c071..64a444f9 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -1,6 +1,7 @@ set(SOURCE_FILES action.cc assert.cc + base_scheduler.cc default_scheduling_policy.cc environment.cc logical_time.cc diff --git a/lib/action.cc b/lib/action.cc index 87eb643c..97a48760 100644 --- a/lib/action.cc +++ b/lib/action.cc @@ -9,6 +9,7 @@ #include "reactor-cpp/action.hh" #include "reactor-cpp/assert.hh" +#include "reactor-cpp/base_scheduler.hh" #include "reactor-cpp/environment.hh" #include "reactor-cpp/reaction.hh" @@ -37,9 +38,9 @@ void BaseAction::register_scheduler(Reaction* reaction) { void Timer::startup() { Tag tag_zero = Tag::from_physical_time(environment()->start_time()); if (offset_ != Duration::zero()) { - environment()->scheduler()->schedule_sync(tag_zero.delay(offset_), this, nullptr); + environment()->scheduler().schedule_sync(tag_zero.delay(offset_), this, nullptr); } else { - environment()->scheduler()->schedule_sync(tag_zero, this, nullptr); + environment()->scheduler().schedule_sync(tag_zero, this, nullptr); } } @@ -48,13 +49,13 @@ void Timer::cleanup() { if (period_ != Duration::zero()) { Tag now = Tag::from_logical_time(environment()->logical_time()); Tag next = now.delay(period_); - environment()->scheduler()->schedule_sync(next, this, nullptr); + environment()->scheduler().schedule_sync(next, this, nullptr); } } void ShutdownAction::shutdown() { Tag tag = Tag::from_logical_time(environment()->logical_time()).delay(); - environment()->scheduler()->schedule_sync(tag, this, nullptr); + environment()->scheduler().schedule_sync(tag, this, nullptr); } } // namespace reactor diff --git a/lib/base_scheduler.cc b/lib/base_scheduler.cc new file mode 100644 index 00000000..b3730a92 --- /dev/null +++ b/lib/base_scheduler.cc @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2022 TU Dresden + * All rights reserved. + * + * Authors: + * Christian Menard + */ + +#include "reactor-cpp/base_scheduler.hh" + +#include "reactor-cpp/action.hh" +#include "reactor-cpp/assert.hh" +#include "reactor-cpp/environment.hh" +#include "reactor-cpp/fwd.hh" +#include "reactor-cpp/logging.hh" +#include "reactor-cpp/port.hh" +#include "reactor-cpp/reaction.hh" +#include "reactor-cpp/trace.hh" + +namespace reactor { + +BaseScheduler::BaseScheduler(Environment* env) + : environment_(env) + , using_workers_(env->num_workers() > 1) {} + +void BaseScheduler::schedule_sync(const Tag& tag, BaseAction* action, std::function pre_handler) { + reactor_assert(logical_time_ < tag); + // TODO verify that the action is indeed allowed to be scheduled by the + // current reaction + log::Debug() << "Schedule action " << action->fqn() << (action->is_logical() ? " synchronously " : " asynchronously ") + << " with tag [" << tag.time_point() << ", " << tag.micro_step() << "]"; + { + auto unique_lock = + using_workers_ ? std::unique_lock(lock_event_queue_) : std::unique_lock(); + + tracepoint(reactor_cpp, schedule_action, action->container()->fqn(), action->name(), tag); // NOLINT + + // create a new event map or retrieve the existing one + auto emplace_result = event_queue_.try_emplace(tag, EventMap()); + auto& event_map = emplace_result.first->second; + + // insert the new event + event_map[action] = std::move(pre_handler); + } +} + +void BaseScheduler::schedule_async(const Tag& tag, BaseAction* action, std::function pre_handler) { + std::lock_guard lock_guard(scheduling_mutex_); + schedule_sync(tag, action, std::move(pre_handler)); + cv_schedule_.notify_one(); +} +} // namespace reactor diff --git a/lib/environment.cc b/lib/environment.cc index 0429e7da..0a109116 100644 --- a/lib/environment.cc +++ b/lib/environment.cc @@ -14,12 +14,27 @@ #include "reactor-cpp/action.hh" #include "reactor-cpp/assert.hh" +#include "reactor-cpp/default_scheduling_policy.hh" #include "reactor-cpp/logging.hh" #include "reactor-cpp/port.hh" #include "reactor-cpp/reaction.hh" +#include "reactor-cpp/scheduler.hh" namespace reactor { +Environment::Environment(unsigned int num_workers, bool run_forever, bool fast_fwd_execution) + : num_workers_(num_workers) + , run_forever_(run_forever) + , fast_fwd_execution_(fast_fwd_execution) + , scheduler_(std::make_unique>(this)) {} + +[[nodiscard]] auto Environment::scheduler() const noexcept -> const BaseScheduler& { return *scheduler_; } +[[nodiscard]] auto Environment::scheduler() noexcept -> BaseScheduler& { return *scheduler_; } + +[[nodiscard]] auto Environment::logical_time() const noexcept -> const LogicalTime& { + return scheduler_->logical_time(); +} + void Environment::register_reactor(Reactor* reactor) { reactor_assert(reactor != nullptr); validate(this->phase() == Phase::Construction, "Reactors may only be registered during construction phase!"); @@ -97,13 +112,13 @@ void Environment::sync_shutdown() { } phase_ = Phase::Deconstruction; - scheduler_.stop(); + scheduler_->stop(); } void Environment::async_shutdown() { - scheduler_.lock(); + scheduler_->lock(); sync_shutdown(); - scheduler_.unlock(); + scheduler_->unlock(); } auto dot_name([[maybe_unused]] ReactorElement* reactor_element) -> std::string { @@ -227,7 +242,7 @@ auto Environment::startup() -> std::thread { // start processing events phase_ = Phase::Execution; - return std::thread([this]() { this->scheduler_.start(); }); + return std::thread([this]() { this->scheduler_->start(); }); } void Environment::dump_trigger_to_yaml(std::ofstream& yaml, const BaseAction& trigger) { diff --git a/lib/port.cc b/lib/port.cc index c626475b..16244757 100644 --- a/lib/port.cc +++ b/lib/port.cc @@ -9,6 +9,7 @@ #include "reactor-cpp/port.hh" #include "reactor-cpp/assert.hh" +#include "reactor-cpp/base_scheduler.hh" #include "reactor-cpp/environment.hh" #include "reactor-cpp/reaction.hh" @@ -28,7 +29,7 @@ void BasePort::base_bind_to(BasePort* port) { } else if (this->is_output() && port->is_input()) { validate(this->container()->container() == port->container()->container(), "An output port can only be bound to an input port if both ports " - "belong to reactors in the same hierarichal level"); + "belong to reactors in the same hierarchical level"); } else if (this->is_output() && port->is_output()) { validate(this->container()->container() == port->container(), "An output port A may only be bound to another output port B if A is " @@ -93,9 +94,9 @@ auto Port::typed_inward_binding() const noexcept -> Port* { void Port::set() { validate(!has_inward_binding(), "set() may only be called on a ports that do not have an inward " "binding!"); - auto* scheduler = environment()->scheduler(); + auto& scheduler = environment()->scheduler(); this->present_ = true; - scheduler->set_port(this); + scheduler.set_port(this); } auto Port::is_present() const noexcept -> bool { diff --git a/lib/reactor.cc b/lib/reactor.cc index b3bf0c09..85b3b9af 100644 --- a/lib/reactor.cc +++ b/lib/reactor.cc @@ -10,6 +10,7 @@ #include "reactor-cpp/action.hh" #include "reactor-cpp/assert.hh" +#include "reactor-cpp/base_scheduler.hh" #include "reactor-cpp/environment.hh" #include "reactor-cpp/logging.hh" #include "reactor-cpp/port.hh" @@ -30,7 +31,7 @@ ReactorElement::ReactorElement(const std::string& name, ReactorElement::Type typ // completely constructed objects. Technically, the casts here return // invalid pointers as the objects they point to do not yet // exists. However, we are good as long as we only store the pointer and do - // not dereference it before construction is completeted. + // not de reference it before construction is completed. // It works, but maybe there is some nicer way of doing this... switch (type) { case Type::Action: @@ -154,15 +155,15 @@ void Reactor::shutdown() { auto Reactor::get_physical_time() noexcept -> TimePoint { return reactor::get_physical_time(); } auto Reactor::get_logical_time() const noexcept -> TimePoint { - return environment()->scheduler()->logical_time().time_point(); + return environment()->scheduler().logical_time().time_point(); } auto Reactor::get_microstep() const noexcept -> mstep_t { - return environment()->scheduler()->logical_time().micro_step(); + return environment()->scheduler().logical_time().micro_step(); } auto Reactor::get_tag() const noexcept -> Tag { - return Tag::from_logical_time(environment()->scheduler()->logical_time()); + return Tag::from_logical_time(environment()->scheduler().logical_time()); } auto Reactor::get_elapsed_logical_time() const noexcept -> Duration { From b57f6ac4087432096e73f8ff94a84aa43cf81ec3 Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Fri, 22 Jul 2022 15:47:59 +0200 Subject: [PATCH 31/49] apply clang-format --- include/reactor-cpp/value_ptr.hh | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/include/reactor-cpp/value_ptr.hh b/include/reactor-cpp/value_ptr.hh index fcb0e20e..edc6f366 100644 --- a/include/reactor-cpp/value_ptr.hh +++ b/include/reactor-cpp/value_ptr.hh @@ -75,7 +75,6 @@ template auto make_mutable_value(Args&&... args) -> Mut } } - namespace detail { /** @@ -510,7 +509,6 @@ public: friend auto reactor::make_immutable_value(Args&&... args) -> reactor::ImmutableValuePtr; }; - // Comparison operators template @@ -529,16 +527,20 @@ template auto operator==(const MutableValuePtr& ptr1, const ImmutableValuePtr& ptr2) noexcept -> bool { return ptr1.get() == ptr2.get(); } -template auto operator==(const MutableValuePtr& ptr1, std::nullptr_t) noexcept -> bool { +template +auto operator==(const MutableValuePtr& ptr1, std::nullptr_t) noexcept -> bool { return ptr1.get() == nullptr; } -template auto operator==(std::nullptr_t, const MutableValuePtr& ptr2) noexcept -> bool { +template +auto operator==(std::nullptr_t, const MutableValuePtr& ptr2) noexcept -> bool { return ptr2.get() == nullptr; } -template auto operator==(const ImmutableValuePtr& ptr1, std::nullptr_t) noexcept -> bool { +template +auto operator==(const ImmutableValuePtr& ptr1, std::nullptr_t) noexcept -> bool { return ptr1.get() == nullptr; } -template auto operator==(std::nullptr_t, const ImmutableValuePtr& ptr1) noexcept -> bool { +template +auto operator==(std::nullptr_t, const ImmutableValuePtr& ptr1) noexcept -> bool { return ptr1.get() == nullptr; } @@ -551,10 +553,12 @@ template auto operator!=(const ImmutableValuePtr& ptr1, const ImmutableValuePtr& ptr2) -> bool { return ptr1.get() != ptr2.get(); } -template auto operator!=(const ImmutableValuePtr& ptr1, const MutableValuePtr& ptr2) -> bool { +template +auto operator!=(const ImmutableValuePtr& ptr1, const MutableValuePtr& ptr2) -> bool { return ptr1.get() != ptr2.get(); } -template auto operator!=(const MutableValuePtr& ptr1, const ImmutableValuePtr& ptr2) -> bool { +template +auto operator!=(const MutableValuePtr& ptr1, const ImmutableValuePtr& ptr2) -> bool { return ptr1.get() != ptr2.get(); } template auto operator!=(const MutableValuePtr& ptr1, std::nullptr_t) -> bool { @@ -570,7 +574,6 @@ template auto operator!=(std::nullptr_t, const ImmutableV return ptr1.get() != nullptr; } - } // namespace detail } // namespace reactor From b8e3976be6f6fb26e8a330b96f4e0479ba1c0456 Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Fri, 22 Jul 2022 15:49:17 +0200 Subject: [PATCH 32/49] fix conversion warning --- include/reactor-cpp/impl/scheduler_impl.hh | 2 +- include/reactor-cpp/scheduler.hh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/reactor-cpp/impl/scheduler_impl.hh b/include/reactor-cpp/impl/scheduler_impl.hh index ebfd01e4..327c8cb6 100644 --- a/include/reactor-cpp/impl/scheduler_impl.hh +++ b/include/reactor-cpp/impl/scheduler_impl.hh @@ -25,7 +25,7 @@ template // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) thread_local std::size_t Worker::current_worker_id_{0}; -template auto Worker::current_worker_id() -> unsigned { +template auto Worker::current_worker_id() -> std::size_t { return current_worker_id_; } diff --git a/include/reactor-cpp/scheduler.hh b/include/reactor-cpp/scheduler.hh index 76184e06..924a6e26 100644 --- a/include/reactor-cpp/scheduler.hh +++ b/include/reactor-cpp/scheduler.hh @@ -53,7 +53,7 @@ public: [[nodiscard]] auto id() const -> std::size_t { return identity_; } - static auto current_worker_id() -> unsigned; + static auto current_worker_id() -> std::size_t; friend SchedulingPolicy; }; From 70bfe97b869113dc6317d9faa4c780f75a7eac1d Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Tue, 26 Jul 2022 11:25:37 +0200 Subject: [PATCH 33/49] delete the group scheduler again --- include/reactor-cpp/environment.hh | 12 +- include/reactor-cpp/group_scheduler.hh | 145 --------- lib/CMakeLists.txt | 1 - lib/group_scheduler.cc | 398 ------------------------- 4 files changed, 6 insertions(+), 550 deletions(-) delete mode 100644 include/reactor-cpp/group_scheduler.hh delete mode 100644 lib/group_scheduler.cc diff --git a/include/reactor-cpp/environment.hh b/include/reactor-cpp/environment.hh index 0c70489c..bbe47dcf 100644 --- a/include/reactor-cpp/environment.hh +++ b/include/reactor-cpp/environment.hh @@ -14,9 +14,9 @@ #include #include -#include "dependency_graph.hh" -#include "group_scheduler.hh" -#include "reactor.hh" +#include "reactor-cpp/dependency_graph.hh" +#include "reactor-cpp/reactor.hh" +#include "reactor-cpp/scheduler.hh" namespace reactor { @@ -40,7 +40,7 @@ private: std::set reactions_{}; std::vector dependencies_{}; - GroupScheduler scheduler_; + Scheduler scheduler_; Phase phase_{Phase::Construction}; TimePoint start_time_{}; @@ -74,9 +74,9 @@ public: [[nodiscard]] auto top_level_reactors() const noexcept -> const auto& { return top_level_reactors_; } [[nodiscard]] auto phase() const noexcept -> Phase { return phase_; } - [[nodiscard]] auto scheduler() const noexcept -> const GroupScheduler* { return &scheduler_; } + [[nodiscard]] auto scheduler() const noexcept -> const Scheduler* { return &scheduler_; } - auto scheduler() noexcept -> GroupScheduler* { return &scheduler_; } + auto scheduler() noexcept -> Scheduler* { return &scheduler_; } [[nodiscard]] auto logical_time() const noexcept -> const LogicalTime& { return scheduler_.logical_time(); } [[nodiscard]] auto start_time() const noexcept -> const TimePoint& { return start_time_; } diff --git a/include/reactor-cpp/group_scheduler.hh b/include/reactor-cpp/group_scheduler.hh deleted file mode 100644 index 38a53f60..00000000 --- a/include/reactor-cpp/group_scheduler.hh +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Copyright (C) 2019 TU Dresden - * All rights reserved. - * - * Authors: - * Christian Menard - */ - -#ifndef REACTOR_CPP_GROUP_SCHEDULER_HH -#define REACTOR_CPP_GROUP_SCHEDULER_HH - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "fwd.hh" -#include "logical_time.hh" -#include "semaphore.hh" - -namespace reactor { - -// forward declarations -class GroupScheduler; -class GroupWorker; - -class GroupWorker { // NOLINT -public: - GroupScheduler& scheduler_; - const unsigned int identity_{0}; - std::thread thread_{}; - - // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) - static thread_local const GroupWorker* current_worker; - - void work() const; - void execute_reaction(Reaction* reaction) const; - - GroupWorker(GroupScheduler& scheduler, unsigned int identity) - : scheduler_{scheduler} - , identity_{identity} {} - GroupWorker(GroupWorker&& worker); // NOLINT(performance-noexcept-move-constructor) - GroupWorker(const GroupWorker& worker) = delete; - - void start_thread() { thread_ = std::thread(&GroupWorker::work, this); } - void join_thread() { thread_.join(); } - - static auto current_worker_id() -> unsigned { return current_worker->identity_; } -}; - -class GroupReadyQueue { -private: - std::vector queue_{}; - std::atomic size_{0}; - Semaphore sem_{0}; - std::ptrdiff_t waiting_workers_{0}; - const unsigned int num_workers_; - -public: - explicit GroupReadyQueue(unsigned num_workers) - : num_workers_(num_workers) {} - - /** - * Retrieve a ready reaction from the queue. - * - * This method may be called concurrently. In case the queue is empty, the - * method blocks and waits until a ready reaction becomes available. - */ - auto pop() -> Reaction*; - - /** - * Fill the queue up with ready reactions. - * - * This method assumes that the internal queue is empty. It moves all - * reactions from the provided `ready_reactions` vector to the internal - * queue, leaving `ready_reactions` empty. - * - * Note that this method is not thread-safe. The caller needs to ensure that - * no other thread will try to read from the queue during this operation. - */ - void fill_up(std::vector& ready_reactions); -}; - -using EventMap = std::map>; - -class GroupScheduler { // NOLINT -private: - const bool using_workers_; - LogicalTime logical_time_{}; - - Environment* environment_; - std::vector workers_{}; - - std::mutex scheduling_mutex_; - std::unique_lock scheduling_lock_{scheduling_mutex_, std::defer_lock}; - std::condition_variable cv_schedule_; - - std::mutex lock_event_queue_; - std::map event_queue_; - - std::vector> set_ports_; - std::vector> triggered_reactions_; - - std::vector> reaction_queue_; - unsigned int reaction_queue_pos_{std::numeric_limits::max()}; - - GroupReadyQueue ready_queue_; - std::atomic reactions_to_process_{0}; // NOLINT - - std::atomic stop_{false}; - bool continue_execution_{true}; - - void schedule() noexcept; - auto schedule_ready_reactions() -> bool; - void next(); - void terminate_all_workers(); - void set_port_helper(BasePort* port); - -public: - explicit GroupScheduler(Environment* env); - ~GroupScheduler(); - - void schedule_sync(const Tag& tag, BaseAction* action, std::function pre_handler); - void schedule_async(const Tag& tag, BaseAction* action, std::function pre_handler); - - void inline lock() noexcept { scheduling_lock_.lock(); } - void inline unlock() noexcept { scheduling_lock_.unlock(); } - - void set_port(BasePort* port); - - [[nodiscard]] inline auto logical_time() const noexcept -> const auto& { return logical_time_; } - - void start(); - void stop(); - - friend GroupWorker; -}; - -} // namespace reactor - -#endif // REACTOR_CPP_GROUP_SCHEDULER_HH diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index 36365f22..95f25892 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -3,7 +3,6 @@ set(SOURCE_FILES assert.cc dependency_graph.cc environment.cc - group_scheduler.cc logical_time.cc port.cc reaction.cc diff --git a/lib/group_scheduler.cc b/lib/group_scheduler.cc deleted file mode 100644 index 0ef2a9fd..00000000 --- a/lib/group_scheduler.cc +++ /dev/null @@ -1,398 +0,0 @@ -/* - * Copyright (C) 2019 TU Dresden - * All rights reserved. - * - * Authors: - * Christian Menard - */ - -#include - -#include "reactor-cpp/group_scheduler.hh" - -#include "reactor-cpp/action.hh" -#include "reactor-cpp/assert.hh" -#include "reactor-cpp/environment.hh" -#include "reactor-cpp/logging.hh" -#include "reactor-cpp/port.hh" -#include "reactor-cpp/reaction.hh" -#include "reactor-cpp/trace.hh" - -namespace reactor { - -// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) -thread_local const GroupWorker* GroupWorker::current_worker = nullptr; - -GroupWorker::GroupWorker(GroupWorker&& work) // NOLINT(performance-noexcept-move-constructor) - : scheduler_{work.scheduler_} - , identity_{work.identity_} { - // Need to provide the move constructor in order to organize workers in a - // std::vector. However, moving is not save if the thread is already running, - // thus we throw an exception here if the worker is moved but the - // internal thread is already running. - - if (work.thread_.joinable()) { - throw std::runtime_error{"Running workers cannot be moved!"}; - } -} - -void GroupWorker::work() const { - // initialize the current worker thread local variable - current_worker = this; - - log::Debug() << "(GroupWorker " << this->identity_ << ") Starting"; - - if (identity_ == 0) { - log::Debug() << "(GroupWorker 0) do the initial scheduling"; - scheduler_.schedule(); - } - - while (true) { - // wait for a ready reaction - auto* reaction = scheduler_.ready_queue_.pop(); - - // receiving a nullptr indicates that the worker should terminate - if (reaction == nullptr) { - break; - } - - // execute the reaction - execute_reaction(reaction); - - // was this the very last reaction? - if (scheduler_.reactions_to_process_.fetch_sub(1, std::memory_order_acq_rel) == 1) { - // Yes, then schedule. The atomic decrement above ensures that only one - // thread enters this block. - scheduler_.schedule(); - } - // continue otherwise - } - - log::Debug() << "(GroupWorker " << identity_ << ") terminates"; -} - -void GroupWorker::execute_reaction(Reaction* reaction) const { - log::Debug() << "(GroupWorker " << identity_ << ") " - << "execute reaction " << reaction->fqn(); - - tracepoint(reactor_cpp, reaction_execution_starts, id, reaction->fqn(), scheduler.logical_time()); - reaction->trigger(); - tracepoint(reactor_cpp, reaction_execution_finishes, id, reaction->fqn(), scheduler.logical_time()); -} - -void GroupScheduler::schedule() noexcept { - bool found_ready_reactions = schedule_ready_reactions(); - - while (!found_ready_reactions) { - log::Debug() << "(GroupScheduler) call next()"; - next(); - reaction_queue_pos_ = 0; - - found_ready_reactions = schedule_ready_reactions(); - - if (!continue_execution_ && !found_ready_reactions) { - // let all workers know that they should terminate - terminate_all_workers(); - break; - } - } -} - -auto GroupReadyQueue::pop() -> Reaction* { - auto old_size = size_.fetch_sub(1, std::memory_order_acq_rel); - - // If there is no ready reaction available, wait until there is one. - while (old_size <= 0) { - log::Debug() << "(GroupWorker " << GroupWorker::current_worker_id() << ") Wait for work"; - sem_.acquire(); - log::Debug() << "(GroupWorker " << GroupWorker::current_worker_id() << ") Waking up"; - old_size = size_.fetch_sub(1, std::memory_order_acq_rel); - // FIXME: Protect against underflow? - } - - auto pos = old_size - 1; - return queue_[pos]; -} - -void GroupReadyQueue::fill_up(std::vector& ready_reactions) { - // clear the internal queue and swap contents - queue_.clear(); - queue_.swap(ready_reactions); - - // update the atomic size counter and release the semaphore to wake up - // waiting worker threads - auto new_size = static_cast(queue_.size()); - auto old_size = size_.exchange(new_size, std::memory_order_acq_rel); - - // calculate how many workers to wake up. -old_size indicates the number of - // workers who started waiting since the last update. - // We want to wake up at most all the waiting workers. If we would release - // more, other workers that are out of work would not block when acquiring - // the semaphore. - // Also, we do not want to wake up more workers than there is work. new_size - // indicates the number of ready reactions. Since there is always at least - // one worker running running, new_size - running_workers indicates the - // number of additional workers needed to process all reactions. - waiting_workers_ += -old_size; - auto running_workers = num_workers_ - waiting_workers_; - auto workers_to_wakeup = std::min(waiting_workers_, new_size - running_workers); - - // wakeup other workers_ - if (workers_to_wakeup > 0) { - waiting_workers_ -= workers_to_wakeup; - log::Debug() << "Wakeup " << workers_to_wakeup << " workers"; - sem_.release(static_cast(workers_to_wakeup)); - } -} - -void GroupScheduler::terminate_all_workers() { - log::Debug() << "(GroupScheduler) Send termination signal to all workers"; - auto num_workers = environment_->num_workers(); - std::vector null_reactions{num_workers, nullptr}; - log::Debug() << null_reactions.size(); - ready_queue_.fill_up(null_reactions); -} - -auto GroupScheduler::schedule_ready_reactions() -> bool { - // insert any triggered reactions_ into the reaction queue - for (auto& vec_reaction : triggered_reactions_) { - for (auto* reaction : vec_reaction) { - reaction_queue_[reaction->index()].push_back(reaction); - } - vec_reaction.clear(); - } - - log::Debug() << "(GroupScheduler) Scanning the reaction queue for ready reactions"; - - // continue iterating over the reaction queue - for (; reaction_queue_pos_ < reaction_queue_.size(); reaction_queue_pos_++) { - auto& reactions = reaction_queue_[reaction_queue_pos_]; - - // any ready reactions of current priority? - if (!reactions.empty()) { - log::Debug() << "(GroupScheduler) Process reactions of priority " << reaction_queue_pos_; - - // Make sure that any reaction is only executed once even if it - // was triggered multiple times. - std::sort(reactions.begin(), reactions.end()); - reactions.erase(std::unique(reactions.begin(), reactions.end()), reactions.end()); - - if constexpr (log::debug_enabled || tracing_enabled) { // NOLINT - for (auto* reaction : reactions) { - log::Debug() << "(GroupScheduler) Reaction " << reaction->fqn() << " is ready for execution"; - tracepoint(reactor_cpp, trigger_reaction, reaction->container()->fqn(), reaction->name(), logical_time_); - } - } - - reactions_to_process_.store(static_cast(reactions.size()), std::memory_order_release); - ready_queue_.fill_up(reactions); - - // break out of the loop and return - return true; - } - } - - log::Debug() << "(GroupScheduler) Reached end of reaction queue"; - return false; -} - -void GroupScheduler::start() { - log::Debug() << "Starting the scheduler..."; - - auto num_workers = environment_->num_workers(); - // initialize the reaction queue, set ports vector, and triggered reactions - // vector - reaction_queue_.resize(environment_->max_reaction_index() + 1); - set_ports_.resize(num_workers); - triggered_reactions_.resize(num_workers); - - // Initialize and start the workers. By resizing the workers vector first, - // we make sure that there is sufficient space for all the workers and non of - // them needs to be moved. This is important because a running worker may not - // be moved. - workers_.reserve(num_workers); - for (unsigned i = 0; i < num_workers; i++) { - workers_.emplace_back(*this, i); - workers_.back().start_thread(); - } - - // join all worker threads - for (auto& worker : workers_) { - worker.join_thread(); - } -} - -void GroupScheduler::next() { // NOLINT - static EventMap events{}; - - // clean up before scheduling any new events - if (!events.empty()) { - // cleanup all triggered actions - for (auto& vec_ports : events) { - vec_ports.first->cleanup(); - } - // cleanup all set ports - for (auto& vec_ports : set_ports_) { - for (auto& port : vec_ports) { - port->cleanup(); - } - vec_ports.clear(); - } - events.clear(); - } - - { - std::unique_lock lock{scheduling_mutex_}; - - // shutdown if there are no more events in the queue - if (event_queue_.empty() && !stop_) { - if (environment_->run_forever()) { - // wait for a new asynchronous event - cv_schedule_.wait(lock, [this]() { return !event_queue_.empty() || stop_; }); - } else { - log::Debug() << "No more events in queue_. -> Terminate!"; - environment_->sync_shutdown(); - } - } - - while (events.empty()) { - if (stop_) { - continue_execution_ = false; - log::Debug() << "Shutting down the scheduler"; - Tag t_next = Tag::from_logical_time(logical_time_).delay(); - if (t_next == event_queue_.begin()->first) { - log::Debug() << "Schedule the last round of reactions including all " - "termination reactions"; - events = std::move(event_queue_.begin()->second); - event_queue_.erase(event_queue_.begin()); - log::Debug() << "advance logical time to tag [" << t_next.time_point() << ", " << t_next.micro_step() << "]"; - logical_time_.advance_to(t_next); - } else { - return; - } - } else { - // collect events of the next tag - auto t_next = event_queue_.begin()->first; - - // synchronize with physical time if not in fast forward mode - if (!environment_->fast_fwd_execution()) { - // keep track of the current physical time in a static variable - static auto physical_time = TimePoint::min(); - - // If physical time is smaller than the next logical time point, - // then update the physical time. This step is small optimization to - // avoid calling get_physical_time() in every iteration as this - // would add a significant overhead. - if (physical_time < t_next.time_point()) { - physical_time = get_physical_time(); - } - - // If physical time is still smaller than the next logical time - // point, then wait until the next tag or until a new event is - // inserted asynchronously into the queue - if (physical_time < t_next.time_point()) { - auto status = cv_schedule_.wait_until(lock, t_next.time_point()); - // Start over if the event queue was modified - if (status == std::cv_status::no_timeout) { - continue; - } - // update physical time and continue otherwise - physical_time = t_next.time_point(); - } - } - - // retrieve all events with tag equal to current logical time from the - // queue - events = std::move(event_queue_.begin()->second); - event_queue_.erase(event_queue_.begin()); - - // advance logical time - log::Debug() << "advance logical time to tag [" << t_next.time_point() << ", " << t_next.micro_step() << "]"; - logical_time_.advance_to(t_next); - } - } - } // mutex schedule_ - - // execute all setup functions; this sets the values of the corresponding - // actions - for (auto& vec_reactor : events) { - auto& setup = vec_reactor.second; - if (setup != nullptr) { - setup(); - } - } - - log::Debug() << "events: " << events.size(); - for (auto& vec_reactor : events) { - log::Debug() << "Action " << vec_reactor.first->fqn(); - for (auto* reaction : vec_reactor.first->triggers()) { - // There is no need to acquire the mutex. At this point the scheduler - // should be the only thread accessing the reaction queue as none of the - // workers_ are running - log::Debug() << "insert reaction " << reaction->fqn() << " with index " << reaction->index(); - reaction_queue_[reaction->index()].push_back(reaction); - } - } -} - -GroupScheduler::GroupScheduler(Environment* env) - : using_workers_(env->num_workers() > 1) - , environment_(env) - , ready_queue_(env->num_workers()) {} - -GroupScheduler::~GroupScheduler() = default; - -void GroupScheduler::schedule_sync(const Tag& tag, BaseAction* action, std::function pre_handler) { - reactor_assert(logical_time_ < tag); - // TODO verify that the action is indeed allowed to be scheduled by the - // current reaction - log::Debug() << "Schedule action " << action->fqn() << (action->is_logical() ? " synchronously " : " asynchronously ") - << " with tag [" << tag.time_point() << ", " << tag.micro_step() << "]"; - { - auto unique_lock = - using_workers_ ? std::unique_lock(lock_event_queue_) : std::unique_lock(); - - tracepoint(reactor_cpp, schedule_action, action->container()->fqn(), action->name(), tag); // NOLINT - - // create a new event map or retrieve the existing one - auto emplace_result = event_queue_.try_emplace(tag, EventMap()); - auto& event_map = emplace_result.first->second; - - // insert the new event - event_map[action] = std::move(pre_handler); - } -} - -void GroupScheduler::schedule_async(const Tag& tag, BaseAction* action, std::function pre_handler) { - std::lock_guard lock_guard(scheduling_mutex_); - schedule_sync(tag, action, std::move(pre_handler)); - cv_schedule_.notify_one(); -} - -void GroupScheduler::set_port(BasePort* port) { - log::Debug() << "Set port " << port->fqn(); - - // We do not check here if port is already in the list. This means clean() - // could be called multiple times for a single port. However, calling - // clean() multiple time is not harmful and more efficient then checking if - set_ports_[GroupWorker::current_worker_id()].push_back(port); - - // recursively search for triggered reactions - set_port_helper(port); -} - -void GroupScheduler::set_port_helper(BasePort* port) { - for (auto* reaction : port->triggers()) { - triggered_reactions_[GroupWorker::current_worker_id()].push_back(reaction); - } - for (auto* binding : port->outward_bindings()) { - set_port_helper(binding); - } -} - -void GroupScheduler::stop() { - stop_ = true; - cv_schedule_.notify_one(); -} - -} // namespace reactor From 81c73545b52d409fc53dd1e2a60d8d69f363ec2b Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Tue, 26 Jul 2022 16:58:34 +0200 Subject: [PATCH 34/49] define grouped scheduling policy --- include/reactor-cpp/environment.hh | 4 +- .../reactor-cpp/grouped_scheduling_policy.hh | 39 +++++++++++++++++++ lib/environment.cc | 4 +- 3 files changed, 43 insertions(+), 4 deletions(-) create mode 100644 include/reactor-cpp/grouped_scheduling_policy.hh diff --git a/include/reactor-cpp/environment.hh b/include/reactor-cpp/environment.hh index 4c9a5237..52413cf2 100644 --- a/include/reactor-cpp/environment.hh +++ b/include/reactor-cpp/environment.hh @@ -16,9 +16,9 @@ #include #include -#include "reactor-cpp/default_scheduling_policy.hh" #include "reactor-cpp/dependency_graph.hh" #include "reactor-cpp/fwd.hh" +#include "reactor-cpp/grouped_scheduling_policy.hh" #include "reactor-cpp/reactor.hh" namespace reactor { @@ -43,7 +43,7 @@ private: std::set reactions_{}; std::vector dependencies_{}; - std::unique_ptr> scheduler_; + std::unique_ptr> scheduler_; Phase phase_{Phase::Construction}; TimePoint start_time_{}; diff --git a/include/reactor-cpp/grouped_scheduling_policy.hh b/include/reactor-cpp/grouped_scheduling_policy.hh new file mode 100644 index 00000000..aadace53 --- /dev/null +++ b/include/reactor-cpp/grouped_scheduling_policy.hh @@ -0,0 +1,39 @@ +/* + * Copyright (C) 2022 TU Dresden + * All rights reserved. + * + * Authors: + * Christian Menard + */ + +#ifndef REACTOR_CPP_GROUPED_SCHEDULING_POLICY_HH +#define REACTOR_CPP_GROUPED_SCHEDULING_POLICY_HH + +#include +#include + +#include "reactor-cpp/fwd.hh" +#include "reactor-cpp/reaction.hh" +#include "reactor-cpp/semaphore.hh" + +namespace reactor { + +class GroupedSchedulingPolicy { +private: + Scheduler& scheduler_; + Environment& environment_; + +public: + GroupedSchedulingPolicy(Scheduler& scheduler, Environment& env); + + void init(); + auto create_worker() -> Worker; + void worker_function(const Worker& worker); + + void trigger_reaction_from_next(Reaction* reaction); + void trigger_reaction_from_set_port(Reaction* reaction); +}; + +} // namespace reactor + +#endif // REACTOR_CPP_GROUPED_SCHEDULING_POLICY_HH diff --git a/lib/environment.cc b/lib/environment.cc index 3bc475de..de6f4592 100644 --- a/lib/environment.cc +++ b/lib/environment.cc @@ -14,8 +14,8 @@ #include "reactor-cpp/action.hh" #include "reactor-cpp/assert.hh" -#include "reactor-cpp/default_scheduling_policy.hh" #include "reactor-cpp/dependency_graph.hh" +#include "reactor-cpp/grouped_scheduling_policy.hh" #include "reactor-cpp/logging.hh" #include "reactor-cpp/port.hh" #include "reactor-cpp/reaction.hh" @@ -27,7 +27,7 @@ Environment::Environment(unsigned int num_workers, bool run_forever, bool fast_f : num_workers_(num_workers) , run_forever_(run_forever) , fast_fwd_execution_(fast_fwd_execution) - , scheduler_(std::make_unique>(this)) {} + , scheduler_(std::make_unique>(this)) {} [[nodiscard]] auto Environment::scheduler() const noexcept -> const BaseScheduler& { return *scheduler_; } [[nodiscard]] auto Environment::scheduler() noexcept -> BaseScheduler& { return *scheduler_; } From 7dc6920cc3792bed40ae0ec76053a7aa6ab51065 Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Tue, 26 Jul 2022 17:34:02 +0200 Subject: [PATCH 35/49] add an dummy implementation --- .../reactor-cpp/grouped_scheduling_policy.hh | 1 + lib/CMakeLists.txt | 1 + lib/grouped_scheduling_policy.cc | 32 +++++++++++++++++++ 3 files changed, 34 insertions(+) create mode 100644 lib/grouped_scheduling_policy.cc diff --git a/include/reactor-cpp/grouped_scheduling_policy.hh b/include/reactor-cpp/grouped_scheduling_policy.hh index aadace53..28594c4a 100644 --- a/include/reactor-cpp/grouped_scheduling_policy.hh +++ b/include/reactor-cpp/grouped_scheduling_policy.hh @@ -20,6 +20,7 @@ namespace reactor { class GroupedSchedulingPolicy { private: + std::size_t identity_counter{0}; Scheduler& scheduler_; Environment& environment_; diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index 459cbbd3..eee1907b 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -4,6 +4,7 @@ set(SOURCE_FILES base_scheduler.cc default_scheduling_policy.cc dependency_graph.cc + grouped_scheduling_policy.cc environment.cc logical_time.cc port.cc diff --git a/lib/grouped_scheduling_policy.cc b/lib/grouped_scheduling_policy.cc new file mode 100644 index 00000000..423c73d2 --- /dev/null +++ b/lib/grouped_scheduling_policy.cc @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2022 TU Dresden + * All rights reserved. + * + * Authors: + * Christian Menard + */ + +#include "reactor-cpp/grouped_scheduling_policy.hh" + +#include "reactor-cpp/dependency_graph.hh" +#include "reactor-cpp/logging.hh" +#include "reactor-cpp/scheduler.hh" + +namespace reactor { + +GroupedSchedulingPolicy::GroupedSchedulingPolicy(Scheduler& scheduler, Environment& env) + : scheduler_(scheduler) + , environment_(env) {} + +void GroupedSchedulingPolicy::init() {} + +auto GroupedSchedulingPolicy::create_worker() -> Worker { return {*this, identity_counter++}; } + +void GroupedSchedulingPolicy::worker_function(const Worker& worker) { + reactor::log::Info() << "Hello from worker " << worker.id() << " [" << scheduler_.workers_.size() << "]\n"; +} + +void GroupedSchedulingPolicy::trigger_reaction_from_next([[maybe_unused]] Reaction* reaction) {} +void GroupedSchedulingPolicy::trigger_reaction_from_set_port([[maybe_unused]] Reaction* reaction) {} + +} // namespace reactor From b6d1d711da1c3b9e4e53bf5c982869cd2c12ec05 Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Wed, 27 Jul 2022 13:55:29 +0200 Subject: [PATCH 36/49] compute the grouped graph within the policy --- include/reactor-cpp/environment.hh | 4 ---- lib/environment.cc | 20 -------------------- lib/grouped_scheduling_policy.cc | 10 +++++++++- 3 files changed, 9 insertions(+), 25 deletions(-) diff --git a/include/reactor-cpp/environment.hh b/include/reactor-cpp/environment.hh index 52413cf2..0a7cf3fd 100644 --- a/include/reactor-cpp/environment.hh +++ b/include/reactor-cpp/environment.hh @@ -47,8 +47,6 @@ private: Phase phase_{Phase::Construction}; TimePoint start_time_{}; - GroupedDependencyGraph grouped_graph_{}; - void build_dependency_graph(Reactor* reactor); void calculate_indexes(); @@ -85,8 +83,6 @@ public: [[nodiscard]] auto fast_fwd_execution() const noexcept -> bool { return fast_fwd_execution_; } [[nodiscard]] auto run_forever() const noexcept -> bool { return run_forever_; } [[nodiscard]] auto max_reaction_index() const noexcept -> unsigned int { return max_reaction_index_; } - - [[nodiscard]] auto grouped_graph() const noexcept -> const GroupedDependencyGraph& { return grouped_graph_; } }; } // namespace reactor diff --git a/lib/environment.cc b/lib/environment.cc index de6f4592..04cf2324 100644 --- a/lib/environment.cc +++ b/lib/environment.cc @@ -14,7 +14,6 @@ #include "reactor-cpp/action.hh" #include "reactor-cpp/assert.hh" -#include "reactor-cpp/dependency_graph.hh" #include "reactor-cpp/grouped_scheduling_policy.hh" #include "reactor-cpp/logging.hh" #include "reactor-cpp/port.hh" @@ -62,25 +61,6 @@ void Environment::assemble() { build_dependency_graph(reactor); } calculate_indexes(); - - // Testbed for the new graph structure - ReactionDependencyGraph graph{top_level_reactors_}; - ReactionDependencyGraph reduced_graph = graph.transitive_reduction(); - - graph.export_graphviz("graph.dot"); - reduced_graph.export_graphviz("reduced_graph.dot"); - - GroupedDependencyGraph grouped_graph{reduced_graph}; - grouped_graph.group_reactions_by_container(top_level_reactors_); - grouped_graph.export_graphviz("grouped_graph.dot"); - - GroupedDependencyGraph reduced_grouped_graph = grouped_graph.transitive_reduction(); - reduced_grouped_graph.export_graphviz("reduced_grouped_graph.dot"); - - reduced_grouped_graph.group_chains(); - reduced_grouped_graph.export_graphviz("grouped_chains_graph.dot"); - - this->grouped_graph_ = reduced_grouped_graph; } void Environment::build_dependency_graph(Reactor* reactor) { // NOLINT diff --git a/lib/grouped_scheduling_policy.cc b/lib/grouped_scheduling_policy.cc index 423c73d2..bc418e3b 100644 --- a/lib/grouped_scheduling_policy.cc +++ b/lib/grouped_scheduling_policy.cc @@ -18,7 +18,15 @@ GroupedSchedulingPolicy::GroupedSchedulingPolicy(Scheduler Worker { return {*this, identity_counter++}; } From 430c5a0c082d5e57e0350c3868938bb8bb3111df Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Thu, 28 Jul 2022 16:15:00 +0200 Subject: [PATCH 37/49] create a data structure of managing all reaction groups --- include/reactor-cpp/dependency_graph.hh | 10 ++-- .../reactor-cpp/grouped_scheduling_policy.hh | 12 +++++ lib/grouped_scheduling_policy.cc | 49 +++++++++++++++++++ 3 files changed, 67 insertions(+), 4 deletions(-) diff --git a/include/reactor-cpp/dependency_graph.hh b/include/reactor-cpp/dependency_graph.hh index 88034671..04835d40 100644 --- a/include/reactor-cpp/dependency_graph.hh +++ b/include/reactor-cpp/dependency_graph.hh @@ -59,8 +59,7 @@ public: }; class GroupedDependencyGraph { - -private: +public: struct group_info_t { using kind = boost::vertex_property_tag; }; @@ -70,11 +69,10 @@ private: using ReactionToVertexMap = std::map; using GroupPropertyMap = boost::property_map::type; +private: GroupGraph graph{}; ReactionToVertexMap vertex_map{}; - [[nodiscard]] auto get_group_property_map() -> GroupPropertyMap { return boost::get(group_info_t{}, graph); } - struct ReachabilityVisitor : public boost::default_bfs_visitor { private: GroupGraph::vertex_descriptor to; @@ -130,6 +128,10 @@ public: // TODO: This should be const, but I don't know how to get immutable access to the reaction graph properties... [[nodiscard]] auto transitive_reduction() -> GroupedDependencyGraph; + + [[nodiscard]] auto get_graph() const -> const GroupGraph& { return graph; } + + [[nodiscard]] auto get_group_property_map() -> GroupPropertyMap { return boost::get(group_info_t{}, graph); } }; } // namespace reactor diff --git a/include/reactor-cpp/grouped_scheduling_policy.hh b/include/reactor-cpp/grouped_scheduling_policy.hh index 28594c4a..d6902867 100644 --- a/include/reactor-cpp/grouped_scheduling_policy.hh +++ b/include/reactor-cpp/grouped_scheduling_policy.hh @@ -18,12 +18,24 @@ namespace reactor { +struct ReactionGroup { + std::size_t id{0}; + std::vector reactions{}; + std::vector successors{}; + std::atomic triggered{false}; + std::atomic waiting_for{0}; + std::size_t num_predecessors{0}; +}; + class GroupedSchedulingPolicy { private: std::size_t identity_counter{0}; Scheduler& scheduler_; Environment& environment_; + std::vector> reaction_groups_; + std::vector initial_groups_; + public: GroupedSchedulingPolicy(Scheduler& scheduler, Environment& env); diff --git a/lib/grouped_scheduling_policy.cc b/lib/grouped_scheduling_policy.cc index bc418e3b..92576e7a 100644 --- a/lib/grouped_scheduling_policy.cc +++ b/lib/grouped_scheduling_policy.cc @@ -26,6 +26,55 @@ void GroupedSchedulingPolicy::init() { GroupedDependencyGraph reduced_grouped_graph = grouped_graph.transitive_reduction(); reduced_grouped_graph.group_chains(); + + auto g = reduced_grouped_graph.get_graph(); + + std::map vertex_to_group; + + // create a reaction group for each vertex and keep track of all groups without dependencies in initial_groups_ + for (auto* vertex : boost::make_iterator_range(vertices(g))) { + auto& group = reaction_groups_.emplace_back(std::make_unique()); + vertex_to_group[vertex] = group.get(); + + if (boost::in_degree(vertex, g) == 0) { + initial_groups_.push_back(group.get()); + } + } + + // initialize all reaction groups + std::size_t id_counter{0}; + for (auto* vertex : boost::make_iterator_range(vertices(g))) { + auto* group = vertex_to_group[vertex]; + group->id = id_counter++; + + // copy the list of reactions from the vertex + group->reactions = boost::get(reduced_grouped_graph.get_group_property_map(), vertex); + + // initialize the number of dependencies (in edges) + std::size_t num_predecessors = boost::in_degree(vertex, g); + group->waiting_for.store(num_predecessors, std::memory_order_release); + group->num_predecessors = num_predecessors; + + // set all successors + for (auto edge : boost::make_iterator_range(boost::out_edges(vertex, g))) { + auto* successor = vertex_to_group[boost::target(edge, g)]; + group->successors.emplace_back(successor); + } + } + + log::Debug() << "Identified reaction groups: "; + for (const auto& group : reaction_groups_) { + log::Debug() << "* Group " << group->id << ':'; + log::Debug() << " + reactions:"; + for (const auto* reaction : group->reactions) { + log::Debug() << " - " << reaction->fqn(); + } + log::Debug() << " + successors:"; + for (const auto* successor : group->successors) { + log::Debug() << " - Group " << successor->id; + } + log::Debug() << " + num_predecessors: " << group->num_predecessors; + } } auto GroupedSchedulingPolicy::create_worker() -> Worker { return {*this, identity_counter++}; } From 1acb6c7d292f44c0fc71aa2d735b40c533dc9620 Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Thu, 28 Jul 2022 17:06:59 +0200 Subject: [PATCH 38/49] store group pointer in each reaction --- include/reactor-cpp/dependency_graph.hh | 8 ++--- .../reactor-cpp/grouped_scheduling_policy.hh | 5 ++- include/reactor-cpp/reaction.hh | 9 +++++ lib/dependency_graph.cc | 4 +-- lib/grouped_scheduling_policy.cc | 36 +++++++++++++------ 5 files changed, 42 insertions(+), 20 deletions(-) diff --git a/include/reactor-cpp/dependency_graph.hh b/include/reactor-cpp/dependency_graph.hh index 04835d40..9d3d2c7d 100644 --- a/include/reactor-cpp/dependency_graph.hh +++ b/include/reactor-cpp/dependency_graph.hh @@ -25,10 +25,10 @@ private: struct dependency_info_t { using kind = boost::edge_property_tag; }; - using ReactionProperty = boost::property; + using ReactionProperty = boost::property; using DependencyProperty = boost::property; using ReactionGraph = boost::directed_graph; - using ReactionToVertexMap = std::map; + using ReactionToVertexMap = std::map; using ReactionPropertyMap = boost::property_map::type; using DependencyPropertyMap = boost::property_map::type; @@ -63,10 +63,10 @@ public: struct group_info_t { using kind = boost::vertex_property_tag; }; - using Group = std::vector; + using Group = std::vector; using GroupProperty = boost::property; using GroupGraph = boost::directed_graph; - using ReactionToVertexMap = std::map; + using ReactionToVertexMap = std::map; using GroupPropertyMap = boost::property_map::type; private: diff --git a/include/reactor-cpp/grouped_scheduling_policy.hh b/include/reactor-cpp/grouped_scheduling_policy.hh index d6902867..c62e6cad 100644 --- a/include/reactor-cpp/grouped_scheduling_policy.hh +++ b/include/reactor-cpp/grouped_scheduling_policy.hh @@ -20,7 +20,7 @@ namespace reactor { struct ReactionGroup { std::size_t id{0}; - std::vector reactions{}; + std::vector reactions{}; std::vector successors{}; std::atomic triggered{false}; std::atomic waiting_for{0}; @@ -33,8 +33,7 @@ private: Scheduler& scheduler_; Environment& environment_; - std::vector> reaction_groups_; - std::vector initial_groups_; + std::vector> initial_groups_; public: GroupedSchedulingPolicy(Scheduler& scheduler, Environment& env); diff --git a/include/reactor-cpp/reaction.hh b/include/reactor-cpp/reaction.hh index 022d0d0a..e4694f45 100644 --- a/include/reactor-cpp/reaction.hh +++ b/include/reactor-cpp/reaction.hh @@ -10,6 +10,7 @@ #define REACTOR_CPP_REACTION_HH #include +#include #include #include "reactor.hh" @@ -32,6 +33,8 @@ private: Duration deadline_{Duration::zero()}; std::function deadline_handler_{nullptr}; + std::shared_ptr scheduler_info_; + void set_deadline_impl(Duration deadline, const std::function& handler); public: @@ -71,6 +74,12 @@ public: [[nodiscard]] auto has_deadline() const noexcept -> bool { return deadline_ != Duration::zero(); } [[nodiscard]] auto index() const noexcept -> unsigned int { return index_; } + + template void set_scheduler_info(const std::shared_ptr& ptr) noexcept { scheduler_info_ = ptr; } + template void set_scheduler_info(std::shared_ptr&& ptr) noexcept { scheduler_info_ = ptr; } + template [[nodiscard]] auto get_scheduler_info() const noexcept -> std::shared_ptr { + return std::static_pointer_cast(scheduler_info_); + } }; } // namespace reactor diff --git a/lib/dependency_graph.cc b/lib/dependency_graph.cc index 9b5d425a..aaccf862 100644 --- a/lib/dependency_graph.cc +++ b/lib/dependency_graph.cc @@ -269,7 +269,7 @@ void GroupedDependencyGraph::try_contract_edge(GroupGraph::vertex_descriptor va, va_reactions.insert(va_reactions.end(), vb_reactions.begin(), vb_reactions.end()); // update the vertex mapping - for (const auto* reaction : vb_reactions) { + for (auto* reaction : vb_reactions) { vertex_map[reaction] = va; } @@ -306,7 +306,7 @@ void GroupedDependencyGraph::clear_all_empty_vertices() { vertex_map.clear(); for (GroupGraph::vertex_descriptor vd : boost::make_iterator_range(vertices(graph))) { const auto& reactions = get(get_group_property_map(), vd); - for (const auto* reaction : reactions) { + for (auto* reaction : reactions) { vertex_map[reaction] = vd; } } diff --git a/lib/grouped_scheduling_policy.cc b/lib/grouped_scheduling_policy.cc index 92576e7a..3e254484 100644 --- a/lib/grouped_scheduling_policy.cc +++ b/lib/grouped_scheduling_policy.cc @@ -29,27 +29,34 @@ void GroupedSchedulingPolicy::init() { auto g = reduced_grouped_graph.get_graph(); - std::map vertex_to_group; + std::map> vertex_to_group; // create a reaction group for each vertex and keep track of all groups without dependencies in initial_groups_ for (auto* vertex : boost::make_iterator_range(vertices(g))) { - auto& group = reaction_groups_.emplace_back(std::make_unique()); - vertex_to_group[vertex] = group.get(); + auto group = std::make_shared(); + vertex_to_group[vertex] = group; if (boost::in_degree(vertex, g) == 0) { - initial_groups_.push_back(group.get()); + initial_groups_.push_back(group); } } // initialize all reaction groups std::size_t id_counter{0}; for (auto* vertex : boost::make_iterator_range(vertices(g))) { - auto* group = vertex_to_group[vertex]; + auto& group = vertex_to_group[vertex]; group->id = id_counter++; // copy the list of reactions from the vertex group->reactions = boost::get(reduced_grouped_graph.get_group_property_map(), vertex); + // Inform each reaction of its group and also set the index within the group + std::size_t index_counter{0}; + for (auto* reaction : group->reactions) { + reaction->set_scheduler_info(group); + reaction->set_index(index_counter++); + } + // initialize the number of dependencies (in edges) std::size_t num_predecessors = boost::in_degree(vertex, g); group->waiting_for.store(num_predecessors, std::memory_order_release); @@ -57,13 +64,13 @@ void GroupedSchedulingPolicy::init() { // set all successors for (auto edge : boost::make_iterator_range(boost::out_edges(vertex, g))) { - auto* successor = vertex_to_group[boost::target(edge, g)]; - group->successors.emplace_back(successor); + auto& successor = vertex_to_group[boost::target(edge, g)]; + group->successors.emplace_back(successor.get()); } } log::Debug() << "Identified reaction groups: "; - for (const auto& group : reaction_groups_) { + for (const auto& [_, group] : vertex_to_group) { log::Debug() << "* Group " << group->id << ':'; log::Debug() << " + reactions:"; for (const auto* reaction : group->reactions) { @@ -80,10 +87,17 @@ void GroupedSchedulingPolicy::init() { auto GroupedSchedulingPolicy::create_worker() -> Worker { return {*this, identity_counter++}; } void GroupedSchedulingPolicy::worker_function(const Worker& worker) { - reactor::log::Info() << "Hello from worker " << worker.id() << " [" << scheduler_.workers_.size() << "]\n"; + if (worker.id() == 0) { + log::Debug() << "(Worker 0) do the initial scheduling"; + scheduler_.next(); + } } -void GroupedSchedulingPolicy::trigger_reaction_from_next([[maybe_unused]] Reaction* reaction) {} -void GroupedSchedulingPolicy::trigger_reaction_from_set_port([[maybe_unused]] Reaction* reaction) {} +void GroupedSchedulingPolicy::trigger_reaction_from_next([[maybe_unused]] Reaction* reaction) { // NOLINT + log::Warn() << "trigger_reaction_from_next is not yet implemented"; +} +void GroupedSchedulingPolicy::trigger_reaction_from_set_port([[maybe_unused]] Reaction* reaction) { // NOLINT + log::Warn() << "trigger_reaction_from_port is not yet implemented"; +} } // namespace reactor From cc03940909d8823cb5adf5e8a96624de95ce091b Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Thu, 28 Jul 2022 17:43:30 +0200 Subject: [PATCH 39/49] implement triggering of reactions --- .../reactor-cpp/grouped_scheduling_policy.hh | 8 +++++--- lib/grouped_scheduling_policy.cc | 19 +++++++++++-------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/include/reactor-cpp/grouped_scheduling_policy.hh b/include/reactor-cpp/grouped_scheduling_policy.hh index c62e6cad..8ce93571 100644 --- a/include/reactor-cpp/grouped_scheduling_policy.hh +++ b/include/reactor-cpp/grouped_scheduling_policy.hh @@ -20,7 +20,7 @@ namespace reactor { struct ReactionGroup { std::size_t id{0}; - std::vector reactions{}; + std::vector> reactions{}; std::vector successors{}; std::atomic triggered{false}; std::atomic waiting_for{0}; @@ -35,6 +35,8 @@ private: std::vector> initial_groups_; + static void trigger_reaction(Reaction* reaction); + public: GroupedSchedulingPolicy(Scheduler& scheduler, Environment& env); @@ -42,8 +44,8 @@ public: auto create_worker() -> Worker; void worker_function(const Worker& worker); - void trigger_reaction_from_next(Reaction* reaction); - void trigger_reaction_from_set_port(Reaction* reaction); + static inline void trigger_reaction_from_next(Reaction* reaction) { trigger_reaction(reaction); }; + static inline void trigger_reaction_from_set_port(Reaction* reaction) { trigger_reaction(reaction); }; }; } // namespace reactor diff --git a/lib/grouped_scheduling_policy.cc b/lib/grouped_scheduling_policy.cc index 3e254484..e2b678cc 100644 --- a/lib/grouped_scheduling_policy.cc +++ b/lib/grouped_scheduling_policy.cc @@ -48,11 +48,13 @@ void GroupedSchedulingPolicy::init() { group->id = id_counter++; // copy the list of reactions from the vertex - group->reactions = boost::get(reduced_grouped_graph.get_group_property_map(), vertex); + for (auto* reaction : boost::get(reduced_grouped_graph.get_group_property_map(), vertex)) { + group->reactions.emplace_back(std::make_pair(false, reaction)); + } // Inform each reaction of its group and also set the index within the group std::size_t index_counter{0}; - for (auto* reaction : group->reactions) { + for (auto [_, reaction] : group->reactions) { reaction->set_scheduler_info(group); reaction->set_index(index_counter++); } @@ -73,7 +75,7 @@ void GroupedSchedulingPolicy::init() { for (const auto& [_, group] : vertex_to_group) { log::Debug() << "* Group " << group->id << ':'; log::Debug() << " + reactions:"; - for (const auto* reaction : group->reactions) { + for (auto [_, reaction] : group->reactions) { log::Debug() << " - " << reaction->fqn(); } log::Debug() << " + successors:"; @@ -93,11 +95,12 @@ void GroupedSchedulingPolicy::worker_function(const Workerget_scheduler_info(); + log::Debug() << "(GroupedSchedulingPolicy) trigger reaction " << reaction->fqn() << " in Group " << group->id; + auto& triggered_reaction_pair = group->reactions[reaction->index()]; + triggered_reaction_pair.first = true; + group->triggered.store(true, std::memory_order_release); } } // namespace reactor From b3fb04fb5a615d978283d15d14bfe79a643718e3 Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Thu, 28 Jul 2022 18:14:01 +0200 Subject: [PATCH 40/49] implement a simple single threaded scheduling strategy --- .../reactor-cpp/grouped_scheduling_policy.hh | 4 +-- lib/grouped_scheduling_policy.cc | 33 +++++++++++++++++-- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/include/reactor-cpp/grouped_scheduling_policy.hh b/include/reactor-cpp/grouped_scheduling_policy.hh index 8ce93571..71deb50b 100644 --- a/include/reactor-cpp/grouped_scheduling_policy.hh +++ b/include/reactor-cpp/grouped_scheduling_policy.hh @@ -22,7 +22,6 @@ struct ReactionGroup { std::size_t id{0}; std::vector> reactions{}; std::vector successors{}; - std::atomic triggered{false}; std::atomic waiting_for{0}; std::size_t num_predecessors{0}; }; @@ -33,8 +32,9 @@ private: Scheduler& scheduler_; Environment& environment_; - std::vector> initial_groups_; + std::vector initial_groups_; + static void process_group(const Worker& worker, ReactionGroup* group); static void trigger_reaction(Reaction* reaction); public: diff --git a/lib/grouped_scheduling_policy.cc b/lib/grouped_scheduling_policy.cc index e2b678cc..35b335cd 100644 --- a/lib/grouped_scheduling_policy.cc +++ b/lib/grouped_scheduling_policy.cc @@ -37,7 +37,7 @@ void GroupedSchedulingPolicy::init() { vertex_to_group[vertex] = group; if (boost::in_degree(vertex, g) == 0) { - initial_groups_.push_back(group); + initial_groups_.push_back(group.get()); } } @@ -91,7 +91,15 @@ auto GroupedSchedulingPolicy::create_worker() -> Worker void GroupedSchedulingPolicy::worker_function(const Worker& worker) { if (worker.id() == 0) { log::Debug() << "(Worker 0) do the initial scheduling"; - scheduler_.next(); + while (scheduler_.next()) { + for (auto* group : initial_groups_) { + process_group(worker, group); + } + } + // process all shutdown reactions + for (auto* group : initial_groups_) { + process_group(worker, group); + } } } @@ -100,7 +108,26 @@ void GroupedSchedulingPolicy::trigger_reaction(Reaction* reaction) { log::Debug() << "(GroupedSchedulingPolicy) trigger reaction " << reaction->fqn() << " in Group " << group->id; auto& triggered_reaction_pair = group->reactions[reaction->index()]; triggered_reaction_pair.first = true; - group->triggered.store(true, std::memory_order_release); +} + +void GroupedSchedulingPolicy::process_group(const Worker& worker, ReactionGroup* group) { + log::Debug() << "(Worker " << worker.id() << ") process Group " << group->id; + + for (auto& triggered_reaction_pair : group->reactions) { + if (triggered_reaction_pair.first) { + triggered_reaction_pair.first = false; + worker.execute_reaction(triggered_reaction_pair.second); + } + } + + for (auto* successor : group->successors) { + auto old = successor->waiting_for.fetch_sub(1, std::memory_order_acq_rel); + if (old == 1) { + process_group(worker, successor); + } + } + + group->waiting_for.store(group->num_predecessors, std::memory_order_release); } } // namespace reactor From 19a88349daa5e211870e36203dd178194fb348fa Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Fri, 29 Jul 2022 18:02:07 +0200 Subject: [PATCH 41/49] be consistent with types --- include/reactor-cpp/environment.hh | 6 +++--- include/reactor-cpp/semaphore.hh | 4 ++-- lib/environment.cc | 3 ++- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/include/reactor-cpp/environment.hh b/include/reactor-cpp/environment.hh index 0a7cf3fd..28cd2fa2 100644 --- a/include/reactor-cpp/environment.hh +++ b/include/reactor-cpp/environment.hh @@ -34,7 +34,7 @@ public: private: using Dependency = std::pair; - const unsigned int num_workers_{default_number_worker}; + const std::size_t num_workers_{default_number_worker}; unsigned int max_reaction_index_{default_max_reaction_index}; const bool run_forever_{default_run_forever}; const bool fast_fwd_execution_{default_fast_fwd_execution}; @@ -51,7 +51,7 @@ private: void calculate_indexes(); public: - explicit Environment(unsigned int num_workers, bool run_forever = default_run_forever, + explicit Environment(std::size_t num_workers, bool run_forever = default_run_forever, bool fast_fwd_execution = default_fast_fwd_execution); void register_reactor(Reactor* reactor); @@ -79,7 +79,7 @@ public: [[nodiscard]] static auto physical_time() noexcept -> TimePoint { return get_physical_time(); } - [[nodiscard]] auto num_workers() const noexcept -> unsigned int { return num_workers_; } + [[nodiscard]] auto num_workers() const noexcept -> std::size_t { return num_workers_; } [[nodiscard]] auto fast_fwd_execution() const noexcept -> bool { return fast_fwd_execution_; } [[nodiscard]] auto run_forever() const noexcept -> bool { return run_forever_; } [[nodiscard]] auto max_reaction_index() const noexcept -> unsigned int { return max_reaction_index_; } diff --git a/include/reactor-cpp/semaphore.hh b/include/reactor-cpp/semaphore.hh index bc198547..e3a37426 100644 --- a/include/reactor-cpp/semaphore.hh +++ b/include/reactor-cpp/semaphore.hh @@ -17,7 +17,7 @@ namespace reactor { class Semaphore { private: - int count_; + std::size_t count_; std::mutex mutex_{}; std::condition_variable cv_{}; @@ -25,7 +25,7 @@ public: explicit Semaphore(int count) : count_(count) {} - void release(int increment) { + void release(std::size_t increment) { { std::lock_guard lock_guard(mutex_); count_ += increment; diff --git a/lib/environment.cc b/lib/environment.cc index 04cf2324..539f7072 100644 --- a/lib/environment.cc +++ b/lib/environment.cc @@ -9,6 +9,7 @@ #include "reactor-cpp/environment.hh" #include +#include #include #include @@ -22,7 +23,7 @@ namespace reactor { -Environment::Environment(unsigned int num_workers, bool run_forever, bool fast_fwd_execution) +Environment::Environment(std::size_t num_workers, bool run_forever, bool fast_fwd_execution) : num_workers_(num_workers) , run_forever_(run_forever) , fast_fwd_execution_(fast_fwd_execution) From 77da4a86dd5338da5ae033b6fa2f3abeb85fe7b7 Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Fri, 29 Jul 2022 18:05:41 +0200 Subject: [PATCH 42/49] implementm multithreaded group based scheduling --- .../reactor-cpp/grouped_scheduling_policy.hh | 20 +++ lib/grouped_scheduling_policy.cc | 121 +++++++++++++++--- 2 files changed, 123 insertions(+), 18 deletions(-) diff --git a/include/reactor-cpp/grouped_scheduling_policy.hh b/include/reactor-cpp/grouped_scheduling_policy.hh index 71deb50b..77620876 100644 --- a/include/reactor-cpp/grouped_scheduling_policy.hh +++ b/include/reactor-cpp/grouped_scheduling_policy.hh @@ -34,9 +34,29 @@ private: std::vector initial_groups_; + std::size_t num_groups_{0}; + std::atomic groups_to_process_{0}; + std::atomic continue_execution_{true}; + static void process_group(const Worker& worker, ReactionGroup* group); static void trigger_reaction(Reaction* reaction); + class GroupQueue { + // this vector only acts as a dynamically sized array + std::vector queue_{}; + std::atomic read_pos_{0}; + std::atomic write_pos_{0}; + Semaphore semaphore_{0}; + + public: + void init(std::size_t max_size) { queue_.resize(max_size); } + void reset(); + auto pop() -> ReactionGroup*; + auto push(const std::vector& groups); + }; + + GroupQueue group_queue_; + public: GroupedSchedulingPolicy(Scheduler& scheduler, Environment& env); diff --git a/lib/grouped_scheduling_policy.cc b/lib/grouped_scheduling_policy.cc index 35b335cd..02a7d5db 100644 --- a/lib/grouped_scheduling_policy.cc +++ b/lib/grouped_scheduling_policy.cc @@ -8,12 +8,46 @@ #include "reactor-cpp/grouped_scheduling_policy.hh" +#include "reactor-cpp/assert.hh" #include "reactor-cpp/dependency_graph.hh" #include "reactor-cpp/logging.hh" #include "reactor-cpp/scheduler.hh" +#include +#include +#include +#include + namespace reactor { +auto GroupedSchedulingPolicy::GroupQueue::pop() -> ReactionGroup* { + log::Debug() << "(Worker " << Worker::current_worker_id() << ") Wait for work"; + semaphore_.acquire(); + log::Debug() << "(Worker " << Worker::current_worker_id() << ") Waking up"; + + auto pos = read_pos_.fetch_add(1, std::memory_order_relaxed); + std::atomic_thread_fence(std::memory_order_acquire); + reactor_assert(pos < queue_.size()); + return queue_[pos]; +} + +auto GroupedSchedulingPolicy::GroupQueue::push(const std::vector& groups) { + auto pos = write_pos_.fetch_add(groups.size(), std::memory_order_relaxed); + for (auto* group : groups) { + reactor_assert(pos < queue_.size()); + queue_[pos++] = group; + } + std::atomic_thread_fence(std::memory_order_acquire); + semaphore_.release(groups.size()); +} + +void GroupedSchedulingPolicy::GroupQueue::reset() { + std::atomic_thread_fence(std::memory_order_release); + read_pos_.store(0, std::memory_order_relaxed); + write_pos_.store(0, std::memory_order_relaxed); + std::fill(queue_.begin(), queue_.end(), nullptr); +} + GroupedSchedulingPolicy::GroupedSchedulingPolicy(Scheduler& scheduler, Environment& env) : scheduler_(scheduler) , environment_(env) {} @@ -71,6 +105,9 @@ void GroupedSchedulingPolicy::init() { } } + num_groups_ = vertex_to_group.size(); + group_queue_.init(std::max(num_groups_, environment_.num_workers() + 1)); + log::Debug() << "Identified reaction groups: "; for (const auto& [_, group] : vertex_to_group) { log::Debug() << "* Group " << group->id << ':'; @@ -88,17 +125,76 @@ void GroupedSchedulingPolicy::init() { auto GroupedSchedulingPolicy::create_worker() -> Worker { return {*this, identity_counter++}; } -void GroupedSchedulingPolicy::worker_function(const Worker& worker) { +void GroupedSchedulingPolicy::worker_function(const Worker& worker) { // NOLINT if (worker.id() == 0) { log::Debug() << "(Worker 0) do the initial scheduling"; - while (scheduler_.next()) { - for (auto* group : initial_groups_) { - process_group(worker, group); + scheduler_.next(); + groups_to_process_.store(num_groups_, std::memory_order_release); + group_queue_.push(initial_groups_); + } + + // This is used as a list for storing new ready groups while processing a group. + std::vector ready_groups; + ready_groups.reserve(num_groups_); + + // We use this variable to pass a group to ourselves (avoiding the queue) + ReactionGroup* next_group{nullptr}; + + while (true) { + auto* group = next_group != nullptr ? next_group : group_queue_.pop(); + // receiving a nullptr indicates that the worker should terminate + if (group == nullptr) { + break; + } + + // first, process the group + process_group(worker, group); + + // Check if this was the last group. + // If so, we call next() again, otherwise we update all successors and check for new ready groups + bool call_next{1 == groups_to_process_.fetch_sub(1, std::memory_order_acq_rel)}; + if (call_next) { + group_queue_.reset(); + if (continue_execution_.load(std::memory_order_acquire)) { + log::Debug() << "(Worker " << worker.id() << ") call next"; + if (!scheduler_.next()) { + continue_execution_.store(false, std::memory_order_release); + } + std::copy(initial_groups_.begin(), initial_groups_.end(), std::back_inserter(ready_groups)); + groups_to_process_.store(num_groups_, std::memory_order_release); + } else { + std::vector null_groups_(environment_.num_workers() + 1, nullptr); + group_queue_.push(null_groups_); + groups_to_process_.store(environment_.num_workers(), std::memory_order_release); + } + } else { + // Check if any of the successors has become ready + for (auto* successor : group->successors) { + auto old = successor->waiting_for.fetch_sub(1, std::memory_order_relaxed); + if (old == 1) { + ready_groups.emplace_back(successor); + } } } - // process all shutdown reactions - for (auto* group : initial_groups_) { - process_group(worker, group); + + // reset the waiting_for counter of the current group + std::atomic_thread_fence(std::memory_order_acq_rel); + group->waiting_for.store(group->num_predecessors, std::memory_order_relaxed); + + if (ready_groups.empty()) { + next_group = nullptr; + } else { + log::Debug() << "(Worker " << worker.id() << ") found " << ready_groups.size() + << " new groups that are ready for execution"; + // keep the first ready group for ourselves to process next + next_group = ready_groups.back(); + ready_groups.pop_back(); + + // all other groups we give to the ready queue + if (!ready_groups.empty()) { + group_queue_.push(ready_groups); + } + ready_groups.clear(); } } } @@ -112,22 +208,11 @@ void GroupedSchedulingPolicy::trigger_reaction(Reaction* reaction) { void GroupedSchedulingPolicy::process_group(const Worker& worker, ReactionGroup* group) { log::Debug() << "(Worker " << worker.id() << ") process Group " << group->id; - for (auto& triggered_reaction_pair : group->reactions) { if (triggered_reaction_pair.first) { triggered_reaction_pair.first = false; worker.execute_reaction(triggered_reaction_pair.second); } } - - for (auto* successor : group->successors) { - auto old = successor->waiting_for.fetch_sub(1, std::memory_order_acq_rel); - if (old == 1) { - process_group(worker, successor); - } - } - - group->waiting_for.store(group->num_predecessors, std::memory_order_release); } - } // namespace reactor From 27abf261dd6b313846eca838df3e55eec5875e42 Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Sat, 30 Jul 2022 16:39:56 +0200 Subject: [PATCH 43/49] clean up code and only process triggered groups --- .../reactor-cpp/grouped_scheduling_policy.hh | 5 + lib/grouped_scheduling_policy.cc | 119 +++++++++++------- 2 files changed, 79 insertions(+), 45 deletions(-) diff --git a/include/reactor-cpp/grouped_scheduling_policy.hh b/include/reactor-cpp/grouped_scheduling_policy.hh index 77620876..13ccf214 100644 --- a/include/reactor-cpp/grouped_scheduling_policy.hh +++ b/include/reactor-cpp/grouped_scheduling_policy.hh @@ -23,6 +23,7 @@ struct ReactionGroup { std::vector> reactions{}; std::vector successors{}; std::atomic waiting_for{0}; + std::atomic triggered{false}; std::size_t num_predecessors{0}; }; @@ -57,6 +58,10 @@ private: GroupQueue group_queue_; + void schedule(); + auto finalize_group_and_notify_successors(ReactionGroup* group, std::vector& out_ready_groups) -> bool; + void notify_groups(const std::vector& groups, std::vector& out_ready_groups); + public: GroupedSchedulingPolicy(Scheduler& scheduler, Environment& env); diff --git a/lib/grouped_scheduling_policy.cc b/lib/grouped_scheduling_policy.cc index 02a7d5db..3c2f9178 100644 --- a/lib/grouped_scheduling_policy.cc +++ b/lib/grouped_scheduling_policy.cc @@ -125,75 +125,103 @@ void GroupedSchedulingPolicy::init() { auto GroupedSchedulingPolicy::create_worker() -> Worker { return {*this, identity_counter++}; } -void GroupedSchedulingPolicy::worker_function(const Worker& worker) { // NOLINT - if (worker.id() == 0) { - log::Debug() << "(Worker 0) do the initial scheduling"; - scheduler_.next(); - groups_to_process_.store(num_groups_, std::memory_order_release); - group_queue_.push(initial_groups_); +void GroupedSchedulingPolicy::schedule() { + group_queue_.reset(); + if (continue_execution_.load(std::memory_order_acquire)) { + log::Debug() << "(Worker " << Worker::current_worker_id() << ") call next"; + bool continue_execution = scheduler_.next(); + std::atomic_thread_fence(std::memory_order_release); + if (!continue_execution) { + continue_execution_.store(false, std::memory_order_relaxed); + } + groups_to_process_.store(num_groups_, std::memory_order_relaxed); + } else { + log::Debug() << "(Worker " << Worker::current_worker_id() + << ") signal all workers to terminate"; + std::vector null_groups_(environment_.num_workers() + 1, nullptr); + group_queue_.push(null_groups_); + groups_to_process_.store(environment_.num_workers(), std::memory_order_release); } +} + +auto GroupedSchedulingPolicy::finalize_group_and_notify_successors(ReactionGroup* group, + std::vector& out_ready_groups) + -> bool { + group->waiting_for.store(group->num_predecessors, std::memory_order_release); + notify_groups(group->successors, out_ready_groups); + + // return true if the group was the last to be processed. + return 1 == groups_to_process_.fetch_sub(1, std::memory_order_acq_rel); +} - // This is used as a list for storing new ready groups while processing a group. +void GroupedSchedulingPolicy::notify_groups(const std::vector& groups, + std::vector& out_ready_groups) { + for (auto* group : groups) { + // decrement the waiting for counter + auto old = group->waiting_for.fetch_sub(1, std::memory_order_relaxed); + // If the old value was 1 (or 0), then all dependencies are fulfilled and the group is ready for execution + if (old <= 1) { + // If the group was triggered, then add it to the ready queue. Otherwise, we skip the group and check its + // successors. + if (group->triggered.exchange(false, std::memory_order_relaxed)) { + out_ready_groups.emplace_back(group); + } else { + finalize_group_and_notify_successors(group, out_ready_groups); + } + } + } + std::atomic_thread_fence(std::memory_order_acquire); +} + +void GroupedSchedulingPolicy::worker_function(const Worker& worker) { + // This is used as a list for storing new ready groups found while processing a group. std::vector ready_groups; ready_groups.reserve(num_groups_); - // We use this variable to pass a group to ourselves (avoiding the queue) + // We use this variable to pass a group to process to ourselves (avoiding the queue) ReactionGroup* next_group{nullptr}; + // Worker 0 does the initial scheduling + if (worker.id() == 0) { + log::Debug() << "(Worker 0) do the initial scheduling"; + schedule(); + notify_groups(initial_groups_, ready_groups); + group_queue_.push(ready_groups); + ready_groups.clear(); + } + while (true) { + // Get a group to process. If we set next_group in the last iteration, we + // process this group. Otherwise we pop a group from the queue. auto* group = next_group != nullptr ? next_group : group_queue_.pop(); + next_group = nullptr; + // receiving a nullptr indicates that the worker should terminate if (group == nullptr) { break; } - // first, process the group + // process the group process_group(worker, group); + bool need_to_schedule = finalize_group_and_notify_successors(group, ready_groups); - // Check if this was the last group. - // If so, we call next() again, otherwise we update all successors and check for new ready groups - bool call_next{1 == groups_to_process_.fetch_sub(1, std::memory_order_acq_rel)}; - if (call_next) { - group_queue_.reset(); - if (continue_execution_.load(std::memory_order_acquire)) { - log::Debug() << "(Worker " << worker.id() << ") call next"; - if (!scheduler_.next()) { - continue_execution_.store(false, std::memory_order_release); - } - std::copy(initial_groups_.begin(), initial_groups_.end(), std::back_inserter(ready_groups)); - groups_to_process_.store(num_groups_, std::memory_order_release); - } else { - std::vector null_groups_(environment_.num_workers() + 1, nullptr); - group_queue_.push(null_groups_); - groups_to_process_.store(environment_.num_workers(), std::memory_order_release); - } - } else { - // Check if any of the successors has become ready - for (auto* successor : group->successors) { - auto old = successor->waiting_for.fetch_sub(1, std::memory_order_relaxed); - if (old == 1) { - ready_groups.emplace_back(successor); - } - } + if (need_to_schedule) { + schedule(); + notify_groups(initial_groups_, ready_groups); } - // reset the waiting_for counter of the current group - std::atomic_thread_fence(std::memory_order_acq_rel); - group->waiting_for.store(group->num_predecessors, std::memory_order_relaxed); - - if (ready_groups.empty()) { - next_group = nullptr; - } else { - log::Debug() << "(Worker " << worker.id() << ") found " << ready_groups.size() - << " new groups that are ready for execution"; - // keep the first ready group for ourselves to process next + log::Debug() << "(Worker " << worker.id() << ") found " << ready_groups.size() + << " new groups that are ready for execution"; + if (!ready_groups.empty()) { + // take one group for ourselves next_group = ready_groups.back(); ready_groups.pop_back(); - // all other groups we give to the ready queue + // if there are more, we put them on the queue if (!ready_groups.empty()) { group_queue_.push(ready_groups); } + ready_groups.clear(); } } @@ -204,6 +232,7 @@ void GroupedSchedulingPolicy::trigger_reaction(Reaction* reaction) { log::Debug() << "(GroupedSchedulingPolicy) trigger reaction " << reaction->fqn() << " in Group " << group->id; auto& triggered_reaction_pair = group->reactions[reaction->index()]; triggered_reaction_pair.first = true; + group->triggered.store(true, std::memory_order_release); } void GroupedSchedulingPolicy::process_group(const Worker& worker, ReactionGroup* group) { From f85a3b8272d6bbf1e02b7ea35356735820efb0ff Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Sat, 30 Jul 2022 17:38:35 +0200 Subject: [PATCH 44/49] fix bug in termination procedure --- .../reactor-cpp/grouped_scheduling_policy.hh | 1 + lib/grouped_scheduling_policy.cc | 41 +++++++++++-------- 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/include/reactor-cpp/grouped_scheduling_policy.hh b/include/reactor-cpp/grouped_scheduling_policy.hh index 13ccf214..71b57494 100644 --- a/include/reactor-cpp/grouped_scheduling_policy.hh +++ b/include/reactor-cpp/grouped_scheduling_policy.hh @@ -61,6 +61,7 @@ private: void schedule(); auto finalize_group_and_notify_successors(ReactionGroup* group, std::vector& out_ready_groups) -> bool; void notify_groups(const std::vector& groups, std::vector& out_ready_groups); + void terminate_workers(); public: GroupedSchedulingPolicy(Scheduler& scheduler, Environment& env); diff --git a/lib/grouped_scheduling_policy.cc b/lib/grouped_scheduling_policy.cc index 3c2f9178..bb173794 100644 --- a/lib/grouped_scheduling_policy.cc +++ b/lib/grouped_scheduling_policy.cc @@ -127,21 +127,13 @@ auto GroupedSchedulingPolicy::create_worker() -> Worker void GroupedSchedulingPolicy::schedule() { group_queue_.reset(); - if (continue_execution_.load(std::memory_order_acquire)) { - log::Debug() << "(Worker " << Worker::current_worker_id() << ") call next"; - bool continue_execution = scheduler_.next(); - std::atomic_thread_fence(std::memory_order_release); - if (!continue_execution) { - continue_execution_.store(false, std::memory_order_relaxed); - } - groups_to_process_.store(num_groups_, std::memory_order_relaxed); - } else { - log::Debug() << "(Worker " << Worker::current_worker_id() - << ") signal all workers to terminate"; - std::vector null_groups_(environment_.num_workers() + 1, nullptr); - group_queue_.push(null_groups_); - groups_to_process_.store(environment_.num_workers(), std::memory_order_release); + log::Debug() << "(Worker " << Worker::current_worker_id() << ") call next"; + bool continue_execution = scheduler_.next(); + std::atomic_thread_fence(std::memory_order_release); + if (!continue_execution) { + continue_execution_.store(false, std::memory_order_relaxed); } + groups_to_process_.store(num_groups_, std::memory_order_relaxed); } auto GroupedSchedulingPolicy::finalize_group_and_notify_successors(ReactionGroup* group, @@ -173,6 +165,14 @@ void GroupedSchedulingPolicy::notify_groups(const std::vector& g std::atomic_thread_fence(std::memory_order_acquire); } +void GroupedSchedulingPolicy::terminate_workers() { + log::Debug() << "(Worker " << Worker::current_worker_id() + << ") signal all workers to terminate"; + std::vector null_groups_(environment_.num_workers() + 1, nullptr); + group_queue_.push(null_groups_); + groups_to_process_.store(environment_.num_workers(), std::memory_order_release); +} + void GroupedSchedulingPolicy::worker_function(const Worker& worker) { // This is used as a list for storing new ready groups found while processing a group. std::vector ready_groups; @@ -206,8 +206,17 @@ void GroupedSchedulingPolicy::worker_function(const Worker Date: Sun, 31 Jul 2022 15:46:46 +0200 Subject: [PATCH 45/49] summarize similar groups in "super groups" for less overhead --- .../reactor-cpp/grouped_scheduling_policy.hh | 9 +- lib/grouped_scheduling_policy.cc | 119 ++++++++++++++++-- 2 files changed, 120 insertions(+), 8 deletions(-) diff --git a/include/reactor-cpp/grouped_scheduling_policy.hh b/include/reactor-cpp/grouped_scheduling_policy.hh index 71b57494..2d982fde 100644 --- a/include/reactor-cpp/grouped_scheduling_policy.hh +++ b/include/reactor-cpp/grouped_scheduling_policy.hh @@ -25,6 +25,11 @@ struct ReactionGroup { std::atomic waiting_for{0}; std::atomic triggered{false}; std::size_t num_predecessors{0}; + + std::shared_ptr super_group{nullptr}; + std::vector sub_groups{}; + std::vector triggered_sub_groups{}; + std::atomic triggered_sub_groups_write_pos{0}; }; class GroupedSchedulingPolicy { @@ -59,8 +64,10 @@ private: GroupQueue group_queue_; void schedule(); - auto finalize_group_and_notify_successors(ReactionGroup* group, std::vector& out_ready_groups) -> bool; + auto finalize_group_and_notify_successors(ReactionGroup* group, std::vector& out_ready_groups) + -> bool; void notify_groups(const std::vector& groups, std::vector& out_ready_groups); + void notify_super_group(ReactionGroup* group, std::vector& out_ready_groups); void terminate_workers(); public: diff --git a/lib/grouped_scheduling_policy.cc b/lib/grouped_scheduling_policy.cc index bb173794..f8487f43 100644 --- a/lib/grouped_scheduling_policy.cc +++ b/lib/grouped_scheduling_policy.cc @@ -11,10 +11,12 @@ #include "reactor-cpp/assert.hh" #include "reactor-cpp/dependency_graph.hh" #include "reactor-cpp/logging.hh" +#include "reactor-cpp/reaction.hh" #include "reactor-cpp/scheduler.hh" #include #include +#include #include #include @@ -52,7 +54,7 @@ GroupedSchedulingPolicy::GroupedSchedulingPolicy(Scheduler> vertex_to_group; + std::vector> all_groups; // create a reaction group for each vertex and keep track of all groups without dependencies in initial_groups_ for (auto* vertex : boost::make_iterator_range(vertices(g))) { auto group = std::make_shared(); vertex_to_group[vertex] = group; + all_groups.emplace_back(group); if (boost::in_degree(vertex, g) == 0) { initial_groups_.push_back(group.get()); @@ -108,12 +112,69 @@ void GroupedSchedulingPolicy::init() { num_groups_ = vertex_to_group.size(); group_queue_.init(std::max(num_groups_, environment_.num_workers() + 1)); - log::Debug() << "Identified reaction groups: "; for (const auto& [_, group] : vertex_to_group) { - log::Debug() << "* Group " << group->id << ':'; - log::Debug() << " + reactions:"; - for (auto [_, reaction] : group->reactions) { - log::Debug() << " - " << reaction->fqn(); + auto& successors = group->successors; + std::set in_super_group; + if (successors.size() > 1) { + for (auto* successor_a : successors) { + if (in_super_group.count(successor_a) == 0 && successor_a->num_predecessors <= 1) { + std::vector same_successors; + std::copy_if(successors.begin(), successors.end(), std::back_insert_iterator(same_successors), + [successor_a](ReactionGroup* successor_b) { + return successor_b->num_predecessors <= 1 && + successor_a->successors == successor_b->successors; + }); + if (same_successors.size() > 1) { + auto super_group = std::make_shared(); + all_groups.emplace_back(super_group); + + super_group->id = id_counter++; + super_group->successors = successor_a->successors; + super_group->num_predecessors = successor_a->num_predecessors; + super_group->waiting_for.store(successor_a->num_predecessors, std::memory_order_release); + super_group->triggered_sub_groups.resize(same_successors.size()); + + for (auto* sub_group : same_successors) { + super_group->sub_groups.emplace_back(sub_group); + sub_group->super_group = super_group; + + // remove sub_group from the successor list of our starting group + auto it = std::find(group->successors.begin(), group->successors.end(), sub_group); + reactor_assert(it != group->successors.end()); + group->successors.erase(it); + } + // add the newly creates super_group as a successor to the starting group + group->successors.emplace_back(super_group.get()); + + log::Debug() << "Super Group for Group " << successor_a->id << ':'; + for (auto* elem : same_successors) { + in_super_group.insert(elem); + log::Debug() << " - Group " << elem->id; + } + } + } + } + } + } + + log::Debug() << "Identified reaction groups: "; + for (const auto& group : all_groups) { + if (group->sub_groups.empty()) { + log::Debug() << "* Group " << group->id << ':'; + log::Debug() << " + reactions:"; + for (auto [_, reaction] : group->reactions) { + log::Debug() << " - " << reaction->fqn(); + } + if (group->super_group != nullptr) { + log::Debug() << " + super group: " << group->super_group->id; + } + } else { + log::Debug() << "* Super Group " << group->id << ':'; + reactor_assert(group->reactions.empty()); + log::Debug() << " + sub groups:"; + for (auto* sub_group : group->sub_groups) { + log::Debug() << " - " << sub_group->id; + } } log::Debug() << " + successors:"; for (const auto* successor : group->successors) { @@ -146,17 +207,53 @@ auto GroupedSchedulingPolicy::finalize_group_and_notify_successors(ReactionGroup return 1 == groups_to_process_.fetch_sub(1, std::memory_order_acq_rel); } +void GroupedSchedulingPolicy::notify_super_group(ReactionGroup* group, std::vector& out_ready_groups) { + // the group is a super group with triggered sub groups + // -> extract all the triggered subgroups + std::atomic_thread_fence(std::memory_order_release); + auto num_triggered = group->triggered_sub_groups_write_pos.load(std::memory_order_relaxed); + for (std::size_t i{0}; i < num_triggered; i++) { + out_ready_groups.emplace_back(group->triggered_sub_groups[i]); + group->triggered_sub_groups[i]->triggered.store(false, std::memory_order_relaxed); + } + group->waiting_for.store(group->num_predecessors, std::memory_order_relaxed); + group->triggered_sub_groups_write_pos.store(0, std::memory_order_relaxed); + + // we do not need to process the untriggered subgroups and can directly decrement the counter + const auto num_untriggered = group->sub_groups.size() - num_triggered; + if (num_untriggered > 0) { + groups_to_process_.fetch_sub(num_untriggered, std::memory_order_acq_rel); + } + + // update successor if there is any + if (!group->successors.empty()) { + reactor_assert(group->successors.size() == 1); + reactor_assert(group->successors[0]->num_predecessors == 1); + reactor_assert(group->successors[0]->sub_groups.empty()); + + group->successors[0]->waiting_for.fetch_sub(num_untriggered, std::memory_order_acq_rel); + // if none of the groups was triggered, then we can directly notify the successor + if (num_triggered == 0) { + notify_groups(group->successors, out_ready_groups); + } + } +} + void GroupedSchedulingPolicy::notify_groups(const std::vector& groups, std::vector& out_ready_groups) { for (auto* group : groups) { // decrement the waiting for counter auto old = group->waiting_for.fetch_sub(1, std::memory_order_relaxed); + // If the old value was 1 (or 0), then all dependencies are fulfilled and the group is ready for execution if (old <= 1) { // If the group was triggered, then add it to the ready queue. Otherwise, we skip the group and check its // successors. + log::Debug() << "Group " << group->id << " is ready"; if (group->triggered.exchange(false, std::memory_order_relaxed)) { out_ready_groups.emplace_back(group); + } else if (!group->sub_groups.empty()) { + notify_super_group(group, out_ready_groups); } else { finalize_group_and_notify_successors(group, out_ready_groups); } @@ -241,7 +338,15 @@ void GroupedSchedulingPolicy::trigger_reaction(Reaction* reaction) { log::Debug() << "(GroupedSchedulingPolicy) trigger reaction " << reaction->fqn() << " in Group " << group->id; auto& triggered_reaction_pair = group->reactions[reaction->index()]; triggered_reaction_pair.first = true; - group->triggered.store(true, std::memory_order_release); + std::atomic_thread_fence(std::memory_order_release); + bool old = group->triggered.exchange(true, std::memory_order_relaxed); + // Also notify the super group if there is one and if we did not notify it yet. + if (!old && group->super_group != nullptr) { + log::Debug() << "(GroupedSchedulingPolicy) trigger group " << group->id << " in super group " + << group->super_group->id; + auto pos = group->super_group->triggered_sub_groups_write_pos.fetch_add(1, std::memory_order_relaxed); + group->super_group->triggered_sub_groups[pos] = group.get(); + } } void GroupedSchedulingPolicy::process_group(const Worker& worker, ReactionGroup* group) { From a4c886cce34077a5c1d699c47beda560190ad4d5 Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Sun, 31 Jul 2022 18:12:55 +0200 Subject: [PATCH 46/49] also reset group queue before terminating --- lib/grouped_scheduling_policy.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/grouped_scheduling_policy.cc b/lib/grouped_scheduling_policy.cc index f8487f43..a688e846 100644 --- a/lib/grouped_scheduling_policy.cc +++ b/lib/grouped_scheduling_policy.cc @@ -263,6 +263,7 @@ void GroupedSchedulingPolicy::notify_groups(const std::vector& g } void GroupedSchedulingPolicy::terminate_workers() { + group_queue_.reset(); log::Debug() << "(Worker " << Worker::current_worker_id() << ") signal all workers to terminate"; std::vector null_groups_(environment_.num_workers() + 1, nullptr); From 823645f7839ec6c3425f6e3d6ddf07be4b069c25 Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Sun, 31 Jul 2022 19:56:01 +0200 Subject: [PATCH 47/49] schedule in loop also for the initial scheduling this is necessary as there might be no startup reactions and then we need to move to the next tag --- .../reactor-cpp/grouped_scheduling_policy.hh | 1 + lib/grouped_scheduling_policy.cc | 29 ++++++++++--------- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/include/reactor-cpp/grouped_scheduling_policy.hh b/include/reactor-cpp/grouped_scheduling_policy.hh index 2d982fde..bf608137 100644 --- a/include/reactor-cpp/grouped_scheduling_policy.hh +++ b/include/reactor-cpp/grouped_scheduling_policy.hh @@ -64,6 +64,7 @@ private: GroupQueue group_queue_; void schedule(); + void schedule_until_ready_or_terminate(std::vector& ready_groups); auto finalize_group_and_notify_successors(ReactionGroup* group, std::vector& out_ready_groups) -> bool; void notify_groups(const std::vector& groups, std::vector& out_ready_groups); diff --git a/lib/grouped_scheduling_policy.cc b/lib/grouped_scheduling_policy.cc index a688e846..fa04c420 100644 --- a/lib/grouped_scheduling_policy.cc +++ b/lib/grouped_scheduling_policy.cc @@ -197,6 +197,20 @@ void GroupedSchedulingPolicy::schedule() { groups_to_process_.store(num_groups_, std::memory_order_relaxed); } +void GroupedSchedulingPolicy::schedule_until_ready_or_terminate(std::vector& ready_groups) { + // We use a do-while loop here as we could have scheduled events that do not trigger any reactions. + // In this case, ready_groups will be empty and we can simply call schedule again. + do { + if (continue_execution_.load(std::memory_order_acquire)) { + schedule(); + notify_groups(initial_groups_, ready_groups); + } else { + terminate_workers(); + break; + } + } while (ready_groups.empty()); +} + auto GroupedSchedulingPolicy::finalize_group_and_notify_successors(ReactionGroup* group, std::vector& out_ready_groups) -> bool { @@ -282,8 +296,7 @@ void GroupedSchedulingPolicy::worker_function(const Worker Date: Sun, 31 Jul 2022 20:08:50 +0200 Subject: [PATCH 48/49] support supergroups with multiple successors --- lib/grouped_scheduling_policy.cc | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/grouped_scheduling_policy.cc b/lib/grouped_scheduling_policy.cc index fa04c420..8399726f 100644 --- a/lib/grouped_scheduling_policy.cc +++ b/lib/grouped_scheduling_policy.cc @@ -241,12 +241,14 @@ void GroupedSchedulingPolicy::notify_super_group(ReactionGroup* group, std::vect // update successor if there is any if (!group->successors.empty()) { - reactor_assert(group->successors.size() == 1); - reactor_assert(group->successors[0]->num_predecessors == 1); - reactor_assert(group->successors[0]->sub_groups.empty()); + for (auto* successor : group->successors) { + reactor_assert(successor->num_predecessors == 1); + reactor_assert(successor->sub_groups.empty()); - group->successors[0]->waiting_for.fetch_sub(num_untriggered, std::memory_order_acq_rel); - // if none of the groups was triggered, then we can directly notify the successor + successor->waiting_for.fetch_sub(num_untriggered, std::memory_order_acq_rel); + } + + // if none of the groups was triggered, then we can directly notify the successors if (num_triggered == 0) { notify_groups(group->successors, out_ready_groups); } From 2c1915210b15f034a57422bce027036001fc7872 Mon Sep 17 00:00:00 2001 From: Christian Menard Date: Tue, 2 Aug 2022 15:00:31 +0200 Subject: [PATCH 49/49] bugfix and simplification of notify mechanism for supergroups --- .../reactor-cpp/grouped_scheduling_policy.hh | 3 +- lib/grouped_scheduling_policy.cc | 40 +++++++++---------- 2 files changed, 20 insertions(+), 23 deletions(-) diff --git a/include/reactor-cpp/grouped_scheduling_policy.hh b/include/reactor-cpp/grouped_scheduling_policy.hh index bf608137..7f4a265a 100644 --- a/include/reactor-cpp/grouped_scheduling_policy.hh +++ b/include/reactor-cpp/grouped_scheduling_policy.hh @@ -67,7 +67,8 @@ private: void schedule_until_ready_or_terminate(std::vector& ready_groups); auto finalize_group_and_notify_successors(ReactionGroup* group, std::vector& out_ready_groups) -> bool; - void notify_groups(const std::vector& groups, std::vector& out_ready_groups); + void notify_groups(const std::vector& groups, std::vector& out_ready_groups, + std::size_t num_ready_dependencies); void notify_super_group(ReactionGroup* group, std::vector& out_ready_groups); void terminate_workers(); diff --git a/lib/grouped_scheduling_policy.cc b/lib/grouped_scheduling_policy.cc index 8399726f..950b7c56 100644 --- a/lib/grouped_scheduling_policy.cc +++ b/lib/grouped_scheduling_policy.cc @@ -203,7 +203,7 @@ void GroupedSchedulingPolicy::schedule_until_ready_or_terminate(std::vector& out_ready_groups) -> bool { group->waiting_for.store(group->num_predecessors, std::memory_order_release); - notify_groups(group->successors, out_ready_groups); + notify_groups(group->successors, out_ready_groups, 1); // return true if the group was the last to be processed. return 1 == groups_to_process_.fetch_sub(1, std::memory_order_acq_rel); @@ -236,43 +236,39 @@ void GroupedSchedulingPolicy::notify_super_group(ReactionGroup* group, std::vect // we do not need to process the untriggered subgroups and can directly decrement the counter const auto num_untriggered = group->sub_groups.size() - num_triggered; if (num_untriggered > 0) { - groups_to_process_.fetch_sub(num_untriggered, std::memory_order_acq_rel); + auto old = groups_to_process_.fetch_sub(num_untriggered, std::memory_order_acq_rel); + log::Debug() << "groups to be processed: " << old - num_untriggered << ", old:" << old; } - // update successor if there is any - if (!group->successors.empty()) { - for (auto* successor : group->successors) { - reactor_assert(successor->num_predecessors == 1); - reactor_assert(successor->sub_groups.empty()); - - successor->waiting_for.fetch_sub(num_untriggered, std::memory_order_acq_rel); - } - - // if none of the groups was triggered, then we can directly notify the successors - if (num_triggered == 0) { - notify_groups(group->successors, out_ready_groups); - } - } + // notify all dependencies about untriggered sub groups + notify_groups(group->successors, out_ready_groups, num_untriggered); } void GroupedSchedulingPolicy::notify_groups(const std::vector& groups, - std::vector& out_ready_groups) { + std::vector& out_ready_groups, + std::size_t num_ready_dependencies) { for (auto* group : groups) { // decrement the waiting for counter - auto old = group->waiting_for.fetch_sub(1, std::memory_order_relaxed); + auto old = group->waiting_for.fetch_sub(num_ready_dependencies, std::memory_order_relaxed); - // If the old value was 1 (or 0), then all dependencies are fulfilled and the group is ready for execution - if (old <= 1) { + // If the old value was num_ready_dependencies, then all dependencies are now fulfilled and the group is ready for + // execution + if (old == num_ready_dependencies) { // If the group was triggered, then add it to the ready queue. Otherwise, we skip the group and check its // successors. - log::Debug() << "Group " << group->id << " is ready"; if (group->triggered.exchange(false, std::memory_order_relaxed)) { + log::Debug() << "Group " << group->id << " is ready"; out_ready_groups.emplace_back(group); } else if (!group->sub_groups.empty()) { + log::Debug() << "Super Group " << group->id << " is ready"; notify_super_group(group, out_ready_groups); } else { + log::Debug() << "Skip Group " << group->id << " because it was not triggered"; finalize_group_and_notify_successors(group, out_ready_groups); } + } else { + log::Debug() << "Group " << group->id << " is still waiting for " << old - num_ready_dependencies + << " dependencies"; } } std::atomic_thread_fence(std::memory_order_acquire);