From af7c88227631e0f0f3f7f457bea9684d508cb4ae Mon Sep 17 00:00:00 2001 From: Luc Grosheintz Date: Mon, 4 Nov 2024 14:25:18 +0100 Subject: [PATCH] Initial attempt at implementing VERBATIM. (#1364) --- .../codegen_coreneuron_cpp_visitor.hpp | 1 - src/codegen/codegen_cpp_visitor.cpp | 1 + src/codegen/codegen_cpp_visitor.hpp | 4 + src/codegen/codegen_naming.hpp | 18 ++ src/codegen/codegen_neuron_cpp_visitor.cpp | 203 +++++++++++++++++- src/codegen/codegen_neuron_cpp_visitor.hpp | 41 +++- src/main.cpp | 1 + src/symtab/symbol_table.cpp | 1 - test/usecases/CMakeLists.txt | 3 +- test/usecases/verbatim/globals.mod | 14 ++ test/usecases/verbatim/internal_function.mod | 14 ++ test/usecases/verbatim/locals.mod | 26 +++ test/usecases/verbatim/nothing.mod | 11 + test/usecases/verbatim/pointer_in_double.mod | 36 ++++ test/usecases/verbatim/test_globals.py | 14 ++ .../verbatim/test_internal_function.py | 11 + test/usecases/verbatim/test_locals.py | 13 ++ test/usecases/verbatim/test_nothing.py | 9 + .../verbatim/test_pointer_in_double.py | 23 ++ 19 files changed, 439 insertions(+), 5 deletions(-) create mode 100644 test/usecases/verbatim/globals.mod create mode 100644 test/usecases/verbatim/internal_function.mod create mode 100644 test/usecases/verbatim/locals.mod create mode 100644 test/usecases/verbatim/nothing.mod create mode 100644 test/usecases/verbatim/pointer_in_double.mod create mode 100644 test/usecases/verbatim/test_globals.py create mode 100644 test/usecases/verbatim/test_internal_function.py create mode 100644 test/usecases/verbatim/test_locals.py create mode 100644 test/usecases/verbatim/test_nothing.py create mode 100644 test/usecases/verbatim/test_pointer_in_double.py diff --git a/src/codegen/codegen_coreneuron_cpp_visitor.hpp b/src/codegen/codegen_coreneuron_cpp_visitor.hpp index fefd60df5..c43c2516f 100644 --- a/src/codegen/codegen_coreneuron_cpp_visitor.hpp +++ b/src/codegen/codegen_coreneuron_cpp_visitor.hpp @@ -123,7 +123,6 @@ class CodegenCoreneuronCppVisitor: public CodegenCppVisitor { */ std::string process_verbatim_token(const std::string& token); - /** * Check if variable is qualified as constant * \param name The name of variable diff --git a/src/codegen/codegen_cpp_visitor.cpp b/src/codegen/codegen_cpp_visitor.cpp index 004c7d959..73c32ce78 100644 --- a/src/codegen/codegen_cpp_visitor.cpp +++ b/src/codegen/codegen_cpp_visitor.cpp @@ -13,6 +13,7 @@ #include "ast/all.hpp" #include "codegen/codegen_helper_visitor.hpp" #include "codegen/codegen_utils.hpp" +#include "utils/string_utils.hpp" #include "visitors/defuse_analyze_visitor.hpp" #include "visitors/rename_visitor.hpp" #include "visitors/symtab_visitor.hpp" diff --git a/src/codegen/codegen_cpp_visitor.hpp b/src/codegen/codegen_cpp_visitor.hpp index 5fbe2fe4f..24791b3a8 100644 --- a/src/codegen/codegen_cpp_visitor.hpp +++ b/src/codegen/codegen_cpp_visitor.hpp @@ -169,6 +169,10 @@ struct ShadowUseStatement { std::string rhs; }; +inline std::string get_name(ast::Ast const* sym) { + return sym->get_node_name(); +} + inline std::string get_name(const std::shared_ptr& sym) { return sym->get_name(); } diff --git a/src/codegen/codegen_naming.hpp b/src/codegen/codegen_naming.hpp index 397edc49e..53d63095c 100644 --- a/src/codegen/codegen_naming.hpp +++ b/src/codegen/codegen_naming.hpp @@ -176,9 +176,27 @@ static constexpr char NRN_WATCH_CHECK_METHOD[] = "nrn_watch_check"; /// verbatim name of the variable for nrn thread arguments static constexpr char THREAD_ARGS[] = "_threadargs_"; +/// verbatim name of the variable for nrn thread arguments, sometimes with trailing comma +static constexpr char THREAD_ARGS_COMMA[] = "_threadargscomma_"; + /// verbatim name of the variable for nrn thread arguments in prototype static constexpr char THREAD_ARGS_PROTO[] = "_threadargsproto_"; +/// verbatim name of the variable for nrn thread arguments in prototype and a comma +static constexpr char THREAD_ARGS_PROTO_COMMA[] = "_threadargsprotocomma_"; + +/// variation of `_threadargs_` for "internal" functions. +static constexpr char INTERNAL_THREAD_ARGS[] = "_internalthreadargs_"; + +/// variation of `_threadargs_` for "internal" functions, with comma (maybe). +static constexpr char INTERNAL_THREAD_ARGS_COMMA[] = "_internalthreadargscomma_"; + +/// variation of `_threadargsproto_` for "internal" functions. +static constexpr char INTERNAL_THREAD_ARGS_PROTO[] = "_internalthreadargsproto_"; + +/// variation of `_threadargsproto_` for "internal" functions, possibly with comma. +static constexpr char INTERNAL_THREAD_ARGS_PROTO_COMMA[] = "_internalthreadargsprotocomma_"; + /// prefix for ion variable static constexpr char ION_VARNAME_PREFIX[] = "ion_"; diff --git a/src/codegen/codegen_neuron_cpp_visitor.cpp b/src/codegen/codegen_neuron_cpp_visitor.cpp index 3db35f7e2..1113c7ad6 100644 --- a/src/codegen/codegen_neuron_cpp_visitor.cpp +++ b/src/codegen/codegen_neuron_cpp_visitor.cpp @@ -16,10 +16,12 @@ #include #include "ast/all.hpp" +#include "ast/procedure_block.hpp" #include "codegen/codegen_cpp_visitor.hpp" #include "codegen/codegen_utils.hpp" #include "codegen_naming.hpp" #include "config/config.h" +#include "parser/c11_driver.hpp" #include "solver/solver.hpp" #include "utils/string_utils.hpp" #include "visitors/rename_visitor.hpp" @@ -610,6 +612,21 @@ const CodegenCppVisitor::ParamVector CodegenNeuronCppVisitor::external_method_pa } +CodegenCppVisitor::ParamVector CodegenNeuronCppVisitor::internalthreadargs_parameters() { + return internal_method_parameters(); +} + + +CodegenCppVisitor::ParamVector CodegenNeuronCppVisitor::threadargs_parameters() { + return {{"", "Memb_list*", "", "_ml"}, + {"", "size_t", "", "_iml"}, + {"", "Datum*", "", "_ppvar"}, + {"", "Datum*", "", "_thread"}, + {"", "double*", "", "_globals"}, + {"", "NrnThread*", "", "_nt"}}; +} + + /// TODO: Edit for NEURON std::string CodegenNeuronCppVisitor::nrn_thread_arguments() const { return {}; @@ -632,8 +649,176 @@ CodegenNeuronCppVisitor::function_table_parameters(const ast::FunctionTableBlock } +/** Map of the non-(global/top-local) LOCAL variables. + * + * The map associates the name in the MOD file, e.g. `a` with + * the current name of that LOCAL variable, e.g. `a_r_4`. + * + * auto map = get_nonglobal_local_variable_names(); + * assert map["a"] == "a_r_4"; + * + * The two names can differ, because an early pass makes all + * names unique by renaming local variables. + */ +std::unordered_map get_nonglobal_local_variable_names( + const symtab::SymbolTable& symtab) { + if (symtab.global_scope()) { + return {}; + } + + auto local_variables = symtab.get_variables(NmodlType::local_var); + auto parent_symtab = symtab.get_parent_table(); + if (parent_symtab == nullptr) { + throw std::runtime_error( + "Internal NMODL error: non top-level symbol table doesn't have a parent."); + } + + auto variable_names = get_nonglobal_local_variable_names(*parent_symtab); + + for (const auto& symbol: local_variables) { + auto status = symbol->get_status(); + bool is_renamed = (status & symtab::syminfo::Status::renamed) != + symtab::syminfo::Status::empty; + auto current_name = symbol->get_name(); + auto mod_name = is_renamed ? symbol->get_original_name() : current_name; + + variable_names[mod_name] = current_name; + } + + return variable_names; +} + + +std::vector CodegenNeuronCppVisitor::print_verbatim_setup( + const ast::Verbatim& node, + const std::string& verbatim) { + // Note, the logic for reducing the number of macros printed, aims to + // improve legibility of the generated code by reducing number of lines of + // code. It would be correct to print all macros, because that's + // essentially what NOCMODL does. Therefore, the logic isn't sharp (and + // doesn't have to be). + + std::vector macros_defined; + auto print_macro = [this, &verbatim, ¯os_defined](const std::string& macro_name, + const std::string& macro_value) { + if (verbatim.find(macro_name) != std::string::npos) { + printer->fmt_line("#define {} {}", macro_name, macro_value); + macros_defined.push_back(macro_name); + } + }; + + printer->add_line("// Setup for VERBATIM"); + for (const auto& var: codegen_float_variables) { + auto name = get_name(var); + print_macro(name, get_variable_name(name)); + } + + for (const auto& var: codegen_int_variables) { + auto name = get_name(var); + std::string macro_value = get_variable_name(name); + print_macro(name, macro_value); + if (verbatim.find("_p_" + name) != std::string::npos) { + print_macro("_p_" + name, get_pointer_name(name)); + } + } + + for (const auto& var: codegen_global_variables) { + auto name = get_name(var); + print_macro(name, get_variable_name(name)); + } + + for (const auto& var: codegen_thread_variables) { + auto name = get_name(var); + print_macro(name, get_variable_name(name)); + } + + for (const auto& func: info.functions) { + auto name = get_name(func); + print_macro(name, method_name(name)); + print_macro(fmt::format("_l{}", name), fmt::format("ret_{}", name)); + } + + for (const auto& proc: info.procedures) { + auto name = get_name(proc); + print_macro(name, method_name(name)); + } + + + auto symtab = node.get_parent()->get_symbol_table(); + auto locals = get_nonglobal_local_variable_names(*symtab); + for (const auto& [mod_name, current_name]: locals) { + print_macro(fmt::format("_l{}", mod_name), get_variable_name(current_name)); + } + + print_macro(naming::NTHREAD_T_VARIABLE, "nt->_t"); + print_macro("_nt", "nt"); + print_macro("_tqitem", "tqitem"); + + auto print_args_macro = [this, print_macro](const std::string& macro_basename, + const ParamVector& params) { + print_macro("_" + macro_basename + "_", get_arg_str(params)); + print_macro("_" + macro_basename + "comma_", get_arg_str(params) + ","); + print_macro("_" + macro_basename + "proto_", get_parameter_str(params)); + print_macro("_" + macro_basename + "protocomma_", get_parameter_str(params) + ","); + }; + + print_args_macro("internalthreadargs", internalthreadargs_parameters()); + print_args_macro("threadargs", threadargs_parameters()); + + return macros_defined; +} + + +void CodegenNeuronCppVisitor::print_verbatim_cleanup( + const std::vector& macros_defined) { + for (const auto& macro: macros_defined) { + printer->fmt_line("#undef {}", macro); + } + printer->add_line("// End of cleanup for VERBATIM"); +} + + +std::string CodegenNeuronCppVisitor::process_verbatim_text(const std::string& verbatim) { + parser::CDriver driver; + driver.scan_string(verbatim); + auto tokens = driver.all_tokens(); + std::string result; + for (size_t i = 0; i < tokens.size(); i++) { + auto token = tokens[i]; + + // check if we have function call in the verbatim block where + // function is defined in the same mod file + if (program_symtab->is_method_defined(token) && tokens[i + 1] == "(") { + result += token + "("; + if (tokens[i + 2] == naming::THREAD_ARGS) { + result += naming::INTERNAL_THREAD_ARGS; + } else if (tokens[i + 2] == naming::THREAD_ARGS_COMMA) { + result += naming::INTERNAL_THREAD_ARGS_COMMA; + } else { + result += tokens[i + 2]; + } + + i += 2; + } else { + result += token; + } + } + return result; +} + + void CodegenNeuronCppVisitor::visit_verbatim(const Verbatim& node) { - // Not implemented yet. + const auto& verbatim_code = node.get_statement()->eval(); + auto massaged_verbatim = process_verbatim_text(verbatim_code); + + auto macros_defined = print_verbatim_setup(node, massaged_verbatim); + printer->add_line("// Begin VERBATIM"); + const auto& lines = stringutils::split_string(massaged_verbatim, '\n'); + for (const auto& line: lines) { + printer->add_line(line); + } + printer->add_line("// End VERBATIM"); + print_verbatim_cleanup(macros_defined); } @@ -793,6 +978,20 @@ std::string CodegenNeuronCppVisitor::global_variable_name(const SymbolType& symb } +std::string CodegenNeuronCppVisitor::get_pointer_name(const std::string& name) const { + auto name_comparator = [&name](const auto& sym) { return name == get_name(sym); }; + + auto var = + std::find_if(codegen_int_variables.begin(), codegen_int_variables.end(), name_comparator); + + if (var == codegen_int_variables.end()) { + throw std::runtime_error("Only integer variables have a 'pointer name'."); + } + auto position = position_of_int_var(name); + return fmt::format("_ppvar[{}].literal_value()", position); +} + + std::string CodegenNeuronCppVisitor::get_variable_name(const std::string& name, bool use_instance) const { std::string varname = update_if_ion_variable_name(name); @@ -2488,6 +2687,7 @@ void CodegenNeuronCppVisitor::print_codegen_routines_regular() { print_point_process_function_definitions(); print_setdata_functions(); print_check_table_entrypoint(); + print_top_verbatim_blocks(); print_functors_definitions(); print_global_variables_for_hoc(); print_thread_memory_callbacks(); @@ -2505,6 +2705,7 @@ void CodegenNeuronCppVisitor::print_codegen_routines_nothing() { print_headers_include(); print_namespace_start(); print_function_prototypes(); + print_top_verbatim_blocks(); print_global_variables_for_hoc(); print_function_definitions(); print_mechanism_register(); diff --git a/src/codegen/codegen_neuron_cpp_visitor.hpp b/src/codegen/codegen_neuron_cpp_visitor.hpp index 9325a8b8c..c0aa55f83 100644 --- a/src/codegen/codegen_neuron_cpp_visitor.hpp +++ b/src/codegen/codegen_neuron_cpp_visitor.hpp @@ -24,6 +24,8 @@ #include #include +#include "ast/function_block.hpp" +#include "ast/procedure_block.hpp" #include "codegen/codegen_cpp_visitor.hpp" #include "codegen/codegen_info.hpp" #include "codegen/codegen_naming.hpp" @@ -343,6 +345,14 @@ class CodegenNeuronCppVisitor: public CodegenCppVisitor { const ParamVector external_method_parameters(bool table = false) noexcept override; + /** The parameters for the four macros `_internalthreadargs*_`. */ + ParamVector internalthreadargs_parameters(); + + + /** The parameters for the four macros `_threadargs*_`. */ + ParamVector threadargs_parameters(); + + /** * Arguments for "_threadargs_" macro in neuron implementation */ @@ -358,6 +368,19 @@ class CodegenNeuronCppVisitor: public CodegenCppVisitor { const ast::FunctionTableBlock& /* node */) override; + /** Print compatibility macros required for VERBATIM blocks. + * + * Returns the names of all macros introduced. + */ + std::vector print_verbatim_setup(const ast::Verbatim& node, + const std::string& verbatim); + + + /** Print `#undef`s to erase all compatibility macros. + */ + void print_verbatim_cleanup(const std::vector& macros_defined); + + /** * Arguments for register_mech or point_register_mech function */ @@ -462,7 +485,10 @@ class CodegenNeuronCppVisitor: public CodegenCppVisitor { /** - * Determine variable name in the structure of mechanism properties + * Determine the C++ string to replace variable names with. + * + * Given a variable name such as `ion_cai` or `v`, return the C++ code + * required to get the value. * * \param name Variable name that is being printed * \param use_instance Should the variable be accessed via instance or data array @@ -471,6 +497,18 @@ class CodegenNeuronCppVisitor: public CodegenCppVisitor { */ std::string get_variable_name(const std::string& name, bool use_instance = true) const override; + /** + * Determine the C++ string to replace pointer names with. + * + * Given a variable name such as `_p_ptr` or `_p_rng`, return the C++ code + * required to get a pointer to `ptr` (or `rng`). + * + * \param name Variable name that is being printed + * \return The C++ string representing the variable. + * thread structure + */ + std::string get_pointer_name(const std::string& name) const; + /****************************************************************************************/ /* Main printing routines for code generation */ @@ -802,6 +840,7 @@ class CodegenNeuronCppVisitor: public CodegenCppVisitor { /* Overloaded visitor routines */ /****************************************************************************************/ + std::string process_verbatim_text(const std::string& verbatim); void visit_verbatim(const ast::Verbatim& node) override; void visit_watch_statement(const ast::WatchStatement& node) override; void visit_for_netcon(const ast::ForNetcon& node) override; diff --git a/src/main.cpp b/src/main.cpp index d38c6bef4..9481d2527 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -294,6 +294,7 @@ int run_nmodl(int argc, const char* argv[]) { CLI11_PARSE(app, argc, argv); std::string simulator_name = neuron_code ? "neuron" : "coreneuron"; + verbatim_rename = neuron_code ? false : verbatim_rename; fs::create_directories(output_dir); fs::create_directories(scratch_dir); diff --git a/src/symtab/symbol_table.cpp b/src/symtab/symbol_table.cpp index 06dfab5a5..c313672c0 100644 --- a/src/symtab/symbol_table.cpp +++ b/src/symtab/symbol_table.cpp @@ -116,7 +116,6 @@ std::vector> SymbolTable::get_variables(NmodlType with, return result; } - std::vector> SymbolTable::get_variables_with_status(Status status, bool all) const { std::vector> variables; diff --git a/test/usecases/CMakeLists.txt b/test/usecases/CMakeLists.txt index 8efd4372e..f776c578d 100644 --- a/test/usecases/CMakeLists.txt +++ b/test/usecases/CMakeLists.txt @@ -31,7 +31,8 @@ set(NMODL_USECASE_DIRS steady_state suffix table - useion) + useion + verbatim) foreach(usecase ${NMODL_USECASE_DIRS}) add_test(NAME usecase_${usecase} diff --git a/test/usecases/verbatim/globals.mod b/test/usecases/verbatim/globals.mod new file mode 100644 index 000000000..7d7293217 --- /dev/null +++ b/test/usecases/verbatim/globals.mod @@ -0,0 +1,14 @@ +NEURON { + SUFFIX globals + GLOBAL gbl +} + +ASSIGNED { + gbl +} + +FUNCTION get_gbl() { +VERBATIM +_lget_gbl = gbl; +ENDVERBATIM +} diff --git a/test/usecases/verbatim/internal_function.mod b/test/usecases/verbatim/internal_function.mod new file mode 100644 index 000000000..eff6756c8 --- /dev/null +++ b/test/usecases/verbatim/internal_function.mod @@ -0,0 +1,14 @@ +NEURON { + SUFFIX internal_function + THREADSAFE +} + +FUNCTION f() { +VERBATIM + return g(_threadargs_); +ENDVERBATIM +} + +FUNCTION g() { + g = 42.0 +} diff --git a/test/usecases/verbatim/locals.mod b/test/usecases/verbatim/locals.mod new file mode 100644 index 000000000..c5fd8b29b --- /dev/null +++ b/test/usecases/verbatim/locals.mod @@ -0,0 +1,26 @@ +NEURON { + SUFFIX locals +} + +PARAMETER { + a = -1.0 +} + +FUNCTION get_a() { LOCAL a + a = 32.0 +VERBATIM + _lget_a = _la; +ENDVERBATIM +} + +FUNCTION get_b() { LOCAL a, b + a = -1.0 + b = 32.0 + { LOCAL a + a = 100.0 + b = b + a + VERBATIM + _lget_b = _lb; + ENDVERBATIM + } +} diff --git a/test/usecases/verbatim/nothing.mod b/test/usecases/verbatim/nothing.mod new file mode 100644 index 000000000..e100e9ddb --- /dev/null +++ b/test/usecases/verbatim/nothing.mod @@ -0,0 +1,11 @@ +NEURON { SUFFIX nothing } + +VERBATIM +double foo = 42.0; +ENDVERBATIM + +FUNCTION get_foo() { +VERBATIM + return foo; +ENDVERBATIM +} diff --git a/test/usecases/verbatim/pointer_in_double.mod b/test/usecases/verbatim/pointer_in_double.mod new file mode 100644 index 000000000..033f84077 --- /dev/null +++ b/test/usecases/verbatim/pointer_in_double.mod @@ -0,0 +1,36 @@ +NEURON { + SUFFIX pointer_in_double + THREADSAFE +} + +ASSIGNED { + ptr +} + +VERBATIM +struct SomeDouble { + SomeDouble(double _value) : _value(_value) {} + + double value() const { + return _value; + } + + double _value; +}; + +static std::vector some_doubles; +ENDVERBATIM + +INITIAL { +VERBATIM + some_doubles.reserve(10); + some_doubles.push_back(SomeDouble(double(some_doubles.size()))); + *reinterpret_cast(&ptr) = &some_doubles.back(); +ENDVERBATIM +} + +FUNCTION use_pointer() { +VERBATIM + return (*reinterpret_cast(&ptr))->value(); +ENDVERBATIM +} diff --git a/test/usecases/verbatim/test_globals.py b/test/usecases/verbatim/test_globals.py new file mode 100644 index 000000000..d9f530da9 --- /dev/null +++ b/test/usecases/verbatim/test_globals.py @@ -0,0 +1,14 @@ +from neuron import h, gui + + +def test_globals(): + s = h.Section() + s.insert("globals") + + h.gbl_globals = 654.0 + + assert s(0.5).globals.get_gbl() == h.gbl_globals + + +if __name__ == "__main__": + test_globals() diff --git a/test/usecases/verbatim/test_internal_function.py b/test/usecases/verbatim/test_internal_function.py new file mode 100644 index 000000000..b75e9a555 --- /dev/null +++ b/test/usecases/verbatim/test_internal_function.py @@ -0,0 +1,11 @@ +from neuron import h + + +def test_internal_function(): + s = h.Section() + s.nseg = 4 + + s.insert("internal_function") + + assert s.internal_function.g() == 42.0 + assert s.internal_function.f() == 42.0 diff --git a/test/usecases/verbatim/test_locals.py b/test/usecases/verbatim/test_locals.py new file mode 100644 index 000000000..b0c673052 --- /dev/null +++ b/test/usecases/verbatim/test_locals.py @@ -0,0 +1,13 @@ +from neuron import h, gui + + +def test_locals(): + s = h.Section() + s.insert("locals") + + assert s(0.5).locals.get_a() == 32.0 + assert s(0.5).locals.get_b() == 132.0 + + +if __name__ == "__main__": + test_locals() diff --git a/test/usecases/verbatim/test_nothing.py b/test/usecases/verbatim/test_nothing.py new file mode 100644 index 000000000..207d38cbd --- /dev/null +++ b/test/usecases/verbatim/test_nothing.py @@ -0,0 +1,9 @@ +from neuron import h, gui + + +def test_nothing(): + assert h.get_foo() == 42.0 + + +if __name__ == "__main__": + test_nothing() diff --git a/test/usecases/verbatim/test_pointer_in_double.py b/test/usecases/verbatim/test_pointer_in_double.py new file mode 100644 index 000000000..3bf76f490 --- /dev/null +++ b/test/usecases/verbatim/test_pointer_in_double.py @@ -0,0 +1,23 @@ +from neuron import h, gui + +# Since a double is 64-bits and a pointer is also 32 or 64-bits, we can store +# the bits of a pointer in the memory that was allocated for use as doubles. +# Concretely, we, using VERBATIM, safe pointers in ASSIGNED or RANGE variables. +# +# The pattern is found in the builtin mod file: `apcount.mod`. + + +def test_pointer_in_double(): + s = h.Section() + s.nseg = 3 + s.insert("pointer_in_double") + + h.finitialize() + for i, x in enumerate([0.25, 0.5, 0.75]): + actual = s(x).pointer_in_double.use_pointer() + expected = i + assert actual == expected, f"{actual} != {expected}" + + +if __name__ == "__main__": + test_pointer_in_double()