From 2541b7a18070e10071b5586827659f6695d54755 Mon Sep 17 00:00:00 2001 From: Michal Bali <38988507+michalbali256@users.noreply.github.com> Date: Tue, 8 Jun 2021 13:16:52 +0200 Subject: [PATCH] perf: Dependency files caching (#129) * WIP initial changes * WIP * notify user when individual third party components are being downloaded * get more info about macros * processor file impl takes reference to file_manager to have access to macro dependencies (copy members) * remove unnecessary indication whether a file has been updated * WIP macro caching * move from unordered set * WIP * simple caching working * macro has set of used copy members * when loading from cache, load also dependencies of macro * benchmark for macro cache also * nullptr bugs * first macro cache test * additional macro testing * opsyn test * additional tests * one more test * erase cached macros of particular opencode when it is closed * code smells * ... * code smell * fix a nullptr bug * code smells * implement suggestions * format & code smells * Update parser_library/src/workspaces/macro_cache.h Co-authored-by: slavek-kucera <53339291+slavek-kucera@users.noreply.github.com> * save diagnostics for statements cached in context * reparse diags fixed * WIP * parser error listener refactor Co-authored-by: slavek-kucera <53339291+slavek-kucera@users.noreply.github.com> --- benchmark/benchmark.cpp | 125 ++++-- cmake/external_antlr4cpp.cmake | 1 + cmake/external_boost.cmake | 1 + cmake/external_gtest.cmake | 1 + cmake/external_json.cmake | 1 + cmake/external_uri.cmake | 1 + parser_library/src/analyzer.cpp | 13 +- parser_library/src/analyzer.h | 5 +- parser_library/src/context/copy_member.h | 36 +- parser_library/src/context/hlasm_context.cpp | 33 +- parser_library/src/context/hlasm_context.h | 9 +- parser_library/src/context/id_storage.cpp | 8 +- parser_library/src/context/id_storage.h | 5 + parser_library/src/context/macro.cpp | 4 +- parser_library/src/context/macro.h | 5 +- .../src/context/statement_cache.cpp | 5 +- parser_library/src/context/statement_cache.h | 10 +- parser_library/src/lsp/lsp_context.cpp | 14 + parser_library/src/lsp/lsp_context.h | 2 + parser_library/src/parsing/CMakeLists.txt | 2 +- .../src/parsing/parser_error_listener.cpp | 2 +- .../src/parsing/parser_error_listener_ctx.cpp | 36 -- .../src/parsing/parser_error_listener_ctx.h | 24 +- .../parser_error_listener_substitution.h | 54 +++ parser_library/src/parsing/parser_impl.cpp | 51 ++- parser_library/src/parsing/parser_impl.h | 11 +- .../instruction_sets/asm_processor.cpp | 2 +- .../low_language_processor.cpp | 3 +- .../src/processing/processing_manager.cpp | 7 +- .../statement_analyzers/lsp_analyzer.cpp | 2 +- .../src/processing/statement_fields_parser.h | 3 +- .../macrodef_processing_info.h | 1 + .../macrodef_processor.cpp | 8 +- .../members_statement_provider.cpp | 26 +- .../members_statement_provider.h | 6 +- .../src/semantics/range_provider.cpp | 12 +- parser_library/src/semantics/range_provider.h | 12 +- parser_library/src/workspaces/CMakeLists.txt | 2 + parser_library/src/workspaces/file_manager.h | 2 +- .../src/workspaces/file_manager_impl.cpp | 23 +- .../src/workspaces/file_manager_impl.h | 6 +- parser_library/src/workspaces/macro_cache.cpp | 201 +++++++++ parser_library/src/workspaces/macro_cache.h | 102 +++++ parser_library/src/workspaces/processor.h | 3 +- .../src/workspaces/processor_file_impl.cpp | 67 +-- .../src/workspaces/processor_file_impl.h | 20 +- parser_library/src/workspaces/workspace.cpp | 7 + parser_library/test/context/context_test.cpp | 8 +- parser_library/test/copy_mock.h | 6 + .../test/parsing/parser_model_test.cpp | 34 +- parser_library/test/processing/copy_test.cpp | 43 ++ parser_library/test/workspace/CMakeLists.txt | 2 + .../test/workspace/file_with_text.h | 36 ++ .../test/workspace/macro_cache_test.cpp | 381 ++++++++++++++++++ .../test/workspace/workspace_test.cpp | 50 +-- 55 files changed, 1256 insertions(+), 278 deletions(-) delete mode 100644 parser_library/src/parsing/parser_error_listener_ctx.cpp create mode 100644 parser_library/src/parsing/parser_error_listener_substitution.h create mode 100644 parser_library/src/workspaces/macro_cache.cpp create mode 100644 parser_library/src/workspaces/macro_cache.h create mode 100644 parser_library/test/workspace/file_with_text.h create mode 100644 parser_library/test/workspace/macro_cache_test.cpp diff --git a/benchmark/benchmark.cpp b/benchmark/benchmark.cpp index 3336adcb7..620008dd6 100644 --- a/benchmark/benchmark.cpp +++ b/benchmark/benchmark.cpp @@ -77,6 +77,13 @@ class diagnostic_counter : public hlasm_plugin::parser_library::diagnostics_cons } } + void clear_counters() + { + error_count = 0; + warning_count = 0; + message_counts.clear(); + } + size_t error_count = 0; size_t warning_count = 0; @@ -102,6 +109,7 @@ struct all_file_stats long long whole_time = 0; size_t program_count = 0; size_t parsing_crashes = 0; + size_t reparsing_crashes = 0; size_t failed_file_opens = 0; }; @@ -124,7 +132,8 @@ json parse_one_file(const std::string& source_file, const std::string& ws_folder, all_file_stats& s, bool write_details, - const std::string& message) + const std::string& message, + bool do_reparse) { auto source_path = ws_folder + "/" + source_file; std::ifstream in(source_path); @@ -141,8 +150,8 @@ json parse_one_file(const std::string& source_file, // new workspace manager hlasm_plugin::parser_library::workspace_manager ws; - diagnostic_counter consumer; - ws.register_diagnostics_consumer(&consumer); + diagnostic_counter diag_counter; + ws.register_diagnostics_consumer(&diag_counter); metrics_collector collector; ws.register_performance_metrics_consumer(&collector); // input folder as new workspace @@ -181,33 +190,12 @@ json parse_one_file(const std::string& source_file, s.all_files += collector.metrics_.files; s.whole_time += time; - auto top_messages = get_top_messages(consumer.message_counts); + auto top_messages = get_top_messages(diag_counter.message_counts); - if (write_details) - std::clog << "Time: " << time << " ms" << '\n' - << "Errors: " << consumer.error_count << '\n' - << "Open Code Statements: " << collector.metrics_.open_code_statements << '\n' - << "Copy Statements: " << collector.metrics_.copy_statements << '\n' - << "Macro Statements: " << collector.metrics_.macro_statements << '\n' - << "Copy Def Statements: " << collector.metrics_.copy_def_statements << '\n' - << "Macro Def Statements: " << collector.metrics_.macro_def_statements << '\n' - << "Lookahead Statements: " << collector.metrics_.lookahead_statements << '\n' - << "Reparsed Statements: " << collector.metrics_.reparsed_statements << '\n' - << "Continued Statements: " << collector.metrics_.continued_statements << '\n' - << "Non-continued Statements: " << collector.metrics_.non_continued_statements << '\n' - << "Lines: " << collector.metrics_.lines << '\n' - << "Executed Statement/ms: " << exec_statements / (double)time << '\n' - << "Line/ms: " << collector.metrics_.lines / (double)time << '\n' - << "Files: " << collector.metrics_.files << '\n' - << "Top messages: " << top_messages.dump() << '\n' - << '\n' - << std::endl; - - return json({ - { "File", source_file }, + json result({ { "File", source_file }, { "Success", true }, - { "Errors", consumer.error_count }, - { "Warnings", consumer.warning_count }, + { "Errors", diag_counter.error_count }, + { "Warnings", diag_counter.warning_count }, { "Wall Time (ms)", time }, { "CPU Time (ms/n)", 1000.0 * (c_end - c_start) / CLOCKS_PER_SEC }, { "Open Code Statements", collector.metrics_.open_code_statements }, @@ -224,8 +212,70 @@ json parse_one_file(const std::string& source_file, { "ExecStatement/ms", exec_statements / (double)time }, { "Line/ms", collector.metrics_.lines / (double)time }, { "Files", collector.metrics_.files }, - { "Top messages", std::move(top_messages) }, - }); + { "Top messages", std::move(top_messages) } }); + + auto first_parse_metrics = collector.metrics_; + auto first_diag_counter = diag_counter; + long long reparse_time = 0; + // Reparse to benchmark macro caching + if (do_reparse) + { + diag_counter.clear_counters(); + + auto c_start_reparse = std::clock(); + auto start_reparse = std::chrono::high_resolution_clock::now(); + std::clog << message << "Reparsing file: " << source_file << std::endl; + // open file/parse + try + { + ws.did_change_file(source_path.c_str(), 1, nullptr, 0); + } + catch (const std::exception& e) + { + ++s.reparsing_crashes; + std::clog << message << "Reparse error: " << e.what() << std::endl; + return json({ { "File", source_file }, { "Success", false }, { "Reason", "Crash" }, { "Reparse", true } }); + } + catch (...) + { + ++s.reparsing_crashes; + std::clog << message << "Reparse failed\n\n" << std::endl; + return json({ { "File", source_file }, { "Success", false }, { "Reason", "Crash" }, { "Reparse", true } }); + } + + auto c_end_reparse = std::clock(); + reparse_time = std::chrono::duration_cast( + std::chrono::high_resolution_clock::now() - start_reparse) + .count(); + result["Reparse Wall Time (ms)"] = reparse_time; + result["Reparse CPU Time (ms/n)"] = 1000.0 * (c_end_reparse - c_start_reparse) / CLOCKS_PER_SEC; + result["Reparse errors"] = diag_counter.error_count; + result["Reparse warnings"] = diag_counter.warning_count; + } + + if (write_details) + std::clog << "Time: " << time << " ms" << '\n' + << "Reparse time: " << reparse_time << " ms" << '\n' + << "Errors: " << first_diag_counter.error_count << '\n' + << "Reparse errors: " << diag_counter.error_count << '\n' + << "Open Code Statements: " << first_parse_metrics.open_code_statements << '\n' + << "Copy Statements: " << first_parse_metrics.copy_statements << '\n' + << "Macro Statements: " << first_parse_metrics.macro_statements << '\n' + << "Copy Def Statements: " << first_parse_metrics.copy_def_statements << '\n' + << "Macro Def Statements: " << first_parse_metrics.macro_def_statements << '\n' + << "Lookahead Statements: " << first_parse_metrics.lookahead_statements << '\n' + << "Reparsed Statements: " << first_parse_metrics.reparsed_statements << '\n' + << "Continued Statements: " << first_parse_metrics.continued_statements << '\n' + << "Non-continued Statements: " << first_parse_metrics.non_continued_statements << '\n' + << "Lines: " << first_parse_metrics.lines << '\n' + << "Executed Statement/ms: " << exec_statements / (double)time << '\n' + << "Line/ms: " << first_parse_metrics.lines / (double)time << '\n' + << "Files: " << first_parse_metrics.files << '\n' + << "Top messages: " << top_messages.dump() << '\n' + << '\n' + << std::endl; + + return result; } std::string get_file_message(size_t iter, size_t begin, size_t end, const std::string& base_message) @@ -243,6 +293,7 @@ int main(int argc, char** argv) std::string single_file = ""; size_t start_range = 0, end_range = 0; bool write_details = true; + bool do_reparse = true; std::string message; for (int i = 1; i < argc - 1; i++) { @@ -286,6 +337,9 @@ int main(int argc, char** argv) { write_details = false; } + // When specified, skip reparsing each program to test out macro caching + else if (arg == "-r") + do_reparse = false; // When specified, the scpecified string will be shown at the beginning of each "Parsing " message else if (arg == "-m") { @@ -330,8 +384,12 @@ int main(int argc, char** argv) end_range = std::numeric_limits::max(); for (size_t i = 0; i < end_range; ++i) { - json j = parse_one_file( - single_file, ws_folder, s, write_details, get_file_message(i, start_range, end_range, message)); + json j = parse_one_file(single_file, + ws_folder, + s, + write_details, + get_file_message(i, start_range, end_range, message), + do_reparse); std::cout << j.dump(2); std::cout.flush(); } @@ -357,7 +415,8 @@ int main(int argc, char** argv) ws_folder, s, write_details, - get_file_message(current_iter, start_range, end_range, message)); + get_file_message(current_iter, start_range, end_range, message), + do_reparse); if (not_first) std::cout << ",\n"; diff --git a/cmake/external_antlr4cpp.cmake b/cmake/external_antlr4cpp.cmake index 1c260e451..a39c5fb65 100644 --- a/cmake/external_antlr4cpp.cmake +++ b/cmake/external_antlr4cpp.cmake @@ -62,6 +62,7 @@ function(add_antlr4) endfunction() if(NOT antlr4cpp_POPULATED) + message("Populating antlr4") FetchContent_Populate(antlr4cpp) add_antlr4() diff --git a/cmake/external_boost.cmake b/cmake/external_boost.cmake index 12ed88d52..da110d6e2 100644 --- a/cmake/external_boost.cmake +++ b/cmake/external_boost.cmake @@ -24,6 +24,7 @@ FetchContent_Declare( ) if(NOT boost_ext_POPULATED) + message("Populating ASIO") FetchContent_Populate(boost_ext) endif() diff --git a/cmake/external_gtest.cmake b/cmake/external_gtest.cmake index d554e15b4..262760cc4 100644 --- a/cmake/external_gtest.cmake +++ b/cmake/external_gtest.cmake @@ -32,6 +32,7 @@ function(add_googletest) endfunction() if(NOT googletest_POPULATED) + message("Populating GTest") FetchContent_Populate(googletest) add_googletest() endif() diff --git a/cmake/external_json.cmake b/cmake/external_json.cmake index d2ca45960..2b3019582 100644 --- a/cmake/external_json.cmake +++ b/cmake/external_json.cmake @@ -27,6 +27,7 @@ set(JSON_BuildTests Off) FetchContent_GetProperties(json) if(NOT json_POPULATED) + message("Populating nlohmann json") set(JSON_MultipleHeaders On) FetchContent_Populate(json) add_subdirectory(${json_SOURCE_DIR} ${json_BINARY_DIR} EXCLUDE_FROM_ALL) diff --git a/cmake/external_uri.cmake b/cmake/external_uri.cmake index b04c288b4..2bc5c1e68 100644 --- a/cmake/external_uri.cmake +++ b/cmake/external_uri.cmake @@ -45,6 +45,7 @@ function(add_uri_ext) endfunction() if(NOT uri_ext_POPULATED) + message("Populating netlib uri") FetchContent_Populate(uri_ext) add_uri_ext() endif() diff --git a/parser_library/src/analyzer.cpp b/parser_library/src/analyzer.cpp index c1904dfac..b0c286a76 100644 --- a/parser_library/src/analyzer.cpp +++ b/parser_library/src/analyzer.cpp @@ -44,19 +44,22 @@ analyzer::analyzer(const std::string& text, parser_->addErrorListener(&listener_); } -analyzer::analyzer( - const std::string& text, std::string file_name, parse_lib_provider& lib_provider, bool collect_hl_info) +analyzer::analyzer(const std::string& text, + std::string file_name, + parse_lib_provider& lib_provider, + bool collect_hl_info, + id_storage ids_init) : analyzer(text, file_name, - analyzing_context { - std::make_unique(file_name, lib_provider.get_asm_options(file_name)), + analyzing_context { std::make_unique( + file_name, lib_provider.get_asm_options(file_name), std::move(ids_init)), std::make_unique() }, lib_provider, library_data { processing::processing_kind::ORDINARY, context::id_storage::empty_id }, collect_hl_info) {} -analyzing_context analyzer::context() { return ctx_; } +analyzing_context analyzer::context() const { return ctx_; } context::hlasm_context& analyzer::hlasm_ctx() { return *ctx_.hlasm_ctx; } diff --git a/parser_library/src/analyzer.h b/parser_library/src/analyzer.h index 547c1c9bf..a63408a59 100644 --- a/parser_library/src/analyzer.h +++ b/parser_library/src/analyzer.h @@ -54,9 +54,10 @@ class analyzer : public diagnosable_ctx analyzer(const std::string& text, std::string file_name = "", workspaces::parse_lib_provider& lib_provider = workspaces::empty_parse_lib_provider::instance, - bool collect_hl_info = false); + bool collect_hl_info = false, + context::id_storage ids_init = {}); - analyzing_context context(); + analyzing_context context() const; context::hlasm_context& hlasm_ctx(); parsing::hlasmparser& parser(); const semantics::source_info_processor& source_processor() const; diff --git a/parser_library/src/context/copy_member.h b/parser_library/src/context/copy_member.h index efdbe6c41..c3e99e4f7 100644 --- a/parser_library/src/context/copy_member.h +++ b/parser_library/src/context/copy_member.h @@ -21,22 +21,6 @@ namespace hlasm_plugin::parser_library::context { -// structure represents invocation of COPY member in HLASM macro library -struct copy_member_invocation -{ - const id_index name; - cached_block& cached_definition; - const location& definition_location; - int current_statement; - - copy_member_invocation(const id_index name, cached_block& cached_definition, const location& definition_location) - : name(name) - , cached_definition(cached_definition) - , definition_location(definition_location) - , current_statement(-1) - {} -}; - struct copy_member; using copy_member_ptr = std::shared_ptr; @@ -58,7 +42,25 @@ struct copy_member cached_definition.emplace_back(std::move(stmt)); } - copy_member_invocation enter() { return copy_member_invocation(name, cached_definition, definition_location); } + // copy_member_invocation enter() { return copy_member_invocation(name, cached_definition, definition_location); } +}; + +// structure represents invocation of COPY member in HLASM macro library +struct copy_member_invocation +{ + const id_index name; + cached_block& cached_definition; + const location& definition_location; + const copy_member_ptr copy_member_definition; + int current_statement; + + explicit copy_member_invocation(copy_member_ptr copy_member) + : name(copy_member->name) + , cached_definition(copy_member->cached_definition) + , definition_location(copy_member->definition_location) + , copy_member_definition(std::move(copy_member)) + , current_statement(-1) + {} }; } // namespace hlasm_plugin::parser_library::context diff --git a/parser_library/src/context/hlasm_context.cpp b/parser_library/src/context/hlasm_context.cpp index ebac7d881..8b9e1cabf 100644 --- a/parser_library/src/context/hlasm_context.cpp +++ b/parser_library/src/context/hlasm_context.cpp @@ -251,8 +251,9 @@ bool hlasm_context::is_opcode(id_index symbol) const return macros_.find(symbol) != macros_.end() || instruction_map_.find(symbol) != instruction_map_.end(); } -hlasm_context::hlasm_context(std::string file_name, asm_option asm_options) - : opencode_file_name_(file_name) +hlasm_context::hlasm_context(std::string file_name, asm_option asm_options, id_storage init_ids) + : ids_(std::move(init_ids)) + , opencode_file_name_(file_name) , asm_options_(std::move(asm_options)) , instruction_map_(init_instruction_map()) , SYSNDX_(0) @@ -668,7 +669,8 @@ macro_def_ptr hlasm_context::add_macro(id_index name, statement_block definition, copy_nest_storage copy_nests, label_storage labels, - location definition_location) + location definition_location, + std::unordered_set used_copy_members) { return macros_ .insert_or_assign(name, @@ -678,12 +680,17 @@ macro_def_ptr hlasm_context::add_macro(id_index name, std::move(definition), std::move(copy_nests), std::move(labels), - std::move(definition_location))) + std::move(definition_location), + std::move(used_copy_members))) .first->second; } +void hlasm_context::add_macro(macro_def_ptr macro) { macros_[macro->id] = std::move(macro); }; + const hlasm_context::macro_storage& hlasm_context::macros() const { return macros_; } +const hlasm_context::opcode_map& hlasm_context::opcode_mnemo_storage() const { return opcode_mnemo_; } + macro_def_ptr hlasm_context::get_macro_definition(id_index name) const { macro_def_ptr macro_def; @@ -741,6 +748,20 @@ copy_member_ptr hlasm_context::add_copy_member( return copydef; } +void hlasm_context::add_copy_member(copy_member_ptr member) +{ + visited_files_.insert(member->definition_location.file); + copy_members_[member->name] = std::move(member); +} + + +copy_member_ptr hlasm_context::get_copy_member(id_index member) const +{ + if (auto it = copy_members_.find(member); it != copy_members_.end()) + return it->second; + return nullptr; +} + void hlasm_context::enter_copy_member(id_index member_name) { auto tmp = copy_members_.find(member_name); @@ -749,7 +770,7 @@ void hlasm_context::enter_copy_member(id_index member_name) const auto& [name, member] = *tmp; - source_stack_.back().copy_stack.emplace_back(member->enter()); + source_stack_.back().copy_stack.emplace_back(copy_member_invocation(member)); } const hlasm_context::copy_member_storage& hlasm_context::copy_members() { return copy_members_; } @@ -769,7 +790,7 @@ void hlasm_context::apply_source_snapshot(source_snapshot snapshot) for (auto& frame : snapshot.copy_frames) { - auto invo = copy_members_.at(frame.copy_member)->enter(); + copy_member_invocation invo(copy_members_.at(frame.copy_member)); invo.current_statement = (int)frame.statement_offset; source_stack_.back().copy_stack.push_back(std::move(invo)); } diff --git a/parser_library/src/context/hlasm_context.h b/parser_library/src/context/hlasm_context.h index 7f8ddb2c9..87aed4ed1 100644 --- a/parser_library/src/context/hlasm_context.h +++ b/parser_library/src/context/hlasm_context.h @@ -85,7 +85,7 @@ class hlasm_context bool is_opcode(id_index symbol) const; public: - hlasm_context(std::string file_name = "", asm_option asm_opts = {}); + hlasm_context(std::string file_name = "", asm_option asm_opts = {}, id_storage init_ids = {}); // gets name of file where is open-code located const std::string& opencode_file_name() const; @@ -161,6 +161,7 @@ class hlasm_context void add_mnemonic(id_index mnemo, id_index op_code); // removes opsyn mnemonic void remove_mnemonic(id_index mnemo); + const opcode_map& opcode_mnemo_storage() const; // checks wheter the symbol is an operation code (is a valid instruction or a mnemonic) opcode_t get_operation_code(id_index symbol) const; @@ -188,7 +189,9 @@ class hlasm_context statement_block definition, copy_nest_storage copy_nests, label_storage labels, - location definition_location); + location definition_location, + std::unordered_set used_copy_members); + void add_macro(macro_def_ptr macro); // enters a macro with actual params macro_invo_ptr enter_macro(id_index name, macro_data_ptr label_param_data, std::vector params); // leaves current macro @@ -198,6 +201,8 @@ class hlasm_context const copy_member_storage& copy_members(); // registers new copy member copy_member_ptr add_copy_member(id_index member, statement_block definition, location definition_location); + void add_copy_member(copy_member_ptr member); + copy_member_ptr get_copy_member(id_index member) const; // enters a copy member void enter_copy_member(id_index member); // leaves current copy member diff --git a/parser_library/src/context/id_storage.cpp b/parser_library/src/context/id_storage.cpp index dcab1237d..cb7723679 100644 --- a/parser_library/src/context/id_storage.cpp +++ b/parser_library/src/context/id_storage.cpp @@ -55,8 +55,7 @@ id_storage::const_pointer id_storage::add(std::string value, bool is_uri) return &*lit_.insert(std::move(value)).first; } -hlasm_plugin::parser_library::context::id_storage::well_known_strings::well_known_strings( - std::unordered_set& ptr) +id_storage::well_known_strings::well_known_strings(std::unordered_set& ptr) : COPY(&*ptr.emplace("COPY").first) , SETA(&*ptr.emplace("SETA").first) , SETB(&*ptr.emplace("SETB").first) @@ -69,6 +68,11 @@ hlasm_plugin::parser_library::context::id_storage::well_known_strings::well_know , LCLC(&*ptr.emplace("LCLC").first) , MACRO(&*ptr.emplace("MACRO").first) , MEND(&*ptr.emplace("MEND").first) + , MEXIT(&*ptr.emplace("MEXIT").first) , ASPACE(&*ptr.emplace("ASPACE").first) + , AIF(&*ptr.emplace("AIF").first) + , AGO(&*ptr.emplace("AGO").first) + , ACTR(&*ptr.emplace("ACTR").first) + , AREAD(&*ptr.emplace("AREAD").first) , empty(&*ptr.emplace("").first) {} diff --git a/parser_library/src/context/id_storage.h b/parser_library/src/context/id_storage.h index c94299771..f4ff98117 100644 --- a/parser_library/src/context/id_storage.h +++ b/parser_library/src/context/id_storage.h @@ -60,7 +60,12 @@ class id_storage const std::string* LCLC; const std::string* MACRO; const std::string* MEND; + const std::string* MEXIT; const std::string* ASPACE; + const std::string* AIF; + const std::string* AGO; + const std::string* ACTR; + const std::string* AREAD; const std::string* empty; well_known_strings(std::unordered_set& ptr); diff --git a/parser_library/src/context/macro.cpp b/parser_library/src/context/macro.cpp index c6a447c62..d707f26c6 100644 --- a/parser_library/src/context/macro.cpp +++ b/parser_library/src/context/macro.cpp @@ -35,12 +35,14 @@ macro_definition::macro_definition(id_index name, statement_block definition, copy_nest_storage copy_nests, label_storage labels, - location definition_location) + location definition_location, + std::unordered_set used_copy_members) : label_param_name_(label_param_name) , id(name) , copy_nests(std::move(copy_nests)) , labels(std::move(labels)) , definition_location(std::move(definition_location)) + , used_copy_members(std::move(used_copy_members)) { for (auto&& stmt : definition) cached_definition.emplace_back(std::move(stmt)); diff --git a/parser_library/src/context/macro.h b/parser_library/src/context/macro.h index 02a8a2af3..f76a8a957 100644 --- a/parser_library/src/context/macro.h +++ b/parser_library/src/context/macro.h @@ -19,6 +19,7 @@ #include #include "common_types.h" +#include "copy_member.h" #include "sequence_symbol.h" #include "statement_cache.h" #include "variables/macro_param.h" @@ -74,6 +75,7 @@ class macro_definition const label_storage labels; // location of the macro definition in code const location definition_location; + const std::unordered_set used_copy_members; // initializes macro with its name and params - positional or keyword macro_definition(id_index name, id_index label_param_name, @@ -81,7 +83,8 @@ class macro_definition statement_block definition, copy_nest_storage copy_nests, label_storage labels, - location definition_location); + location definition_location, + std::unordered_set used_copy_members); // returns object with parameters' data set to actual parameters in macro call macro_invo_ptr call(macro_data_ptr label_param_data, std::vector actual_params, id_index syslist_name); diff --git a/parser_library/src/context/statement_cache.cpp b/parser_library/src/context/statement_cache.cpp index 2967157d9..d22e60290 100644 --- a/parser_library/src/context/statement_cache.cpp +++ b/parser_library/src/context/statement_cache.cpp @@ -35,14 +35,15 @@ void statement_cache::insert(processing::processing_form format, cached_statemen cache_.emplace_back(format, std::move(statement)); } -statement_cache::cached_statement_t statement_cache::get(processing::processing_form format) const +const statement_cache::cached_statement_t* statement_cache::get(processing::processing_form format) const { for (const auto& entry : cache_) if (entry.first == format) - return entry.second; + return &entry.second; return nullptr; } shared_stmt_ptr statement_cache::get_base() const { return base_stmt_; } + } // namespace hlasm_plugin::parser_library::context diff --git a/parser_library/src/context/statement_cache.h b/parser_library/src/context/statement_cache.h index b2f100c9d..fe48a2d0f 100644 --- a/parser_library/src/context/statement_cache.h +++ b/parser_library/src/context/statement_cache.h @@ -15,6 +15,7 @@ #ifndef CONTEXT_PROCESSING_STATEMENT_CACHE_H #define CONTEXT_PROCESSING_STATEMENT_CACHE_H +#include "diagnostic.h" #include "hlasm_statement.h" #include "processing/processing_format.h" @@ -29,8 +30,11 @@ namespace hlasm_plugin::parser_library::context { class statement_cache { public: - using cached_statement_t = std::shared_ptr; - + struct cached_statement_t + { + std::shared_ptr stmt; + std::vector diags; + }; // pair of processing format and reparsed statement // processing format serves as an identifier of reparsing kind using cache_t = std::pair; @@ -46,7 +50,7 @@ class statement_cache void insert(processing::processing_form format, cached_statement_t statement); - cached_statement_t get(processing::processing_form format) const; + const cached_statement_t* get(processing::processing_form format) const; shared_stmt_ptr get_base() const; }; diff --git a/parser_library/src/lsp/lsp_context.cpp b/parser_library/src/lsp/lsp_context.cpp index 01c2c57c3..8f77baba0 100644 --- a/parser_library/src/lsp/lsp_context.cpp +++ b/parser_library/src/lsp/lsp_context.cpp @@ -106,6 +106,20 @@ void lsp_context::add_opencode(opencode_info_ptr opencode_i, text_data_ref_t tex distribute_file_occurences(opencode_->file_occurences); } +macro_info_ptr lsp_context::get_macro_info(context::id_index macro_name) const +{ + // Opencode may be empty, if parsing was interrupted by cancellation token, or when this function is called in the + // middle of processing + if (!opencode_) + return nullptr; + // This function does not respect OPSYN, so we do not use hlasm_context::get_macro_definition + auto it = opencode_->hlasm_ctx.macros().find(macro_name); + if (it == opencode_->hlasm_ctx.macros().end()) + return nullptr; + else + return macros_.at(it->second); +} + location lsp_context::definition(const std::string& document_uri, const position pos) const { auto [occ, macro_scope] = find_occurence_with_scope(document_uri, pos); diff --git a/parser_library/src/lsp/lsp_context.h b/parser_library/src/lsp/lsp_context.h index 9abc63278..0809dcde0 100644 --- a/parser_library/src/lsp/lsp_context.h +++ b/parser_library/src/lsp/lsp_context.h @@ -38,6 +38,8 @@ class lsp_context : public feature_provider void add_macro(macro_info_ptr macro_i, text_data_ref_t text_data = text_data_ref_t()); void add_opencode(opencode_info_ptr opencode_i, text_data_ref_t text_data); + [[nodiscard]] macro_info_ptr get_macro_info(context::id_index macro_name) const; + location definition(const std::string& document_uri, position pos) const override; location_list references(const std::string& document_uri, position pos) const override; hover_result hover(const std::string& document_uri, position pos) const override; diff --git a/parser_library/src/parsing/CMakeLists.txt b/parser_library/src/parsing/CMakeLists.txt index b2e0904b1..3820afa4b 100644 --- a/parser_library/src/parsing/CMakeLists.txt +++ b/parser_library/src/parsing/CMakeLists.txt @@ -14,8 +14,8 @@ target_sources(parser_library PRIVATE error_strategy.h parser_error_listener.cpp parser_error_listener.h - parser_error_listener_ctx.cpp parser_error_listener_ctx.h + parser_error_listener_substitution.h parser_impl.cpp parser_impl.h parser_tools.cpp diff --git a/parser_library/src/parsing/parser_error_listener.cpp b/parser_library/src/parsing/parser_error_listener.cpp index 620a7853a..e47b99a8c 100644 --- a/parser_library/src/parsing/parser_error_listener.cpp +++ b/parser_library/src/parsing/parser_error_listener.cpp @@ -215,7 +215,7 @@ void parser_error_listener_base::syntaxError( add_parser_diagnostic(range(position(line, char_pos_in_line)), diagnostic_severity::error, "S0006", - "Unexpected sign in an epxression"); + "Unexpected sign in an expression"); // apostrophe expected else if (odd_apostrophes && is_expected(APOSTROPHE, expected_tokens)) add_parser_diagnostic( diff --git a/parser_library/src/parsing/parser_error_listener_ctx.cpp b/parser_library/src/parsing/parser_error_listener_ctx.cpp deleted file mode 100644 index 383adf73f..000000000 --- a/parser_library/src/parsing/parser_error_listener_ctx.cpp +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2019 Broadcom. - * The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. - * - * This program and the accompanying materials are made - * available under the terms of the Eclipse Public License 2.0 - * which is available at https://www.eclipse.org/legal/epl-2.0/ - * - * SPDX-License-Identifier: EPL-2.0 - * - * Contributors: - * Broadcom, Inc. - initial API and implementation - */ - -#include "parser_error_listener_ctx.h" - -using namespace hlasm_plugin::parser_library; -using namespace hlasm_plugin::parser_library::parsing; - -parser_error_listener_ctx::parser_error_listener_ctx( - context::hlasm_context& hlasm_ctx, std::optional substituted, semantics::range_provider provider) - : diagnosable_ctx(hlasm_ctx) - , substituted_(std::move(substituted)) - , provider_(std::move(provider)) -{} - -void parser_error_listener_ctx::collect_diags() const {} - -void parser_error_listener_ctx::add_parser_diagnostic( - range diagnostic_range, diagnostic_severity severity, std::string code, std::string message) -{ - if (substituted_) - message = "While substituting to '" + *substituted_ + "' => " + message; - add_diagnostic( - diagnostic_op(severity, std::move(code), std::move(message), provider_.adjust_range(diagnostic_range))); -} diff --git a/parser_library/src/parsing/parser_error_listener_ctx.h b/parser_library/src/parsing/parser_error_listener_ctx.h index ae778f2d4..5442d8c33 100644 --- a/parser_library/src/parsing/parser_error_listener_ctx.h +++ b/parser_library/src/parsing/parser_error_listener_ctx.h @@ -16,29 +16,27 @@ #define HLASMPLUGIN_PARSERLIBRARY_ERROR_LISTENER_CTX_H #include "context/hlasm_context.h" -#include "parser_error_listener.h" -#include "semantics/range_provider.h" +#include "parser_error_listener_substitution.h" // implementation of parser error listener that provide additional error handling // used during recursed parsing when nested diagnostic is needed namespace hlasm_plugin::parser_library::parsing { -class parser_error_listener_ctx : public parser_error_listener_base, public diagnosable_ctx +class parser_error_listener_ctx : public diagnosable_ctx, public parser_error_listener_substitution { public: parser_error_listener_ctx(context::hlasm_context& hlasm_ctx, - std::optional substituted, - semantics::range_provider provider = semantics::range_provider()); - - void collect_diags() const override; - -protected: - void add_parser_diagnostic( - range diagnostic_range, diagnostic_severity severity, std::string code, std::string message) override; + const std::string* substituted, + semantics::range_provider provider = semantics::range_provider()) + : diagnosable_ctx(hlasm_ctx) + , parser_error_listener_substitution(add_diag_with_ctx, substituted, std::move(provider)) + {} + void collect_diags() const override {}; private: - std::optional substituted_; - semantics::range_provider provider_; + std::function add_diag_with_ctx = [this](diagnostic_op diag) { + this->add_diagnostic(std::move(diag)); + }; }; } // namespace hlasm_plugin::parser_library::parsing diff --git a/parser_library/src/parsing/parser_error_listener_substitution.h b/parser_library/src/parsing/parser_error_listener_substitution.h new file mode 100644 index 000000000..9542257f0 --- /dev/null +++ b/parser_library/src/parsing/parser_error_listener_substitution.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2019 Broadcom. + * The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Broadcom, Inc. - initial API and implementation + */ + +#ifndef HLASMPLUGIN_PARSERLIBRARY_ERROR_LISTENER_SUBSTITUTION_H +#define HLASMPLUGIN_PARSERLIBRARY_ERROR_LISTENER_SUBSTITUTION_H + +#include "parser_error_listener.h" +#include "semantics/range_provider.h" + +// implementation of parser error listener that provide additional error handling +// used during recursed parsing when nested diagnostic is needed +namespace hlasm_plugin::parser_library::parsing { + +class parser_error_listener_substitution : public parser_error_listener_base +{ +public: + parser_error_listener_substitution(const std::function& add_diag, + const std::string* subst_text, + semantics::range_provider rng_provider) + : add_diag_(&add_diag) + , subst_text_(subst_text) + , rng_provider_(std::move(rng_provider)) {}; + +protected: + void add_parser_diagnostic( + range diagnostic_range, diagnostic_severity severity, std::string code, std::string message) override + { + if (subst_text_) + message = "While substituting to '" + *subst_text_ + "' => " + message; + (*add_diag_)( + diagnostic_op(severity, std::move(code), std::move(message), rng_provider_.adjust_range(diagnostic_range))); + } + +private: + const std::function* add_diag_; + const std::string* subst_text_; + semantics::range_provider rng_provider_; +}; + +} // namespace hlasm_plugin::parser_library::parsing + + +#endif diff --git a/parser_library/src/parsing/parser_impl.cpp b/parser_library/src/parsing/parser_impl.cpp index 4278dd8b6..3ef4ce042 100644 --- a/parser_library/src/parsing/parser_impl.cpp +++ b/parser_library/src/parsing/parser_impl.cpp @@ -111,10 +111,11 @@ std::unique_ptr create_parser_holder() return h; } -std::pair parser_impl::parse_operand_field(std::string field, +statement_fields_parser::parse_result parser_impl::parse_operand_field(std::string field, bool after_substitution, semantics::range_provider field_range, - processing::processing_status status) + processing::processing_status status, + const std::function& add_diag) { if (!rest_parser_) rest_parser_ = create_parser_holder(); @@ -122,10 +123,7 @@ std::pair parser_impl::parse_oper hlasm_ctx->metrics.reparsed_statements++; const parser_holder& h = *rest_parser_; - std::optional sub; - if (after_substitution) - sub = field; - parser_error_listener_ctx listener(*hlasm_ctx, std::move(sub)); + parser_error_listener_substitution listener(add_diag, after_substitution ? &field : nullptr, field_range); h.input->reset(field); @@ -155,7 +153,7 @@ std::pair parser_impl::parse_oper case processing::processing_form::MAC: line = std::move(h.parser->op_rem_body_mac_r()->line); proc_status = status; - parse_macro_operands(line); + parse_macro_operands(line, add_diag); break; case processing::processing_form::ASM: line = std::move(h.parser->op_rem_body_asm_r()->line); @@ -171,8 +169,6 @@ std::pair parser_impl::parse_oper } } - collect_diags_from_child(listener); - for (size_t i = 0; i < line.operands.size(); i++) { if (!line.operands[i]) @@ -193,8 +189,8 @@ std::pair parser_impl::parse_oper ? range(op_range.end) : semantics::range_provider::union_range(line.remarks.front(), line.remarks.back()); - return std::make_pair(semantics::operands_si(op_range, std::move(line.operands)), - semantics::remarks_si(rem_range, std::move(line.remarks))); + return { semantics::operands_si(op_range, std::move(line.operands)), + semantics::remarks_si(rem_range, std::move(line.remarks)) }; } void parser_impl::collect_diags() const @@ -266,7 +262,7 @@ context::id_index parser_impl::parse_identifier(std::string value, range id_rang return hlasm_ctx->ids().add(std::move(value)); } -void parser_impl::parse_macro_operands(semantics::op_rem& line) +void parser_impl::parse_macro_operands(semantics::op_rem& line, const std::function& add_diag) { if (line.operands.size()) { @@ -291,7 +287,7 @@ void parser_impl::parse_macro_operands(semantics::op_rem& line) auto r = semantics::range_provider::union_range( line.operands.begin()->get()->operand_range, line.operands.back()->operand_range); - line.operands = parse_macro_operands(std::move(to_parse), r, std::move(ranges)); + line.operands = parse_macro_operands(std::move(to_parse), r, std::move(ranges), add_diag); } } @@ -313,14 +309,14 @@ void parser_impl::resolve_expression(std::vector& expr void parser_impl::resolve_expression(expressions::ca_expr_ptr& expr) const { auto [_, opcode] = *proc_status; - if (opcode.value == hlasm_ctx->ids().add("SETA") || opcode.value == hlasm_ctx->ids().add("ACTR") - || opcode.value == hlasm_ctx->ids().add("ASPACE") || opcode.value == hlasm_ctx->ids().add("AGO")) + if (opcode.value == hlasm_ctx->ids().well_known.SETA || opcode.value == hlasm_ctx->ids().well_known.ACTR + || opcode.value == hlasm_ctx->ids().well_known.ASPACE || opcode.value == hlasm_ctx->ids().well_known.AGO) resolve_expression(expr, context::SET_t_enum::A_TYPE); - else if (opcode.value == hlasm_ctx->ids().add("SETB") || opcode.value == hlasm_ctx->ids().add("AIF")) + else if (opcode.value == hlasm_ctx->ids().well_known.SETB || opcode.value == hlasm_ctx->ids().well_known.AIF) resolve_expression(expr, context::SET_t_enum::B_TYPE); - else if (opcode.value == hlasm_ctx->ids().add("SETC")) + else if (opcode.value == hlasm_ctx->ids().well_known.SETC) resolve_expression(expr, context::SET_t_enum::C_TYPE); - else if (opcode.value == hlasm_ctx->ids().add("AREAD")) + else if (opcode.value == hlasm_ctx->ids().well_known.AREAD) { // aread operand is just enumeration } @@ -471,8 +467,10 @@ void parser_impl::initialize( proc_status = proc_stat; } -semantics::operand_list parser_impl::parse_macro_operands( - std::string operands, range field_range, std::vector operand_ranges) +semantics::operand_list parser_impl::parse_macro_operands(std::string operands, + range field_range, + std::vector operand_ranges, + const std::function& add_diag) { if (!rest_parser_) rest_parser_ = create_parser_holder(); @@ -481,7 +479,7 @@ semantics::operand_list parser_impl::parse_macro_operands( semantics::range_provider tmp_provider(field_range, operand_ranges, semantics::adjusting_state::MACRO_REPARSE); - parser_error_listener_ctx listener(*hlasm_ctx, std::nullopt, tmp_provider); + parser_error_listener_substitution listener(add_diag, nullptr, tmp_provider); h.input->reset(operands); @@ -502,8 +500,6 @@ semantics::operand_list parser_impl::parse_macro_operands( auto list = std::move(h.parser->macro_ops()->list); - collect_diags_from_child(listener); - return list; } @@ -545,7 +541,7 @@ void parser_impl::parse_operands(const std::string& text, range text_range) parser_holder& h = *rest_parser_; - parser_error_listener_ctx listener(*hlasm_ctx, std::nullopt); + parser_error_listener_ctx listener(*hlasm_ctx, nullptr); h.input->reset(text); @@ -585,7 +581,8 @@ void parser_impl::parse_operands(const std::string& text, range text_range) auto rule = h.parser->op_rem_body_mac(); auto line = std::move(rule->line); auto line_range = rule->line_range; - parse_macro_operands(line); + parse_macro_operands( + line, [&listener](diagnostic_op diag) { listener.add_diagnostic(std::move(diag)); }); h.parser->collector.set_operand_remark_field( std::move(line.operands), std::move(line.remarks), line_range); } @@ -631,7 +628,7 @@ void parser_impl::parse_lookahead_operands(const std::string& text, range text_r { context::id_index tmp; tmp = std::get(collector.current_instruction().value); - if (tmp != hlasm_ctx->ids().add("COPY")) + if (tmp != hlasm_ctx->ids().well_known.COPY) { process_statement(); return; @@ -641,7 +638,7 @@ void parser_impl::parse_lookahead_operands(const std::string& text, range text_r const parser_holder& h = *rest_parser_; - parser_error_listener_ctx listener(*hlasm_ctx, std::nullopt); + parser_error_listener_ctx listener(*hlasm_ctx, nullptr); h.input->reset(text); diff --git a/parser_library/src/parsing/parser_impl.h b/parser_library/src/parsing/parser_impl.h index de534fd79..3bb57ca81 100644 --- a/parser_library/src/parsing/parser_impl.h +++ b/parser_library/src/parsing/parser_impl.h @@ -64,7 +64,8 @@ class parser_impl : public antlr4::Parser, processing::statement_fields_parser::parse_result parse_operand_field(std::string field, bool after_substitution, semantics::range_provider field_range, - processing::processing_status status) override; + processing::processing_status status, + const std::function& add_diag) override; context::shared_stmt_ptr get_next(const processing::statement_processor& processor) override; @@ -82,7 +83,7 @@ class parser_impl : public antlr4::Parser, self_def_t parse_self_def_term(const std::string& option, const std::string& value, range term_range); context::data_attr_kind get_attribute(std::string attr_data, range data_range); context::id_index parse_identifier(std::string value, range id_range); - void parse_macro_operands(semantics::op_rem& line); + void parse_macro_operands(semantics::op_rem& line, const std::function& add_diag); void resolve_expression(expressions::ca_expr_ptr& expr, context::SET_t_enum type) const; void resolve_expression(std::vector& expr, context::SET_t_enum type) const; @@ -125,8 +126,10 @@ class parser_impl : public antlr4::Parser, semantics::range_provider range_prov, processing::processing_status proc_stat); - semantics::operand_list parse_macro_operands( - std::string operands, range field_range, std::vector operand_ranges); + semantics::operand_list parse_macro_operands(std::string operands, + range field_range, + std::vector operand_ranges, + const std::function& add_diag); // process methods return true if attribute lookahead needed bool process_instruction(); diff --git a/parser_library/src/processing/instruction_sets/asm_processor.cpp b/parser_library/src/processing/instruction_sets/asm_processor.cpp index 60714b065..b3ca4ca31 100644 --- a/parser_library/src/processing/instruction_sets/asm_processor.cpp +++ b/parser_library/src/processing/instruction_sets/asm_processor.cpp @@ -588,7 +588,7 @@ asm_processor::process_table_t asm_processor::create_table(context::hlasm_contex table.emplace(h_ctx.ids().add("EQU"), std::bind(&asm_processor::process_EQU, this, std::placeholders::_1)); table.emplace(h_ctx.ids().add("DC"), std::bind(&asm_processor::process_DC, this, std::placeholders::_1)); table.emplace(h_ctx.ids().add("DS"), std::bind(&asm_processor::process_DS, this, std::placeholders::_1)); - table.emplace(h_ctx.ids().add("COPY"), std::bind(&asm_processor::process_COPY, this, std::placeholders::_1)); + table.emplace(h_ctx.ids().well_known.COPY, std::bind(&asm_processor::process_COPY, this, std::placeholders::_1)); table.emplace(h_ctx.ids().add("EXTRN"), std::bind(&asm_processor::process_EXTRN, this, std::placeholders::_1)); table.emplace(h_ctx.ids().add("ORG"), std::bind(&asm_processor::process_ORG, this, std::placeholders::_1)); table.emplace(h_ctx.ids().add("OPSYN"), std::bind(&asm_processor::process_OPSYN, this, std::placeholders::_1)); diff --git a/parser_library/src/processing/instruction_sets/low_language_processor.cpp b/parser_library/src/processing/instruction_sets/low_language_processor.cpp index fb01e4f4d..6c38b09d2 100644 --- a/parser_library/src/processing/instruction_sets/low_language_processor.cpp +++ b/parser_library/src/processing/instruction_sets/low_language_processor.cpp @@ -109,7 +109,8 @@ low_language_processor::preprocessed_part low_language_processor::preprocess_inn true, semantics::range_provider(stmt.operands_ref().value[0]->operand_range, semantics::adjusting_state::SUBSTITUTION), - processing_status(stmt.format_ref(), stmt.opcode_ref())) + processing_status(stmt.format_ref(), stmt.opcode_ref()), + [this](diagnostic_op diag) { this->add_diagnostic(std::move(diag)); }) .first); } diff --git a/parser_library/src/processing/processing_manager.cpp b/parser_library/src/processing/processing_manager.cpp index 371a6fa55..9a9a0705d 100644 --- a/parser_library/src/processing/processing_manager.cpp +++ b/parser_library/src/processing/processing_manager.cpp @@ -175,7 +175,8 @@ void processing_manager::finish_macro_definition(macrodef_processing_result resu std::move(result.definition), std::move(result.nests), std::move(result.sequence_symbols), - std::move(result.definition_location)); + std::move(result.definition_location), + std::move(result.used_copy_members)); lsp_analyzer_.macrodef_finished(mac, std::move(result)); } @@ -320,6 +321,10 @@ void processing_manager::collect_diags() const for (auto& proc : procs_) collect_diags_from_child(*proc); + + collect_diags_from_child(dynamic_cast(*provs_[0])); + if (provs_.size() > 2) + collect_diags_from_child(dynamic_cast(*provs_[1])); collect_diags_from_child(dynamic_cast(*provs_.back())); } diff --git a/parser_library/src/processing/statement_analyzers/lsp_analyzer.cpp b/parser_library/src/processing/statement_analyzers/lsp_analyzer.cpp index 7ae4f19fe..620d4ada4 100644 --- a/parser_library/src/processing/statement_analyzers/lsp_analyzer.cpp +++ b/parser_library/src/processing/statement_analyzers/lsp_analyzer.cpp @@ -259,7 +259,7 @@ void lsp_analyzer::collect_copy_operands(const context::hlasm_statement& stateme if (!res_stmt) return; - if (res_stmt->opcode_ref().value == hlasm_ctx_.ids().add("COPY") && res_stmt->operands_ref().value.size() == 1 + if (res_stmt->opcode_ref().value == hlasm_ctx_.ids().well_known.COPY && res_stmt->operands_ref().value.size() == 1 && res_stmt->operands_ref().value.front()->access_asm()) { auto sym_expr = dynamic_cast( diff --git a/parser_library/src/processing/statement_fields_parser.h b/parser_library/src/processing/statement_fields_parser.h index 280d40a73..f4098a312 100644 --- a/parser_library/src/processing/statement_fields_parser.h +++ b/parser_library/src/processing/statement_fields_parser.h @@ -34,7 +34,8 @@ class statement_fields_parser virtual parse_result parse_operand_field(std::string field, bool after_substitution, semantics::range_provider field_range, - processing::processing_status status) = 0; + processing::processing_status status, + const std::function& add_diag) = 0; virtual ~statement_fields_parser() = default; }; diff --git a/parser_library/src/processing/statement_processors/macrodef_processing_info.h b/parser_library/src/processing/statement_processors/macrodef_processing_info.h index 7b7b1f6a8..3d13202aa 100644 --- a/parser_library/src/processing/statement_processors/macrodef_processing_info.h +++ b/parser_library/src/processing/statement_processors/macrodef_processing_info.h @@ -54,6 +54,7 @@ struct macrodef_processing_result context::statement_block definition; context::copy_nest_storage nests; context::label_storage sequence_symbols; + std::unordered_set used_copy_members; lsp::vardef_storage variable_symbols; lsp::file_scopes_t file_scopes; diff --git a/parser_library/src/processing/statement_processors/macrodef_processor.cpp b/parser_library/src/processing/statement_processors/macrodef_processor.cpp index 407e4b9c6..b5720d27c 100644 --- a/parser_library/src/processing/statement_processors/macrodef_processor.cpp +++ b/parser_library/src/processing/statement_processors/macrodef_processor.cpp @@ -462,10 +462,12 @@ void macrodef_processor::add_correct_copy_nest() { auto& nest = hlasm_ctx.current_copy_stack()[i]; auto pos = nest.cached_definition[nest.current_statement].get_base()->statement_position(); - auto loc = location(pos, nest.definition_location.file); - result_.nests.back().push_back(std::move(loc)); + result_.nests.back().emplace_back(pos, nest.definition_location.file); } + if (initial_copy_nest_ < hlasm_ctx.current_copy_stack().size()) + result_.used_copy_members.insert(hlasm_ctx.current_copy_stack().back().copy_member_definition); + const auto& current_file = result_.nests.back().back().file; bool in_inner_macro = macro_nest_ > 1; @@ -484,7 +486,7 @@ void macrodef_processor::add_correct_copy_nest() if (inner_macro_ended) // add new scope when inner macro ended result_.file_scopes[current_file].emplace_back(curr_line_, in_inner_macro); else if (!in_inner_macro - || inner_macro_started) // if we are not in inner macro update the end of old scope. Update also when inner + || inner_macro_started) // if we are not in inner macro, update the end of old scope. Update also when inner // macro just started, since we use half-open intervals. { auto& last_scope = result_.file_scopes[current_file].back(); diff --git a/parser_library/src/processing/statement_providers/members_statement_provider.cpp b/parser_library/src/processing/statement_providers/members_statement_provider.cpp index 3724fd055..1ec731466 100644 --- a/parser_library/src/processing/statement_providers/members_statement_provider.cpp +++ b/parser_library/src/processing/statement_providers/members_statement_provider.cpp @@ -22,6 +22,7 @@ members_statement_provider::members_statement_provider(const statement_provider_ workspaces::parse_lib_provider& lib_provider, processing::processing_state_listener& listener) : statement_provider(kind) + , diagnosable_ctx(*ctx.hlasm_ctx) , ctx(std::move(ctx)) , parser(parser) , lib_provider(lib_provider) @@ -83,7 +84,7 @@ const semantics::instruction_si& members_statement_provider::retrieve_instructio void members_statement_provider::fill_cache( context::statement_cache& cache, const semantics::deferred_statement& def_stmt, const processing_status& status) { - context::statement_cache::cached_statement_t ptr; + context::statement_cache::cached_statement_t reparsed_stmt; auto def_impl = std::dynamic_pointer_cast(cache.get_base()); if (status.first.occurence == operand_occurence::ABSENT || status.first.form == processing_form::UNKNOWN @@ -92,18 +93,21 @@ void members_statement_provider::fill_cache( semantics::operands_si op(def_stmt.deferred_ref().field_range, semantics::operand_list()); semantics::remarks_si rem(def_stmt.deferred_ref().field_range, {}); - ptr = std::make_shared(def_impl, std::move(op), std::move(rem)); + reparsed_stmt.stmt = + std::make_shared(def_impl, std::move(op), std::move(rem)); } else { auto [op, rem] = parser.parse_operand_field(def_stmt.deferred_ref().value, false, semantics::range_provider(def_stmt.deferred_ref().field_range, semantics::adjusting_state::NONE), - status); + status, + [&reparsed_stmt](diagnostic_op diag) { reparsed_stmt.diags.push_back(std::move(diag)); }); - ptr = std::make_shared(def_impl, std::move(op), std::move(rem)); + reparsed_stmt.stmt = + std::make_shared(def_impl, std::move(op), std::move(rem)); } - cache.insert(status.first.form, ptr); + cache.insert(status.first.form, reparsed_stmt); } context::shared_stmt_ptr members_statement_provider::preprocess_deferred( @@ -119,7 +123,17 @@ context::shared_stmt_ptr members_statement_provider::preprocess_deferred( if (!cache.contains(status.first.form)) fill_cache(cache, def_stmt, status); - return std::make_shared(cache.get(status.first.form), status); + const auto& cache_item = cache.get(status.first.form); + + for (const diagnostic_op& diag : cache_item->diags) + add_diagnostic(diag); + + return std::make_shared(cache_item->stmt, status); +} + +void members_statement_provider::collect_diags() const +{ + // No diagnosable children } } // namespace hlasm_plugin::parser_library::processing diff --git a/parser_library/src/processing/statement_providers/members_statement_provider.h b/parser_library/src/processing/statement_providers/members_statement_provider.h index e3530551f..90b79c0a2 100644 --- a/parser_library/src/processing/statement_providers/members_statement_provider.h +++ b/parser_library/src/processing/statement_providers/members_statement_provider.h @@ -24,7 +24,7 @@ namespace hlasm_plugin::parser_library::processing { // common class for copy and macro statement providers (provider of copy and macro members) -class members_statement_provider : public statement_provider +class members_statement_provider : public statement_provider, public diagnosable_ctx { public: members_statement_provider(const statement_provider_kind kind, @@ -35,6 +35,7 @@ class members_statement_provider : public statement_provider context::shared_stmt_ptr get_next(const statement_processor& processor) override; + protected: analyzing_context ctx; statement_fields_parser& parser; @@ -51,6 +52,9 @@ class members_statement_provider : public statement_provider const processing_status& status); context::shared_stmt_ptr preprocess_deferred(const statement_processor& processor, context::statement_cache& cache); + + // Inherited via diagnosable_ctx + void collect_diags() const override; }; } // namespace hlasm_plugin::parser_library::processing diff --git a/parser_library/src/semantics/range_provider.cpp b/parser_library/src/semantics/range_provider.cpp index 9cf7b02cd..e12595521 100644 --- a/parser_library/src/semantics/range_provider.cpp +++ b/parser_library/src/semantics/range_provider.cpp @@ -34,7 +34,7 @@ range range_provider::union_range(const range& lhs, const range& rhs) return range(ret[0], ret[1]); } -range range_provider::get_range(const antlr4::Token* start, const antlr4::Token* stop) +range range_provider::get_range(const antlr4::Token* start, const antlr4::Token* stop) const { range ret; @@ -53,14 +53,14 @@ range range_provider::get_range(const antlr4::Token* start, const antlr4::Token* return adjust_range(ret); } -range range_provider::get_range(const antlr4::Token* terminal) { return get_range(terminal, terminal); } +range range_provider::get_range(const antlr4::Token* terminal) const { return get_range(terminal, terminal); } -range range_provider::get_range(antlr4::ParserRuleContext* non_terminal) +range range_provider::get_range(antlr4::ParserRuleContext* non_terminal) const { return get_range(non_terminal->getStart(), non_terminal->getStop()); } -range range_provider::get_empty_range(const antlr4::Token* start) +range range_provider::get_empty_range(const antlr4::Token* start) const { range ret; ret.start.line = start->getLine(); @@ -69,7 +69,7 @@ range range_provider::get_empty_range(const antlr4::Token* start) return adjust_range(ret); } -range range_provider::adjust_range(range r) +range range_provider::adjust_range(range r) const { if (state == adjusting_state::MACRO_REPARSE) return range(adjust_position(r.start), adjust_position(r.end)); @@ -81,7 +81,7 @@ range range_provider::adjust_range(range r) return r; } -position range_provider::adjust_position(position pos) +position range_provider::adjust_position(position pos) const { size_t idx = 1; auto orig_range = original_operand_ranges.empty() ? original_range : original_operand_ranges.front(); diff --git a/parser_library/src/semantics/range_provider.h b/parser_library/src/semantics/range_provider.h index ac269152f..d5828e255 100644 --- a/parser_library/src/semantics/range_provider.h +++ b/parser_library/src/semantics/range_provider.h @@ -43,16 +43,16 @@ struct range_provider static range union_range(const range& lhs, const range& rhs); - range get_range(const antlr4::Token* start, const antlr4::Token* stop); - range get_range(const antlr4::Token* terminal); - range get_range(antlr4::ParserRuleContext* non_terminal); + range get_range(const antlr4::Token* start, const antlr4::Token* stop) const; + range get_range(const antlr4::Token* terminal) const; + range get_range(antlr4::ParserRuleContext* non_terminal) const; - range get_empty_range(const antlr4::Token* start); + range get_empty_range(const antlr4::Token* start) const; - range adjust_range(range r); + range adjust_range(range r) const; private: - position adjust_position(position pos); + position adjust_position(position pos) const; }; } // namespace hlasm_plugin::parser_library::semantics diff --git a/parser_library/src/workspaces/CMakeLists.txt b/parser_library/src/workspaces/CMakeLists.txt index 7f662c785..529f984dd 100644 --- a/parser_library/src/workspaces/CMakeLists.txt +++ b/parser_library/src/workspaces/CMakeLists.txt @@ -20,6 +20,8 @@ target_sources(parser_library PRIVATE library.h library_local.cpp library_local.h + macro_cache.cpp + macro_cache.h parse_lib_provider.cpp parse_lib_provider.h processor.h diff --git a/parser_library/src/workspaces/file_manager.h b/parser_library/src/workspaces/file_manager.h index a5896afb0..9b3770831 100644 --- a/parser_library/src/workspaces/file_manager.h +++ b/parser_library/src/workspaces/file_manager.h @@ -50,7 +50,7 @@ class file_manager : public virtual diagnosable virtual void remove_file(const file_uri&) = 0; // Finds file with specified file name, return nullptr if not found. - virtual file_ptr find(const std::string& key) = 0; + virtual file_ptr find(const std::string& key) const = 0; // Finds processor file with specified file name. // If there is a file with the file name, it is changed to processor_file. // Returns nullptr if there is no such file. diff --git a/parser_library/src/workspaces/file_manager_impl.cpp b/parser_library/src/workspaces/file_manager_impl.cpp index 43d43cbe1..1def4acd0 100644 --- a/parser_library/src/workspaces/file_manager_impl.cpp +++ b/parser_library/src/workspaces/file_manager_impl.cpp @@ -44,7 +44,7 @@ processor_file_ptr file_manager_impl::change_into_processor_file_if_not_already_ return processor; else { - auto proc_file = std::make_shared(std::move(*to_change), cancel_); + auto proc_file = std::make_shared(std::move(*to_change), *this, cancel_); to_change = proc_file; return proc_file; } @@ -56,7 +56,7 @@ processor_file_ptr file_manager_impl::add_processor_file(const file_uri& uri) auto ret = files_.find(uri); if (ret == files_.end()) { - auto ptr = std::make_shared(uri, cancel_); + auto ptr = std::make_shared(uri, *this, cancel_); files_.emplace(uri, ptr); return ptr; } @@ -70,7 +70,7 @@ processor_file_ptr file_manager_impl::get_processor_file(const file_uri& uri) auto ret = files_.find(uri); if (ret == files_.end()) { - return std::make_shared(uri); + return std::make_shared(uri, *this); } else return change_into_processor_file_if_not_already_(ret->second); @@ -87,7 +87,7 @@ void file_manager_impl::remove_file(const file_uri& document_uri) files_.erase(document_uri); } -file_ptr file_manager_impl::find(const std::string& key) +file_ptr file_manager_impl::find(const std::string& key) const { std::lock_guard guard(files_mutex); auto ret = files_.find(key); @@ -107,19 +107,6 @@ processor_file_ptr file_manager_impl::find_processor_file(const std::string& key return change_into_processor_file_if_not_already_(ret->second); } - -std::vector file_manager_impl::list_updated_files() -{ - std::vector list; - for (auto& item : files_) - { - auto p = dynamic_cast(item.second.get()); - if (p && p->parse_info_updated()) - list.push_back(p); - } - return list; -} - list_directory_result file_manager_impl::list_directory_files(const std::string& path) { std::filesystem::path lib_p(path); @@ -138,7 +125,7 @@ void file_manager_impl::prepare_file_for_change_(std::shared_ptr& fil // another shared ptr to this file exists, we need to create a copy auto proc_file = std::dynamic_pointer_cast(file); if (proc_file) - file = std::make_shared(*file, cancel_); + file = std::make_shared(*file, *this, cancel_); else file = std::make_shared(*file); } diff --git a/parser_library/src/workspaces/file_manager_impl.h b/parser_library/src/workspaces/file_manager_impl.h index 77a5f299c..7a29604d9 100644 --- a/parser_library/src/workspaces/file_manager_impl.h +++ b/parser_library/src/workspaces/file_manager_impl.h @@ -45,11 +45,9 @@ class file_manager_impl : public file_manager, public diagnosable_impl processor_file_ptr get_processor_file(const file_uri&) override; void remove_file(const file_uri&) override; - file_ptr find(const std::string& key) override; + file_ptr find(const std::string& key) const override; processor_file_ptr find_processor_file(const std::string& key) override; - // Returns array of files that were updated since this method was last called - virtual std::vector list_updated_files(); list_directory_result list_directory_files(const std::string& path) override; void did_open_file(const std::string& document_uri, version_t version, std::string text) override; @@ -66,7 +64,7 @@ class file_manager_impl : public file_manager, public diagnosable_impl std::unordered_map> files_; private: - std::mutex files_mutex; + mutable std::mutex files_mutex; std::atomic* cancel_; diff --git a/parser_library/src/workspaces/macro_cache.cpp b/parser_library/src/workspaces/macro_cache.cpp new file mode 100644 index 000000000..93cbd7e75 --- /dev/null +++ b/parser_library/src/workspaces/macro_cache.cpp @@ -0,0 +1,201 @@ +/* + * Copyright (c) 2021 Broadcom. + * The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Broadcom, Inc. - initial API and implementation + */ + +#include "macro_cache.h" + +#include + +#include "file_manager.h" + +namespace hlasm_plugin::parser_library::workspaces { + +void macro_cache::collect_diags() const +{ + // No collectible children +} + +macro_cache::macro_cache(const file_manager& file_mngr, file& macro_file) + : file_mngr_(&file_mngr) + , macro_file_(¯o_file) +{} + +std::vector macro_cache_key::get_opsyn_state(context::hlasm_context& ctx) +{ + auto& wn = ctx.ids().well_known; + // List of instructions that are resolved during macro definition - therefore are affected by OPSYN + std::array cached_instr { wn.COPY, + wn.ASPACE, + wn.GBLA, + wn.GBLB, + wn.GBLC, + wn.LCLA, + wn.LCLB, + wn.LCLC, + wn.SETA, + wn.SETB, + wn.SETC, + wn.MEND, + wn.MACRO, + wn.MEXIT, + wn.AIF, + wn.AREAD, + wn.ACTR, + wn.AGO }; + + std::vector result; + + for (const auto& [from, opcode] : ctx.opcode_mnemo_storage()) + { + // If there is an opsyn, that aliases an instruction to be CA instruction, add it to result + if (std::find(cached_instr.begin(), cached_instr.end(), opcode.opcode) != cached_instr.end()) + result.push_back( + { from, opcode.opcode, std::holds_alternative(opcode.opcode_detail) }); + + if (std::find(cached_instr.begin(), cached_instr.end(), from) != cached_instr.end()) + result.push_back( + { from, opcode.opcode, std::holds_alternative(opcode.opcode_detail) }); + } + + // Also macros with the same name as CA instructions may alias the instructions + for (const auto& [id, macro] : ctx.macros()) + { + if (!macro) + continue; + auto opcode = ctx.get_operation_code(id); + // Macros for which there is an opsyn in effect are already captured + if (auto pval = std::get_if(&opcode.opcode_detail); !pval || *pval != macro) + continue; + + // If there is an opsyn, that aliases an instruction to be CA instruction, add it to result + if (std::find(cached_instr.begin(), cached_instr.end(), macro->id) != cached_instr.end()) + result.push_back({ id, id, true }); + } + + sort_opsyn_state(result); + + return result; +} + +macro_cache_key macro_cache_key::create_from_context(context::hlasm_context& hlasm_ctx, library_data data) +{ + return { hlasm_ctx.opencode_file_name(), data, get_opsyn_state(hlasm_ctx) }; +} + +void macro_cache_key::sort_opsyn_state(std::vector& opsyn_state) +{ + std::sort(opsyn_state.begin(), opsyn_state.end(), [](const cached_opsyn_mnemo& lhs, const cached_opsyn_mnemo& rhs) { + return std::tie(lhs.from_instr, lhs.to_instr, lhs.is_macro) + < std::tie(rhs.from_instr, rhs.to_instr, rhs.is_macro); + }); +} + +const analyzer* macro_cache::find_cached_analyzer(const macro_cache_key& key) const +{ + auto it = cache_.find(key); + if (it == cache_.end()) + return nullptr; + + const auto& cached_data = it->second; + + for (const auto& [fname, cached_version] : cached_data.stamps) + { + auto file = file_mngr_->find(fname); + if (!file) + return nullptr; // Reparse needed + if (file->get_version() != cached_version) + return nullptr; + } + + // Version of all dependent files are the same. + return cached_data.cached_analyzer.get(); +} + +bool macro_cache::load_from_cache(const macro_cache_key& key, const analyzing_context& ctx) +{ + if (auto cached_analyzer = find_cached_analyzer(key)) + { + if (key.data.proc_kind == processing::processing_kind::MACRO) + { + lsp::macro_info_ptr info = cached_analyzer->context().lsp_ctx->get_macro_info(key.data.library_member); + if (!info) + return true; // The file for which the analyzer is cached does not contain definition of macro + ctx.hlasm_ctx->add_macro(info->macro_definition); + ctx.lsp_ctx->add_macro(info, lsp::text_data_ref_t(macro_file_->get_text())); + + // Add all copy members on which this macro is dependant + for (const auto& copy_ptr : info->macro_definition->used_copy_members) + { + auto copy_file = file_mngr_->find(copy_ptr->definition_location.file); + ctx.hlasm_ctx->add_copy_member(copy_ptr); + ctx.lsp_ctx->add_copy(copy_ptr, lsp::text_data_ref_t(copy_file->get_text())); + } + } + else if (key.data.proc_kind == processing::processing_kind::COPY) + { + auto copy_member = cached_analyzer->context().hlasm_ctx->get_copy_member(key.data.library_member); + ctx.hlasm_ctx->add_copy_member(copy_member); + ctx.lsp_ctx->add_copy(copy_member, lsp::text_data_ref_t(macro_file_->get_text())); + } + + return true; + } + return false; +} + +version_stamp macro_cache::get_copy_member_versions(context::macro_def_ptr macro) const +{ + version_stamp result; + + for (const auto& copy_ptr : macro->used_copy_members) + { + auto file = file_mngr_->find(copy_ptr->definition_location.file); + if (!file) + throw std::runtime_error("Dependencies of a macro must be open right after parsing the macro."); + result.try_emplace(file->get_file_name(), file->get_version()); + } + return result; +} + +void macro_cache::save_analyzer(const macro_cache_key& key, std::unique_ptr analyzer) +{ + auto& cache_data = cache_[key]; + if (key.data.proc_kind == processing::processing_kind::MACRO) + { + // Add stamps for all macro dependencies + auto parsed_macro = analyzer->context().hlasm_ctx->get_macro_definition(key.data.library_member); + if (parsed_macro) + cache_data.stamps = get_copy_member_versions(std::move(parsed_macro)); + else + cache_data.stamps.clear(); + } + else // Copy members do not have additional dependencies + cache_data.stamps.clear(); + + cache_data.stamps.try_emplace(macro_file_->get_file_name(), macro_file_->get_version()); + cache_data.cached_analyzer = std::move(analyzer); +} + +void macro_cache::erase_cache_of_opencode(const std::string& opencode_file_name) +{ + auto it = cache_.begin(); + while (it != cache_.end()) + { + if (it->first.opencode_file_name == opencode_file_name) + it = cache_.erase(it); + else + ++it; + } +} + +} // namespace hlasm_plugin::parser_library::workspaces diff --git a/parser_library/src/workspaces/macro_cache.h b/parser_library/src/workspaces/macro_cache.h new file mode 100644 index 000000000..997caf142 --- /dev/null +++ b/parser_library/src/workspaces/macro_cache.h @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2021 Broadcom. + * The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Broadcom, Inc. - initial API and implementation + */ + +#ifndef HLASMPLUGIN_PARSERLIBRARY_MACRO_CACHE_H +#define HLASMPLUGIN_PARSERLIBRARY_MACRO_CACHE_H + +#include "analyzer.h" + +namespace hlasm_plugin::parser_library::workspaces { + +class file_manager; +class file; + +struct cached_opsyn_mnemo +{ + context::id_index from_instr; + context::id_index to_instr; + bool is_macro; +}; + +// Contains all the context that affects parsing an external file (macro or copy member) +struct macro_cache_key +{ + [[nodiscard]] static macro_cache_key create_from_context(context::hlasm_context& hlasm_ctx, library_data data); + static void sort_opsyn_state(std::vector& opsyn_state); + static std::vector get_opsyn_state(context::hlasm_context& hlasm_ctx); + std::string opencode_file_name; + library_data data; + std::vector opsyn_state; +}; + + + +bool inline operator<(const macro_cache_key& lhs, const macro_cache_key& rhs) +{ + const static auto tie_macro_cache_key = [](const macro_cache_key& key) { + return std::tie(key.data.proc_kind, key.data.library_member, key.opsyn_state); + }; + + return tie_macro_cache_key(lhs) < tie_macro_cache_key(rhs); +} + +bool inline operator<(const cached_opsyn_mnemo& lhs, const cached_opsyn_mnemo& rhs) +{ + const static auto tie_cached_opsyn_mnemo = [](const cached_opsyn_mnemo& item) { + return std::tie(item.from_instr, item.to_instr, item.is_macro); + }; + + return tie_cached_opsyn_mnemo(lhs) < tie_cached_opsyn_mnemo(rhs); +} +bool inline operator==(const cached_opsyn_mnemo& lhs, const cached_opsyn_mnemo& rhs) +{ + return lhs.from_instr == rhs.from_instr && lhs.to_instr == rhs.to_instr && lhs.is_macro == rhs.is_macro; +} + +using version_stamp = std::unordered_map; + +// Pair of version stamp and analyzer that parsed the version of file(s) +struct macro_cache_data +{ + // Versions of respective macro (copy member) with all dependencies (COPY instruction is evaluated during macro + // definition and the statements are part of macro definition) + version_stamp stamps; + std::unique_ptr cached_analyzer; +}; + +class macro_cache final : public diagnosable_impl +{ + std::map cache_; + const file_manager* file_mngr_; + file* macro_file_; + +public: + macro_cache(const file_manager& file_mngr, file& macro_file); + // Checks whether any dependencies with specified macro cache key (macro context) have changed. If not, loads the + // cached macro to the specified context. Returns true, if the macro was loaded. + bool load_from_cache(const macro_cache_key& key, const analyzing_context& ctx); + void save_analyzer(const macro_cache_key& key, std::unique_ptr analyzer); + void erase_cache_of_opencode(const std::string& opencode_file_name); + +private: + [[nodiscard]] const analyzer* find_cached_analyzer(const macro_cache_key& key) const; + [[nodiscard]] version_stamp get_copy_member_versions(context::macro_def_ptr ctx) const; + + void collect_diags() const override; +}; + +} // namespace hlasm_plugin::parser_library::workspaces + + +#endif diff --git a/parser_library/src/workspaces/processor.h b/parser_library/src/workspaces/processor.h index 15da48f5f..863ec258c 100644 --- a/parser_library/src/workspaces/processor.h +++ b/parser_library/src/workspaces/processor.h @@ -30,8 +30,6 @@ namespace hlasm_plugin::parser_library::workspaces { class processor : public virtual diagnosable { public: - virtual bool parse_info_updated() = 0; - // starts parser with new (empty) context virtual parse_result parse(parse_lib_provider&) = 0; // starts parser with in the context of parameter @@ -52,6 +50,7 @@ class processor_file : public virtual file, public processor virtual const lsp::feature_provider& get_lsp_feature_provider() = 0; virtual const std::set& files_to_close() = 0; virtual const performance_metrics& get_metrics() = 0; + virtual void erase_cache_of_opencode(const std::string& opencode_file_name) = 0; protected: ~processor_file() = default; diff --git a/parser_library/src/workspaces/processor_file_impl.cpp b/parser_library/src/workspaces/processor_file_impl.cpp index d7f6cd129..408a629ef 100644 --- a/parser_library/src/workspaces/processor_file_impl.cpp +++ b/parser_library/src/workspaces/processor_file_impl.cpp @@ -21,19 +21,24 @@ namespace hlasm_plugin::parser_library::workspaces { -processor_file_impl::processor_file_impl(std::string file_name, std::atomic* cancel) +processor_file_impl::processor_file_impl( + std::string file_name, const file_manager& file_mngr, std::atomic* cancel) : file_impl(std::move(file_name)) , cancel_(cancel) + , macro_cache_(file_mngr, *this) {} -processor_file_impl::processor_file_impl(file_impl&& f_impl, std::atomic* cancel) +processor_file_impl::processor_file_impl(file_impl&& f_impl, const file_manager& file_mngr, std::atomic* cancel) : file_impl(std::move(f_impl)) , cancel_(cancel) + , macro_cache_(file_mngr, *this) {} -processor_file_impl::processor_file_impl(const file_impl& file, std::atomic* cancel) +processor_file_impl::processor_file_impl( + const file_impl& file, const file_manager& file_mngr, std::atomic* cancel) : file_impl(file) , cancel_(cancel) + , macro_cache_(file_mngr, *this) {} void processor_file_impl::collect_diags() const { file_impl::collect_diags(); } @@ -42,16 +47,23 @@ bool processor_file_impl::is_once_only() const { return false; } parse_result processor_file_impl::parse(parse_lib_provider& lib_provider) { - analyzer_ = std::make_unique(get_text(), get_file_name(), lib_provider, get_lsp_editing()); + if (opencode_analyzer_) + opencode_analyzer_ = std::make_unique(get_text(), + get_file_name(), + lib_provider, + get_lsp_editing(), + std::move(opencode_analyzer_->hlasm_ctx().ids())); + else + opencode_analyzer_ = std::make_unique(get_text(), get_file_name(), lib_provider, get_lsp_editing()); auto old_dep = dependencies_; - auto res = parse_inner(*analyzer_); + auto res = parse_inner(*opencode_analyzer_); if (!cancel_ || !*cancel_) { dependencies_.clear(); - for (auto& file : analyzer_->hlasm_ctx().get_visited_files()) + for (auto& file : opencode_analyzer_->hlasm_ctx().get_visited_files()) if (file != get_file_name()) dependencies_.insert(file); } @@ -64,6 +76,7 @@ parse_result processor_file_impl::parse(parse_lib_provider& lib_provider) files_to_close_.insert(file); } + last_analyzer_ = opencode_analyzer_.get(); return res; } @@ -71,10 +84,21 @@ parse_result processor_file_impl::parse(parse_lib_provider& lib_provider) parse_result processor_file_impl::parse_macro( parse_lib_provider& lib_provider, analyzing_context ctx, const library_data data) { - analyzer_ = - std::make_unique(get_text(), get_file_name(), std::move(ctx), lib_provider, data, get_lsp_editing()); + auto cache_key = macro_cache_key::create_from_context(*ctx.hlasm_ctx, data); + + if (macro_cache_.load_from_cache(cache_key, ctx)) + return true; + + auto a = std::make_unique(get_text(), get_file_name(), ctx, lib_provider, data, get_lsp_editing()); - return parse_inner(*analyzer_); + auto ret = parse_inner(*a); + + if (!ret) // Parsing was interrupted by cancellation token, do not save the result into cache + return false; + + last_analyzer_ = a.get(); + macro_cache_.save_analyzer(cache_key, std::move(a)); + return ret; } parse_result processor_file_impl::parse_no_lsp_update( @@ -86,25 +110,26 @@ parse_result processor_file_impl::parse_no_lsp_update( return true; } -bool processor_file_impl::parse_info_updated() -{ - bool ret = parse_info_updated_; - parse_info_updated_ = false; - return ret; -} - const std::set& processor_file_impl::dependencies() { return dependencies_; } const semantics::lines_info& processor_file_impl::get_hl_info() { - return analyzer_->source_processor().semantic_tokens(); + return last_analyzer_->source_processor().semantic_tokens(); } -const lsp::feature_provider& processor_file_impl::get_lsp_feature_provider() { return *analyzer_->context().lsp_ctx; } +const lsp::feature_provider& processor_file_impl::get_lsp_feature_provider() +{ + return *last_analyzer_->context().lsp_ctx; +} const std::set& processor_file_impl::files_to_close() { return files_to_close_; } -const performance_metrics& processor_file_impl::get_metrics() { return analyzer_->get_metrics(); } +const performance_metrics& processor_file_impl::get_metrics() { return last_analyzer_->get_metrics(); } + +void processor_file_impl::erase_cache_of_opencode(const std::string& opencode_file_name) +{ + macro_cache_.erase_cache_of_opencode(opencode_file_name); +} bool processor_file_impl::parse_inner(analyzer& new_analyzer) { @@ -114,10 +139,6 @@ bool processor_file_impl::parse_inner(analyzer& new_analyzer) collect_diags_from_child(new_analyzer); - // collect semantic info if the file is open in IDE - if (get_lsp_editing()) - parse_info_updated_ = true; - if (cancel_ && *cancel_) return false; return true; diff --git a/parser_library/src/workspaces/processor_file_impl.h b/parser_library/src/workspaces/processor_file_impl.h index 3dfe41944..fbb343b8b 100644 --- a/parser_library/src/workspaces/processor_file_impl.h +++ b/parser_library/src/workspaces/processor_file_impl.h @@ -17,19 +17,22 @@ #include "analyzer.h" #include "file_impl.h" +#include "macro_cache.h" #include "processor.h" namespace hlasm_plugin::parser_library::workspaces { +class file_manager; + // Implementation of the processor_file interface. Uses analyzer to parse the file // Then stores it until the next parsing so it is possible to retrieve parsing // information from it. class processor_file_impl : public virtual file_impl, public virtual processor_file { public: - processor_file_impl(std::string file_uri, std::atomic* cancel = nullptr); - processor_file_impl(file_impl&&, std::atomic* cancel = nullptr); - processor_file_impl(const file_impl& file, std::atomic* cancel = nullptr); + processor_file_impl(std::string file_uri, const file_manager& file_mngr, std::atomic* cancel = nullptr); + processor_file_impl(file_impl&&, const file_manager& file_mngr, std::atomic* cancel = nullptr); + processor_file_impl(const file_impl& file, const file_manager& file_mngr, std::atomic* cancel = nullptr); void collect_diags() const override; bool is_once_only() const override; // Starts parser with new (empty) context @@ -40,9 +43,6 @@ class processor_file_impl : public virtual file_impl, public virtual processor_f // Used by the macro tracer. parse_result parse_no_lsp_update(parse_lib_provider&, analyzing_context ctx, const library_data) override; - // Returns true if parsing occured since this method was called last. - bool parse_info_updated() override; - const std::set& dependencies() override; const semantics::lines_info& get_hl_info() override; @@ -50,16 +50,20 @@ class processor_file_impl : public virtual file_impl, public virtual processor_f const std::set& files_to_close() override; const performance_metrics& get_metrics() override; + void erase_cache_of_opencode(const std::string& opencode_file_name) override; + private: - std::unique_ptr analyzer_; + std::unique_ptr opencode_analyzer_; + const analyzer* last_analyzer_; bool parse_inner(analyzer&); - bool parse_info_updated_ = false; std::atomic* cancel_; std::set dependencies_; std::set files_to_close_; + + macro_cache macro_cache_; }; } // namespace hlasm_plugin::parser_library::workspaces diff --git a/parser_library/src/workspaces/workspace.cpp b/parser_library/src/workspaces/workspace.cpp index 1afd391d3..e26836cc1 100644 --- a/parser_library/src/workspaces/workspace.cpp +++ b/parser_library/src/workspaces/workspace.cpp @@ -252,6 +252,13 @@ void workspace::did_close_file(const std::string& file_uri) auto file = file_manager_.find_processor_file(*fname); // filter the dependencies that should not be closed filter_and_close_dependencies_(file->dependencies(), file); + // Erase macros cached for this opencode from all its dependencies + for (const std::string& dep_name : file->dependencies()) + { + auto proc_file = file_manager_.get_processor_file(dep_name); + if (proc_file) + proc_file->erase_cache_of_opencode(file_uri); + } // remove it from dependants dependants_.erase(fname); } diff --git a/parser_library/test/context/context_test.cpp b/parser_library/test/context/context_test.cpp index 6052b6d31..e8d1dbb77 100644 --- a/parser_library/test/context/context_test.cpp +++ b/parser_library/test/context/context_test.cpp @@ -260,7 +260,7 @@ TEST(context_macro, add_macro) args.push_back({ nullptr, op3 }); // prototype->|&LBL MAC &KEY=,&OP1,,&OP3 - auto& m = *ctx.add_macro(idx, lbl, move(args), {}, {}, {}, {}); + auto& m = *ctx.add_macro(idx, lbl, move(args), {}, {}, {}, {}, {}); EXPECT_EQ(m.named_params().size(), (size_t)4); EXPECT_NE(m.named_params().find(key), m.named_params().end()); @@ -292,7 +292,7 @@ TEST(context_macro, call_and_leave_macro) args.push_back({ nullptr, op3 }); // prototype->| MAC &KEY=,&OP1,,&OP3 - auto& m = *ctx.add_macro(idx, nullptr, move(args), {}, {}, {}, {}); + auto& m = *ctx.add_macro(idx, nullptr, move(args), {}, {}, {}, {}, {}); // creating param data macro_data_ptr p2(make_unique("ada")); @@ -354,7 +354,7 @@ TEST(context_macro, repeat_call_same_macro) args.push_back({ nullptr, op3 }); // prototype->|&LBL MAC &KEY=,&OP1,,&OP3 - ctx.add_macro(idx, lbl, move(args), {}, {}, {}, {}); + ctx.add_macro(idx, lbl, move(args), {}, {}, {}, {}, {}); // creating param data macro_data_ptr lb(make_unique("lbl")); @@ -448,7 +448,7 @@ TEST(context_macro, recurr_call) args.push_back({ nullptr, op3 }); // prototype->|&LBL MAC &KEY=,&OP1,,&OP3 - ctx.add_macro(idx, lbl, move(args), {}, {}, {}, {}); + ctx.add_macro(idx, lbl, move(args), {}, {}, {}, {}, {}); // creating param data macro_data_ptr lb(make_unique("lbl")); diff --git a/parser_library/test/copy_mock.h b/parser_library/test/copy_mock.h index e44f1194d..81b88e0db 100644 --- a/parser_library/test/copy_mock.h +++ b/parser_library/test/copy_mock.h @@ -52,6 +52,10 @@ class copy_mock : public workspaces::parse_lib_provider return &content_COPYND2; else if (library == "COPYBM") return &content_COPYBM; + else if (library == "EMPTY") + return &content_EMPTY; + else if (library == "COPYEMPTY") + return &content_COPYEMPTY; else return nullptr; } @@ -193,6 +197,8 @@ class copy_mock : public workspaces::parse_lib_provider LR 1 MEND )"; + const std::string content_EMPTY = ""; + const std::string content_COPYEMPTY = " COPY EMPTY"; }; } // namespace hlasm_plugin::parser_library diff --git a/parser_library/test/parsing/parser_model_test.cpp b/parser_library/test/parsing/parser_model_test.cpp index a46c2a7cf..c3b780085 100644 --- a/parser_library/test/parsing/parser_model_test.cpp +++ b/parser_library/test/parsing/parser_model_test.cpp @@ -20,14 +20,19 @@ // various invalid statements parsing // checking correct ranges -auto parse_model(std::string s, range r, bool after_substitution = false) +auto parse_model( + std::string s, + range r, + bool after_substitution = false, + const std::function& add_diag = [](diagnostic_op d) {}) { std::string input(" LR &var,1"); analyzer a(input); return a.parser().parse_operand_field(std::move(s), after_substitution, range_provider(r, adjusting_state::NONE), - std::make_pair(processing_format(processing_kind::ORDINARY, processing_form::MACH), op_code())); + std::make_pair(processing_format(processing_kind::ORDINARY, processing_form::MACH), op_code()), + add_diag); } TEST(parser, parse_model) @@ -135,9 +140,28 @@ TEST(parser, parse_model_with_apostrophe_escaping) TEST(parser, parse_bad_model) { + std::vector diags; range r(position(0, 4), position(0, 5)); - auto [op, rem] = parse_model("'", r, true); + auto [op, rem] = parse_model("'", r, true, [&diags](diagnostic_op diag) { diags.push_back(std::move(diag)); }); - ASSERT_EQ(op.value.size(), (size_t)0); - ASSERT_EQ(rem.value.size(), (size_t)0); + ASSERT_EQ(op.value.size(), 0U); + ASSERT_EQ(rem.value.size(), 0U); + + ASSERT_EQ(diags.size(), 1U); + EXPECT_EQ(diags[0].message, "While substituting to ''' => Unexpected end of statement"); + + range expected_range = { { 0, 5 }, { 0, 5 } }; + EXPECT_EQ(diags[0].diag_range, expected_range); +} + +TEST(parser, parse_bad_model_no_substitution) +{ + std::vector diags; + range r(position(0, 4), position(0, 5)); + auto [op, rem] = parse_model("'", r, false, [&diags](diagnostic_op diag) { diags.push_back(std::move(diag)); }); + + ASSERT_EQ(diags.size(), 1U); + EXPECT_EQ(diags[0].message, "Unexpected end of statement"); + range expected_range = { { 0, 5 }, { 0, 5 } }; + EXPECT_EQ(diags[0].diag_range, expected_range); } diff --git a/parser_library/test/processing/copy_test.cpp b/parser_library/test/processing/copy_test.cpp index a9b15b1de..80cc6362b 100644 --- a/parser_library/test/processing/copy_test.cpp +++ b/parser_library/test/processing/copy_test.cpp @@ -203,6 +203,9 @@ TEST(copy, copy_enter_from_macro_call) EXPECT_TRUE(mac->second->labels.find(a.hlasm_ctx().ids().add("A")) != mac->second->labels.end()); EXPECT_TRUE(mac->second->labels.find(a.hlasm_ctx().ids().add("B")) != mac->second->labels.end()); + ASSERT_EQ(mac->second->used_copy_members.size(), 1U); + EXPECT_EQ(mac->second->used_copy_members.begin()->get()->name, a.hlasm_ctx().ids().add("COPYR")); + ASSERT_EQ(a.diags().size(), (size_t)1); EXPECT_EQ(a.diags()[0].diag_range.start.line, (position_t)16); @@ -439,3 +442,43 @@ TEST(copy, copy_call_with_jump_before_comment) EXPECT_EQ(a.diags().size(), (size_t)0); EXPECT_EQ(a.parser().getNumberOfSyntaxErrors(), (size_t)0); } + +TEST(copy, copy_empty_file) +{ + std::string input = + R"( + MACRO + M + COPY COPYEMPTY + MEND + + MACRO + M2 + COPY EMPTY + MEND +)"; + copy_mock mock; + analyzer a(input, "start", mock); + a.analyze(); + + a.collect_diags(); + + EXPECT_EQ(a.hlasm_ctx().copy_members().size(), (size_t)2); + + EXPECT_EQ(a.hlasm_ctx().macros().size(), (size_t)2); + + auto mac = a.hlasm_ctx().get_macro_definition(a.hlasm_ctx().ids().add("M")); + ASSERT_TRUE(mac != nullptr); + + ASSERT_EQ(mac->used_copy_members.size(), 2U); + EXPECT_EQ(mac->used_copy_members.count(a.hlasm_ctx().get_copy_member(a.hlasm_ctx().ids().add("EMPTY"))), 1U); + EXPECT_EQ(mac->used_copy_members.count(a.hlasm_ctx().get_copy_member(a.hlasm_ctx().ids().add("COPYEMPTY"))), 1U); + + auto mac2 = a.hlasm_ctx().get_macro_definition(a.hlasm_ctx().ids().add("M2")); + ASSERT_TRUE(mac2 != nullptr); + + ASSERT_EQ(mac2->used_copy_members.size(), 1U); + EXPECT_EQ(mac2->used_copy_members.count(a.hlasm_ctx().get_copy_member(a.hlasm_ctx().ids().add("EMPTY"))), 1U); + + ASSERT_EQ(a.diags().size(), 0U); +} diff --git a/parser_library/test/workspace/CMakeLists.txt b/parser_library/test/workspace/CMakeLists.txt index ced5a74ac..c292d9314 100644 --- a/parser_library/test/workspace/CMakeLists.txt +++ b/parser_library/test/workspace/CMakeLists.txt @@ -14,7 +14,9 @@ target_sources(library_test PRIVATE diags_suppress_test.cpp empty_configs.h extension_handling_test.cpp + file_with_text.h load_config_test.cpp + macro_cache_test.cpp text_synchronization_test.cpp workspace_test.cpp ) diff --git a/parser_library/test/workspace/file_with_text.h b/parser_library/test/workspace/file_with_text.h new file mode 100644 index 000000000..f4a87909b --- /dev/null +++ b/parser_library/test/workspace/file_with_text.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2021 Broadcom. + * The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Broadcom, Inc. - initial API and implementation + */ +#ifndef TEST_WORKSPACE_WORKSPACE_FILE_WITH_TEXT_TEST_H +#define TEST_WORKSPACE_WORKSPACE_FILE_WITH_TEXT_TEST_H + +#include "workspaces/processor_file_impl.h" + +class file_with_text : public hlasm_plugin::parser_library::workspaces::processor_file_impl +{ +public: + file_with_text(const std::string& name, + const std::string& text, + const hlasm_plugin::parser_library::workspaces::file_manager& file_mngr) + : file_impl(name) + , processor_file_impl(name, file_mngr) + { + did_open(text, 1); + } + + virtual const std::string& get_text() override { return get_text_ref(); } + + virtual bool update_and_get_bad() override { return false; } +}; + +#endif diff --git a/parser_library/test/workspace/macro_cache_test.cpp b/parser_library/test/workspace/macro_cache_test.cpp new file mode 100644 index 000000000..8308fbff3 --- /dev/null +++ b/parser_library/test/workspace/macro_cache_test.cpp @@ -0,0 +1,381 @@ +/* + * Copyright (c) 2021 Broadcom. + * The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Broadcom, Inc. - initial API and implementation + */ + +#include +#include +#include + +#include "gtest/gtest.h" + +#include "analyzer.h" +#include "file_with_text.h" +#include "workspaces/file_manager_impl.h" +#include "workspaces/processor_file_impl.h" + + +using namespace hlasm_plugin::parser_library; +using namespace hlasm_plugin::parser_library::workspaces; + +namespace { + +struct file_manager_cache_test_mock : public file_manager_impl, public parse_lib_provider +{ + const static inline size_t lib_prefix_length = 4; + + std::unordered_map> files_by_fname_; + std::unordered_map, macro_cache>> files_by_library_; + + context::hlasm_ctx_ptr hlasm_ctx; + + auto& add_macro_or_copy(std::string file_name, std::string text) + { + auto file = std::make_shared(file_name, text, *this); + + auto [it, succ] = files_by_library_.emplace(file_name.substr(lib_prefix_length), + std::pair, macro_cache>( + std::piecewise_construct, std::forward_as_tuple(file), std::forward_as_tuple(*this, *file))); + files_by_fname_.emplace(std::move(file_name), file); + + return it->second; + } + + auto add_opencode(std::string file_name, std::string text) + { + auto file = std::make_shared(file_name, text, *this); + files_by_fname_.emplace(std::move(file_name), file); + return file; + } + + + file_ptr find(const std::string& key) const override + { + auto it = files_by_fname_.find(key); + return it == files_by_fname_.end() ? nullptr : it->second; + }; + + std::pair get_proc_file_from_library(const std::string& library) + { + auto it = files_by_library_.find(library); + if (it == files_by_library_.end()) + return { nullptr, nullptr }; + else + return { it->second.first, &it->second.second }; + }; + + workspaces::parse_result parse_library( + const std::string& library, analyzing_context ctx, const workspaces::library_data data) override + { + auto [m, cache] = get_proc_file_from_library(library); + auto a = std::make_unique(m->get_text(), m->get_file_name(), ctx, *this, data); + a->analyze(); + auto key = macro_cache_key::create_from_context(*ctx.hlasm_ctx, data); + cache->save_analyzer(key, std::move(a)); + hlasm_ctx = ctx.hlasm_ctx; + return true; + } + + bool has_library(const std::string& library, const std::string&) const override + { + return files_by_library_.count(library) > 0; + }; + + asm_option empty_options; + const asm_option& get_asm_options(const std::string&) override { return empty_options; }; +}; + +analyzing_context create_analyzing_context(std::string file_name, context::id_storage ids) +{ + analyzing_context new_ctx { std::make_shared( + std::move(file_name), asm_option(), std::move(ids)), + std::make_shared() }; + lsp::opencode_info_ptr oip = + std::make_unique(*new_ctx.hlasm_ctx, lsp::vardef_storage(), lsp::file_occurences_t {}); + new_ctx.lsp_ctx->add_opencode(std::move(oip), lsp::text_data_ref_t("")); + + return new_ctx; +} + +} // namespace + +TEST(macro_cache_test, copy_from_macro) +{ + std::string opencode_file_name = "opencode"; + std::string opencode_text = + R"( + MAC 1 + +)"; + std::string macro_file_name = "lib/MAC"; + std::string macro_text = + R"( MACRO + MAC &PARAM + COPY COPYFILE + MEND +)"; + std::string copyfile_file_name = "lib/COPYFILE"; + std::string copyfile_text = + R"( + LR 15,1 +)"; + + file_manager_cache_test_mock file_mngr; + // file_mngr.add_macro_or_copy() + auto opencode = file_mngr.add_opencode(opencode_file_name, opencode_text); + auto& [macro, macro_c] = file_mngr.add_macro_or_copy(macro_file_name, macro_text); + auto& [copyfile, copy_c] = file_mngr.add_macro_or_copy(copyfile_file_name, copyfile_text); + + + opencode->parse(file_mngr); + opencode->collect_diags(); + EXPECT_EQ(opencode->diags().size(), 0U); + + auto macro_id = file_mngr.hlasm_ctx->ids().add("MAC"); + auto copy_id = file_mngr.hlasm_ctx->ids().add("COPYFILE"); + + analyzing_context new_ctx = create_analyzing_context(opencode_file_name, file_mngr.hlasm_ctx->ids()); + + + macro_cache_key macro_key { opencode_file_name, { processing::processing_kind::MACRO, macro_id }, {} }; + + + EXPECT_TRUE(macro_c.load_from_cache(macro_key, new_ctx)); + + EXPECT_NE(new_ctx.hlasm_ctx->get_macro_definition(macro_id), nullptr); + EXPECT_NE(new_ctx.lsp_ctx->get_macro_info(macro_id), nullptr); + EXPECT_NE(new_ctx.hlasm_ctx->get_copy_member(copy_id), nullptr); + + + macro->did_change({}, " "); + + analyzing_context ctx_macro_changed = create_analyzing_context(opencode_file_name, new_ctx.hlasm_ctx->ids()); + + macro_cache_key copy_key { opencode_file_name, { processing::processing_kind::COPY, copy_id }, {} }; + // After macro change, copy should still be cached + EXPECT_TRUE(copy_c.load_from_cache(copy_key, ctx_macro_changed)); + EXPECT_NE(ctx_macro_changed.hlasm_ctx->get_copy_member(copy_id), nullptr); + EXPECT_FALSE(macro_c.load_from_cache(macro_key, ctx_macro_changed)); + + + // Reparse the change so everything is cached again + opencode->parse(file_mngr); + + copyfile->did_change({}, " "); + + analyzing_context ctx_copy_changed = create_analyzing_context(opencode_file_name, file_mngr.hlasm_ctx->ids()); + + // Macro depends on the copyfile, so none should be cached. + EXPECT_FALSE(macro_c.load_from_cache(macro_key, ctx_copy_changed)); + EXPECT_FALSE(copy_c.load_from_cache(copy_key, ctx_copy_changed)); +} + +TEST(macro_cache_test, opsyn_change) +{ + std::string opencode_file_name = "opencode"; + std::string opencode_text = + R"( +SETA OPSYN LR + MAC 1 +)"; + std::string macro_file_name = "lib/MAC"; + std::string macro_text = + R"( MACRO + MAC &PARAM + MEND +)"; + + file_manager_cache_test_mock file_mngr; + auto opencode = file_mngr.add_opencode(opencode_file_name, opencode_text); + auto& [macro, macro_c] = file_mngr.add_macro_or_copy(macro_file_name, macro_text); + + opencode->parse(file_mngr); + opencode->collect_diags(); + EXPECT_EQ(opencode->diags().size(), 0U); + + + auto macro_id = file_mngr.hlasm_ctx->ids().add("MAC"); + auto& ids = file_mngr.hlasm_ctx->ids(); + + macro_cache_key macro_key_one_opsyn { opencode_file_name, + { processing::processing_kind::MACRO, macro_id }, + { cached_opsyn_mnemo { ids.well_known.SETA, ids.add("LR"), false } } }; + + + analyzing_context new_ctx = create_analyzing_context(opencode_file_name, ids); + + + macro_cache_key macro_key { opencode_file_name, { processing::processing_kind::MACRO, macro_id }, {} }; + EXPECT_FALSE(macro_c.load_from_cache(macro_key, new_ctx)); + EXPECT_TRUE(macro_c.load_from_cache(macro_key_one_opsyn, new_ctx)); + + + + opencode->did_change({}, "L OPSYN SETB\n"); + opencode->parse(file_mngr); + + analyzing_context ctx_second_opsyn1 = create_analyzing_context(opencode_file_name, file_mngr.hlasm_ctx->ids()); + EXPECT_TRUE(macro_c.load_from_cache(macro_key_one_opsyn, ctx_second_opsyn1)); + + + macro_cache_key macro_key_two_opsyns = macro_key_one_opsyn; + macro_key_two_opsyns.opsyn_state.push_back( + cached_opsyn_mnemo { file_mngr.hlasm_ctx->ids().add("L"), file_mngr.hlasm_ctx->ids().well_known.SETB, false }); + + macro_cache_key::sort_opsyn_state(macro_key_two_opsyns.opsyn_state); + + analyzing_context ctx_second_opsyn2 = create_analyzing_context(opencode_file_name, file_mngr.hlasm_ctx->ids()); + EXPECT_TRUE(macro_c.load_from_cache(macro_key_two_opsyns, ctx_second_opsyn2)); +} + +TEST(macro_cache_test, empty_macro) +{ + std::string opencode_file_name = "opencode"; + // This tests a caveat where parse_library is called twice for the same macro, if the macro is not defined in its + // file. + std::string opencode_text = R"( + MAC + MAC)"; + std::string macro_file_name = "lib/MAC"; + std::string macro_text = ""; + + file_manager_cache_test_mock file_mngr; + auto opencode = file_mngr.add_opencode(opencode_file_name, opencode_text); + auto& [macro, macro_c] = file_mngr.add_macro_or_copy(macro_file_name, macro_text); + + opencode->parse(file_mngr); + + auto macro_id = file_mngr.hlasm_ctx->ids().add("MAC"); + + analyzing_context new_ctx = create_analyzing_context(opencode_file_name, file_mngr.hlasm_ctx->ids()); + + macro_cache_key macro_key { opencode_file_name, { processing::processing_kind::MACRO, macro_id }, {} }; + EXPECT_TRUE(macro_c.load_from_cache(macro_key, new_ctx)); + EXPECT_EQ(new_ctx.hlasm_ctx->macros().count(macro_id), 0U); +} + +TEST(macro_cache_test, get_opsyn_state) +{ + std::string opencode_file_name = "opencode"; + std::string opencode_text = + R"( +SETA OPSYN LR +L OPSYN SETB + MACRO + SETC + MEND + + MACRO + MAC + MEND +MAC OPSYN AREAD +)"; + + analyzer a(opencode_text); + a.analyze(); + auto state = macro_cache_key::get_opsyn_state(a.hlasm_ctx()); + + auto& ids = a.hlasm_ctx().ids(); + + std::vector expected { + { ids.well_known.SETA, ids.add("LR"), false }, + { ids.add("L"), ids.well_known.SETB, false }, + { ids.well_known.SETC, ids.well_known.SETC, true }, + { ids.add("MAC"), ids.add("AREAD"), false }, + }; + + macro_cache_key::sort_opsyn_state(expected); + + EXPECT_EQ(state, expected); +} + +TEST(macro_cache_test, overwrite_by_inline) +{ + std::string opencode_file_name = "opencode"; + std::string opencode_text = + R"( + MAC + + MACRO + MAC + LR 1,16 + MEND + + MAC +)"; + std::string macro_file_name = "lib/MAC"; + std::string macro_text = + R"( MACRO + MAC + LR 1,16 + MEND +)"; + + file_manager_cache_test_mock file_mngr; + auto opencode = file_mngr.add_opencode(opencode_file_name, opencode_text); + auto& [macro, macro_c] = file_mngr.add_macro_or_copy(macro_file_name, macro_text); + + opencode->parse(file_mngr); + opencode->collect_diags(); + EXPECT_EQ(opencode->diags().size(), 2U); + + + auto macro_diag = std::find_if(opencode->diags().begin(), opencode->diags().end(), [&](const diagnostic_s& d) { + return d.file_name == macro_file_name; + }); + EXPECT_NE(macro_diag, opencode->diags().end()); + + auto opencode_diag = std::find_if(opencode->diags().begin(), opencode->diags().end(), [&](const diagnostic_s& d) { + return d.file_name == opencode_file_name; + }); + EXPECT_NE(opencode_diag, opencode->diags().end()); +} + +TEST(macro_cache_test, inline_depends_on_copy) +{ + std::string opencode_file_name = "opencode"; + std::string opencode_text = + R"( + MACRO + MAC + COPY COPYFILE + MEND + + MAC +)"; + std::string copy_file_name = "lib/COPYFILE"; + std::string copy_text = R"( LR 1,1 arbitrary instruction)"; + + file_manager_cache_test_mock file_mngr; + auto opencode = file_mngr.add_opencode(opencode_file_name, opencode_text); + auto& [copyfile, copy_c] = file_mngr.add_macro_or_copy(copy_file_name, copy_text); + + opencode->parse(file_mngr); + opencode->collect_diags(); + EXPECT_EQ(opencode->diags().size(), 0U); + + + auto copy_id = file_mngr.hlasm_ctx->ids().add("COPYFILE"); + auto& ids = file_mngr.hlasm_ctx->ids(); + + analyzing_context new_ctx = create_analyzing_context(opencode_file_name, ids); + + + macro_cache_key copy_key { opencode_file_name, { processing::processing_kind::COPY, copy_id }, {} }; + EXPECT_TRUE(copy_c.load_from_cache(copy_key, new_ctx)); + + copyfile->did_change({ { 0, 4 }, { 0, 5 } }, "16"); + opencode->parse(file_mngr); + opencode->collect_diags(); + ASSERT_EQ(opencode->diags().size(), 1U); + EXPECT_EQ(opencode->diags()[0].code, "M120"); +} diff --git a/parser_library/test/workspace/workspace_test.cpp b/parser_library/test/workspace/workspace_test.cpp index 9c971f792..835333d7b 100644 --- a/parser_library/test/workspace/workspace_test.cpp +++ b/parser_library/test/workspace/workspace_test.cpp @@ -19,6 +19,7 @@ #include "gtest/gtest.h" +#include "file_with_text.h" #include "utils/path.h" #include "utils/platform.h" #include "workspaces/file_impl.h" @@ -230,21 +231,6 @@ label std::string source_using_macro_file_no_error = R"( CORRECT)"; -class file_with_text : public processor_file_impl -{ -public: - file_with_text(const std::string& name, const std::string& text) - : file_impl(name) - , processor_file_impl(name) - { - did_open(text, 1); - } - - const std::string& get_text() override { return get_text_ref(); } - - bool update_and_get_bad() override { return false; } -}; - const char* faulty_macro_path = is_windows() ? "lib\\ERROR" : "lib/ERROR"; const char* correct_macro_path = is_windows() ? "lib\\CORRECT" : "lib/CORRECT"; std::string hlasmplugin_folder = is_windows() ? ".hlasmplugin\\" : ".hlasmplugin/"; @@ -254,15 +240,17 @@ class file_manager_extended : public file_manager_impl public: file_manager_extended() { + files_.emplace(hlasmplugin_folder + "proc_grps.json", + std::make_unique("proc_grps.json", pgroups_file, *this)); + files_.emplace(hlasmplugin_folder + "pgm_conf.json", + std::make_unique("pgm_conf.json", pgmconf_file, *this)); + files_.emplace("source1", std::make_unique("source1", source_using_macro_file, *this)); + files_.emplace("source2", std::make_unique("source2", source_using_macro_file, *this)); + files_.emplace("source3", std::make_unique("source3", source_using_macro_file_no_error, *this)); files_.emplace( - hlasmplugin_folder + "proc_grps.json", std::make_unique("proc_grps.json", pgroups_file)); + faulty_macro_path, std::make_unique(faulty_macro_path, faulty_macro_file, *this)); files_.emplace( - hlasmplugin_folder + "pgm_conf.json", std::make_unique("pgm_conf.json", pgmconf_file)); - files_.emplace("source1", std::make_unique("source1", source_using_macro_file)); - files_.emplace("source2", std::make_unique("source2", source_using_macro_file)); - files_.emplace("source3", std::make_unique("source3", source_using_macro_file_no_error)); - files_.emplace(faulty_macro_path, std::make_unique(faulty_macro_path, faulty_macro_file)); - files_.emplace(correct_macro_path, std::make_unique(correct_macro_path, correct_macro_file)); + correct_macro_path, std::make_unique(correct_macro_path, correct_macro_file, *this)); } list_directory_result list_directory_files(const std::string&) override @@ -292,15 +280,16 @@ class file_manager_opt : public file_manager_impl switch (variant) { case file_manager_opt_variant::old_school: - return std::make_unique("proc_grps.json", pgroups_file_old_school); + return std::make_unique("proc_grps.json", pgroups_file_old_school, *this); case file_manager_opt_variant::default_to_required: - return std::make_unique("proc_grps.json", pgroups_file_default); + return std::make_unique("proc_grps.json", pgroups_file_default, *this); case file_manager_opt_variant::required: - return std::make_unique("proc_grps.json", pgroups_file_required); + return std::make_unique("proc_grps.json", pgroups_file_required, *this); case file_manager_opt_variant::optional: - return std::make_unique("proc_grps.json", pgroups_file_optional); + return std::make_unique("proc_grps.json", pgroups_file_optional, *this); case file_manager_opt_variant::invalid_assembler_options: - return std::make_unique("proc_grps.json", pgroups_file_invalid_assembler_options); + return std::make_unique( + "proc_grps.json", pgroups_file_invalid_assembler_options, *this); } throw std::logic_error("Not implemented"); } @@ -309,10 +298,11 @@ class file_manager_opt : public file_manager_impl file_manager_opt(file_manager_opt_variant variant) { files_.emplace(hlasmplugin_folder + "proc_grps.json", generate_proc_grps_file(variant)); + files_.emplace(hlasmplugin_folder + "pgm_conf.json", + std::make_unique("pgm_conf.json", pgmconf_file, *this)); + files_.emplace("source1", std::make_unique("source1", source_using_macro_file_no_error, *this)); files_.emplace( - hlasmplugin_folder + "pgm_conf.json", std::make_unique("pgm_conf.json", pgmconf_file)); - files_.emplace("source1", std::make_unique("source1", source_using_macro_file_no_error)); - files_.emplace(correct_macro_path, std::make_unique(correct_macro_path, correct_macro_file)); + correct_macro_path, std::make_unique(correct_macro_path, correct_macro_file, *this)); } list_directory_result list_directory_files(const std::string& path) override