refactor: Remove highlighting symbol production from lexer

language_server/test/lsp/feature_language_features_test.cpp

@@ -243,7 +243,7 @@ IIIIIIIIIIIIIII1
             0,2,3,1,0,    // instruction   EQU
             0,68,1,10,0,  // number        1
             0,1,1,5,0,    // continuation  X
-            0,1,5,3,0,    // ignored       3145\n
+            0,1,4,3,0,    // ignored       3145
             1,0,15,3,0,   // ignored       IIIIIIIIIIIIIII
             0,15,1,10,0   // number        1
         } } };

parser_library/src/debugging/debugger.cpp

@@ -698,7 +698,7 @@ class debugger::impl final : public processing::statement_analyzer, output_handl
         std::string error_msg;
         error_collector diags(error_msg);
 
-        auto p = parsing::parser_holder::create(nullptr, ctx_, nullptr, false);
+        auto p = parsing::parser_holder::create(ctx_, nullptr, false);
         p->prepare_parser(expr, ctx_, &diags, semantics::range_provider(), range(), 1, status, true);
 
         semantics::operand_ptr op =

parser_library/src/expressions/conditional_assembly/terms/ca_symbol_attribute.cpp

@@ -461,7 +461,7 @@ semantics::literal_si ca_symbol_attribute::reparse_substituted_literal(
         diag.message = diagnostic_decorate_message(text, diag.message);
         eval_ctx.diags.add_diagnostic(std::move(diag));
     });
-    auto h = parsing::parser_holder::create(nullptr, &eval_ctx.hlasm_ctx, &add_diag_subst, false);
+    auto h = parsing::parser_holder::create(&eval_ctx.hlasm_ctx, &add_diag_subst, false);
 
     h->prepare_parser(text,
         &eval_ctx.hlasm_ctx,

parser_library/src/lexing/lexer.cpp

@@ -30,9 +30,8 @@ using namespace hlasm_plugin;
 using namespace parser_library;
 using namespace lexing;
 
-lexer::lexer(input_source* input, semantics::source_info_processor* lsp_proc)
+lexer::lexer(input_source* input)
     : input_(input)
-    , src_proc_(lsp_proc)
 {
     file_input_state_.input = input;
 
@@ -100,24 +99,6 @@ void lexer::create_token(size_t ttype, size_t channel)
         end.char_position_in_line_utf16);
 
     ++last_token_id_;
-
-    if (src_proc_)
-        switch (ttype)
-        {
-            case CONTINUATION:
-                src_proc_->add_hl_symbol(
-                    token_info(range(position(token_start_state_.line, token_start_state_.char_position_in_line_utf16),
-                                   position(input_state_->line, end.char_position_in_line_utf16)),
-                        hl_scopes::continuation));
-                break;
-            case IGNORED:
-                src_proc_->add_hl_symbol(
-                    token_info(range(position(token_start_state_.line, token_start_state_.char_position_in_line_utf16),
-                                   position(token_start_state_.line, end.char_position_in_line_utf16)),
-                        hl_scopes::ignored));
-                break;
-                // case COMMENT: Line comments are already handled in opencode_provider::process_comment()
-        }
 }
 
 void lexer::consume()

parser_library/src/lexing/lexer.h

@@ -19,7 +19,6 @@
 #include <vector>
 
 #include "range.h"
-#include "semantics/source_info_processor.h"
 #include "token.h"
 
 namespace hlasm_plugin::parser_library::lexing {
@@ -34,7 +33,7 @@ class lexer final
         size_t line;
         size_t offset;
     };
-    lexer(input_source*, semantics::source_info_processor* lsp_proc);
+    explicit lexer(input_source*);
 
     lexer(const lexer&) = delete;
     lexer(lexer&&) = delete;
@@ -107,7 +106,6 @@ class lexer final
     size_t continue_ = 15;
 
     input_source* input_;
-    semantics::source_info_processor* src_proc_;
 
     struct input_state
     {

parser_library/src/parsing/parser_impl.cpp

@@ -69,12 +69,11 @@ template<bool multiline>
 struct parser_holder_impl final : parser_holder
 {
     using parser_t = std::conditional_t<multiline, hlasmparser_multiline, hlasmparser_singleline>;
-    parser_holder_impl(
-        semantics::source_info_processor* lsp_proc, context::hlasm_context* hl_ctx, diagnostic_op_consumer* d)
+    parser_holder_impl(context::hlasm_context* hl_ctx, diagnostic_op_consumer* d)
     {
         error_handler = std::make_shared<parsing::error_strategy>();
         input = std::make_unique<lexing::input_source>();
-        lex = std::make_unique<lexing::lexer>(input.get(), lsp_proc);
+        lex = std::make_unique<lexing::lexer>(input.get());
         stream = std::make_unique<lexing::token_stream>(lex.get());
         parser = std::make_unique<parser_t>(stream.get());
         parser->setErrorHandler(error_handler);
@@ -129,15 +128,13 @@ struct parser_holder_impl final : parser_holder
     semantics::literal_si literal_reparse() const override { return std::move(get_parser().literal_reparse()->value); }
 };
 
-std::unique_ptr<parser_holder> parser_holder::create(semantics::source_info_processor* lsp_proc,
-    context::hlasm_context* hl_ctx,
-    diagnostic_op_consumer* d,
-    bool multiline)
+std::unique_ptr<parser_holder> parser_holder::create(
+    context::hlasm_context* hl_ctx, diagnostic_op_consumer* d, bool multiline)
 {
     if (multiline)
-        return std::make_unique<parser_holder_impl<true>>(lsp_proc, hl_ctx, d);
+        return std::make_unique<parser_holder_impl<true>>(hl_ctx, d);
     else
-        return std::make_unique<parser_holder_impl<false>>(lsp_proc, hl_ctx, d);
+        return std::make_unique<parser_holder_impl<false>>(hl_ctx, d);
 }
 
 void parser_impl::enable_lookahead_recovery()

parser_library/src/parsing/parser_impl.h

@@ -20,7 +20,6 @@
 #include "Parser.h"
 #include "parser_error_listener.h"
 #include "semantics/collector.h"
-#include "semantics/source_info_processor.h"
 
 namespace hlasm_plugin::parser_library::context {
 class hlasm_context;
@@ -245,10 +244,8 @@ struct parser_holder
         const processing::processing_status& proc_status,
         bool unlimited_line) const;
 
-    static std::unique_ptr<parser_holder> create(semantics::source_info_processor* lsp_proc,
-        context::hlasm_context* hl_ctx,
-        diagnostic_op_consumer* d,
-        bool multiline);
+    static std::unique_ptr<parser_holder> create(
+        context::hlasm_context* hl_ctx, diagnostic_op_consumer* d, bool multiline);
 };
 
 } // namespace hlasm_plugin::parser_library::parsing

parser_library/src/processing/opencode_provider.cpp

@@ -48,12 +48,12 @@ opencode_provider::opencode_provider(std::string_view text,
     : statement_provider(statement_provider_kind::OPEN)
     , m_input_document(text)
     , m_virtual_files(std::make_shared<std::unordered_map<context::id_index, std::string>>())
-    , m_singleline { parsing::parser_holder::create(&src_proc, ctx.hlasm_ctx.get(), &diag_consumer, false),
-        parsing::parser_holder::create(nullptr, ctx.hlasm_ctx.get(), nullptr, false),
-        parsing::parser_holder::create(nullptr, ctx.hlasm_ctx.get(), nullptr, false) }
-    , m_multiline { parsing::parser_holder::create(&src_proc, ctx.hlasm_ctx.get(), &diag_consumer, true),
-        parsing::parser_holder::create(nullptr, ctx.hlasm_ctx.get(), nullptr, true),
-        parsing::parser_holder::create(nullptr, ctx.hlasm_ctx.get(), nullptr, true) }
+    , m_singleline { parsing::parser_holder::create(ctx.hlasm_ctx.get(), &diag_consumer, false),
+        parsing::parser_holder::create(ctx.hlasm_ctx.get(), nullptr, false),
+        parsing::parser_holder::create(ctx.hlasm_ctx.get(), nullptr, false) }
+    , m_multiline { parsing::parser_holder::create(ctx.hlasm_ctx.get(), &diag_consumer, true),
+        parsing::parser_holder::create(ctx.hlasm_ctx.get(), nullptr, true),
+        parsing::parser_holder::create(ctx.hlasm_ctx.get(), nullptr, true) }
     , m_ctx(ctx)
     , m_lib_provider(&lib_provider)
     , m_state_listener(&state_listener)
@@ -193,22 +193,48 @@ void opencode_provider::ainsert(const std::string& rec, ainsert_destination dest
     suspend_copy_processing(remove_empty::no);
 }
 
-void opencode_provider::feed_line(const parsing::parser_holder& p, bool is_process)
+namespace {
+void produce_hl_symbols(
+    const lexing::logical_line<utils::utf8_iterator<std::string_view::iterator, utils::utf8_utf16_counter>>& ll,
+    size_t lineno,
+    semantics::source_info_processor& sip)
+{
+    static constexpr auto ll_range = [](size_t lno, const auto& b, const auto& m, const auto& e) {
+        return range({ lno, lexing::logical_distance(b, m) }, { lno, lexing::logical_distance(b, e) });
+    };
+
+    for (size_t offset = 0; const auto& l : ll.segments)
+    {
+        const auto ln = lineno + offset++;
+        if (l.begin != l.code)
+            sip.add_hl_symbol({ ll_range(ln, l.begin, l.begin, l.code), hl_scopes::ignored });
+
+        if (l.continuation != l.ignore)
+            sip.add_hl_symbol({ ll_range(ln, l.begin, l.continuation, l.ignore), hl_scopes::continuation });
+
+        if (l.ignore != l.end)
+            sip.add_hl_symbol({ ll_range(ln, l.begin, l.ignore, l.end), hl_scopes::ignored });
+    }
+}
+} // namespace
+
+void opencode_provider::feed_line(const parsing::parser_holder& p, bool is_process, bool produce_source_info)
 {
     m_line_fed = true;
 
+    const auto lineno = m_current_logical_line_source.begin_line;
+    if (produce_source_info)
+        produce_hl_symbols(m_current_logical_line, lineno, *m_src_proc);
+
     const auto& subs = p.input->new_input(m_current_logical_line);
 
     if (subs.server && !std::exchange(m_encoding_warning_issued.server, true))
-        m_diagnoser->add_diagnostic(
-            diagnostic_op::warning_W017(range(position(m_current_logical_line_source.begin_line, 0))));
+        m_diagnoser->add_diagnostic(diagnostic_op::warning_W017(range(position(lineno, 0))));
 
     if (subs.client && !std::exchange(m_encoding_warning_issued.client, true))
-        m_diagnoser->add_diagnostic(
-            diagnostic_op::warning_W018(range(position(m_current_logical_line_source.begin_line, 0))));
+        m_diagnoser->add_diagnostic(diagnostic_op::warning_W018(range(position(lineno, 0))));
 
-    p.lex->set_file_offset(
-        { m_current_logical_line_source.begin_line, 0 /*lexing::default_ictl.begin-1 really*/ }, 0, is_process);
+    p.lex->set_file_offset({ lineno, 0 /*lexing::default_ictl.begin-1 really*/ }, 0, is_process);
     p.lex->set_unlimited_line(false);
     p.lex->reset();
 
@@ -663,7 +689,7 @@ context::shared_stmt_ptr opencode_provider::get_next(const statement_processor&
 
     auto& ph = lookahead ? (multiline ? *m_multiline.m_lookahead_parser : *m_singleline.m_lookahead_parser)
                          : (multiline ? *m_multiline.m_parser : *m_singleline.m_parser);
-    feed_line(ph, is_process);
+    feed_line(ph, is_process, !lookahead);
 
     auto& collector = ph.parser->get_collector();
     auto* diag_target = nested ? collector.diag_collector() : static_cast<diagnostic_op_consumer*>(m_diagnoser);
@@ -757,7 +783,7 @@ parsing::hlasmparser_multiline& opencode_provider::parser()
     if (!m_line_fed)
     {
         auto ll_res = extract_next_logical_line();
-        feed_line(*m_multiline.m_parser, ll_res == extract_next_logical_line_result::process);
+        feed_line(*m_multiline.m_parser, ll_res == extract_next_logical_line_result::process, true);
     }
     assert(m_line_fed);
     return static_cast<parsing::hlasmparser_multiline&>(*m_multiline.m_parser->parser);

parser_library/src/processing/opencode_provider.h

@@ -194,7 +194,7 @@ class opencode_provider final : public statement_provider, virtual_file_monitor
     void onetime_action();
 
 private:
-    void feed_line(const parsing::parser_holder& p, bool is_process);
+    void feed_line(const parsing::parser_holder& p, bool is_process, bool produce_source_info);
     bool is_comment();
     void process_comment();
     void generate_aread_highlighting(std::string_view text, size_t line_no) const;

parser_library/src/processing/statement_fields_parser.cpp

@@ -23,8 +23,8 @@
 namespace hlasm_plugin::parser_library::processing {
 
 statement_fields_parser::statement_fields_parser(context::hlasm_context* hlasm_ctx)
-    : m_parser_singleline(parsing::parser_holder::create(nullptr, hlasm_ctx, nullptr, false))
-    , m_parser_multiline(parsing::parser_holder::create(nullptr, hlasm_ctx, nullptr, true))
+    : m_parser_singleline(parsing::parser_holder::create(hlasm_ctx, nullptr, false))
+    , m_parser_multiline(parsing::parser_holder::create(hlasm_ctx, nullptr, true))
     , m_hlasm_ctx(hlasm_ctx)
 {}
 

parser_library/test/lexing/lexer_test.cpp

@@ -45,9 +45,8 @@ ORDSYMBOL
 EOF
 )";
 
-    semantics::source_info_processor src_proc(false);
     lexing::input_source input("TEST TEST \r\n TEST1 TEST2");
-    lexing::lexer l(&input, &src_proc);
+    lexing::lexer l(&input);
     lexing::token_stream tokens(&l);
     parser parser(&tokens);
 
@@ -109,9 +108,8 @@ ORDSYMBOL
 EOF
 )";
 
-    semantics::source_info_processor src_proc(false);
     lexing::input_source input(in);
-    lexing::lexer l(&input, &src_proc);
+    lexing::lexer l(&input);
     lexing::token_stream tokens(&l);
     parser parser(&tokens);
     l.set_unlimited_line(true);
@@ -130,8 +128,7 @@ TEST(lexer_test, special_spaces)
 {
     std::string in = "A\v\f\t LR";
     lexing::input_source input(in);
-    semantics::source_info_processor src_proc(false);
-    lexing::lexer l(&input, &src_proc);
+    lexing::lexer l(&input);
 
     while (l.more_tokens())
         ;
@@ -151,9 +148,8 @@ TEST(lexer_test, attribute_in_continuation)
                'SYMBOL
 )";
 
-    semantics::source_info_processor src_proc(false);
     lexing::input_source input(in);
-    lexing::lexer l(&input, &src_proc);
+    lexing::lexer l(&input);
 
     while (l.more_tokens())
         ;

parser_library/test/semantics/highlighting_test.cpp

@@ -86,7 +86,6 @@ TEST(highlighting, mach_noop_multiline)
         token_info({ { 1, 4 }, { 1, 9 } }, hl_scopes::instruction),
         token_info({ { 1, 10 }, { 1, 71 } }, hl_scopes::remark),
         token_info({ { 1, 71 }, { 1, 72 } }, hl_scopes::continuation),
-        token_info({ { 1, 72 }, { 1, 73 } }, hl_scopes::ignored),
         token_info({ { 2, 0 }, { 2, 15 } }, hl_scopes::ignored),
         token_info({ { 2, 15 }, { 2, 22 } }, hl_scopes::remark),
     };
@@ -220,7 +219,7 @@ IgnoredIgnoredI1 remark)";
         token_info({ { 0, 2 }, { 0, 5 } }, hl_scopes::instruction),
         token_info({ { 0, 70 }, { 0, 71 } }, hl_scopes::number),
         token_info({ { 0, 71 }, { 0, 72 } }, hl_scopes::continuation),
-        token_info({ { 0, 72 }, { 0, 80 } }, hl_scopes::ignored),
+        token_info({ { 0, 72 }, { 0, 79 } }, hl_scopes::ignored),
         token_info({ { 1, 0 }, { 1, 15 } }, hl_scopes::ignored),
         token_info({ { 1, 15 }, { 1, 16 } }, hl_scopes::number),
         token_info({ { 1, 17 }, { 1, 23 } }, hl_scopes::remark) };
@@ -247,7 +246,6 @@ TEST(highlighting, macro_alternative_continuation)
         token_info({ { 4, 8 }, { 4, 9 } }, hl_scopes::operator_symbol),
         token_info({ { 4, 10 }, { 4, 71 } }, hl_scopes::remark),
         token_info({ { 4, 71 }, { 4, 72 } }, hl_scopes::continuation),
-        token_info({ { 4, 72 }, { 4, 73 } }, hl_scopes::ignored),
         token_info({ { 5, 0 }, { 5, 15 } }, hl_scopes::ignored),
         token_info({ { 5, 15 }, { 5, 18 } }, hl_scopes::operand),
         token_info({ { 5, 19 }, { 5, 26 } }, hl_scopes::remark) };
@@ -402,23 +400,20 @@ TEST(highlighting, multiline_macro_param)
         token_info({ { 5, 39 }, { 5, 71 } }, hl_scopes::remark),
 
         token_info({ { 5, 71 }, { 5, 72 } }, hl_scopes::continuation),
-        token_info({ { 5, 72 }, { 5, 73 } }, hl_scopes::ignored),
         token_info({ { 6, 0 }, { 6, 15 } }, hl_scopes::ignored),
 
         token_info({ { 6, 15 }, { 6, 17 } }, hl_scopes::operand),
         token_info({ { 6, 17 }, { 6, 18 } }, hl_scopes::operator_symbol),
         token_info({ { 6, 39 }, { 6, 71 } }, hl_scopes::remark),
 
         token_info({ { 6, 71 }, { 6, 72 } }, hl_scopes::continuation),
-        token_info({ { 6, 72 }, { 6, 73 } }, hl_scopes::ignored),
         token_info({ { 7, 0 }, { 7, 15 } }, hl_scopes::ignored),
 
         token_info({ { 7, 15 }, { 7, 17 } }, hl_scopes::operand),
         token_info({ { 7, 17 }, { 7, 18 } }, hl_scopes::operator_symbol),
         token_info({ { 7, 39 }, { 7, 71 } }, hl_scopes::remark),
 
         token_info({ { 7, 71 }, { 7, 72 } }, hl_scopes::continuation),
-        token_info({ { 7, 72 }, { 7, 73 } }, hl_scopes::ignored),
         token_info({ { 8, 0 }, { 8, 15 } }, hl_scopes::ignored),
 
         token_info({ { 8, 15 }, { 8, 17 } }, hl_scopes::operand),
@@ -457,21 +452,18 @@ INSTANTIATE_TEST_SUITE_P(highlighting,
                 token_info({ { 1, 17 }, { 1, 18 } }, hl_scopes::operator_symbol),
                 token_info({ { 1, 39 }, { 1, 71 } }, hl_scopes::remark),
                 token_info({ { 1, 71 }, { 1, 72 } }, hl_scopes::continuation),
-                token_info({ { 1, 72 }, { 1, 73 } }, hl_scopes::ignored),
 
                 token_info({ { 2, 0 }, { 2, 15 } }, hl_scopes::ignored),
                 token_info({ { 2, 15 }, { 2, 17 } }, hl_scopes::operand),
                 token_info({ { 2, 17 }, { 2, 18 } }, hl_scopes::operator_symbol),
                 token_info({ { 2, 39 }, { 2, 71 } }, hl_scopes::remark),
                 token_info({ { 2, 71 }, { 2, 72 } }, hl_scopes::continuation),
-                token_info({ { 2, 72 }, { 2, 73 } }, hl_scopes::ignored),
 
                 token_info({ { 3, 0 }, { 3, 15 } }, hl_scopes::ignored),
                 token_info({ { 3, 15 }, { 3, 17 } }, hl_scopes::operand),
                 token_info({ { 3, 17 }, { 3, 18 } }, hl_scopes::operator_symbol),
                 token_info({ { 3, 39 }, { 3, 71 } }, hl_scopes::remark),
                 token_info({ { 3, 71 }, { 3, 72 } }, hl_scopes::continuation),
-                token_info({ { 3, 72 }, { 3, 73 } }, hl_scopes::ignored),
 
                 token_info({ { 4, 0 }, { 4, 15 } }, hl_scopes::ignored),
                 token_info({ { 4, 15 }, { 4, 17 } }, hl_scopes::operand),
@@ -490,7 +482,6 @@ INSTANTIATE_TEST_SUITE_P(highlighting,
                 token_info({ { 1, 14 }, { 1, 15 } }, hl_scopes::operator_symbol),
                 token_info({ { 1, 15 }, { 1, 16 } }, hl_scopes::operand),
                 token_info({ { 1, 71 }, { 1, 72 } }, hl_scopes::continuation),
-                token_info({ { 1, 72 }, { 1, 73 } }, hl_scopes::ignored),
 
                 token_info({ { 2, 0 }, { 2, 15 } }, hl_scopes::ignored),
                 token_info({ { 2, 15 }, { 2, 18 } }, hl_scopes::remark),