From d14e56a65d3aae34c03a82b1e59c4e1632fb3ae1 Mon Sep 17 00:00:00 2001 From: tompng Date: Sun, 7 Aug 2022 20:58:17 +0900 Subject: [PATCH 1/3] Scan every single characters in IRB::Color.scan --- lib/irb/color.rb | 55 +++++++++++++++++++++--------------------- test/irb/test_color.rb | 2 ++ 2 files changed, 30 insertions(+), 27 deletions(-) diff --git a/lib/irb/color.rb b/lib/irb/color.rb index 8307af25a..401966d18 100644 --- a/lib/irb/color.rb +++ b/lib/irb/color.rb @@ -128,10 +128,14 @@ def colorize_code(code, complete: true, ignore_error: false, colorable: colorabl symbol_state = SymbolState.new colored = +'' - length = 0 - end_seen = false scan(code, allow_last_error: !complete) do |token, str, expr| + # handle uncolorable code + if token.nil? + colored << Reline::Unicode.escape_for_print(str) + next + end + # IRB::ColorPrinter skips colorizing fragments with any invalid token if ignore_error && ERROR_TOKENS.include?(token) return Reline::Unicode.escape_for_print(code) @@ -147,15 +151,7 @@ def colorize_code(code, complete: true, ignore_error: false, colorable: colorabl colored << line end end - length += str.bytesize - end_seen = true if token == :on___end__ - end - - # give up colorizing incomplete Ripper tokens - unless end_seen or length == code.bytesize - return Reline::Unicode.escape_for_print(code) end - colored end @@ -170,33 +166,38 @@ def without_circular_ref(obj, seen:, &block) end def scan(code, allow_last_error:) - pos = [1, 0] - verbose, $VERBOSE = $VERBOSE, nil RubyLex.compile_with_errors_suppressed(code) do |inner_code, line_no| lexer = Ripper::Lexer.new(inner_code, '(ripper)', line_no) + byte_pos = 0 + line_positions = [0] + inner_code.lines.each do |line| + line_positions << line_positions.last + line.bytesize + end + + on_scan = proc do |elem| + str = elem.tok + start_pos = line_positions[elem.pos[0] - 1] + elem.pos[1] + end_pos = start_pos + str.bytesize + next if start_pos < byte_pos + + yield(nil, inner_code.byteslice(byte_pos...start_pos), nil) if byte_pos < start_pos + yield(elem.event, str, elem.state) + byte_pos = end_pos + end + if lexer.respond_to?(:scan) # Ruby 2.7+ lexer.scan.each do |elem| - str = elem.tok next if allow_last_error and /meets end of file|unexpected end-of-input/ =~ elem.message - next if ([elem.pos[0], elem.pos[1] + str.bytesize] <=> pos) <= 0 - - str.each_line do |line| - if line.end_with?("\n") - pos[0] += 1 - pos[1] = 0 - else - pos[1] += line.bytesize - end - end - - yield(elem.event, str, elem.state) + on_scan.call(elem) end else - lexer.parse.each do |elem| - yield(elem.event, elem.tok, elem.state) + lexer.parse.sort_by(&:pos).each do |elem| + on_scan.call(elem) end end + # yield uncolorable DATA section + yield(nil, inner_code.byteslice(byte_pos...inner_code.bytesize), nil) if byte_pos < inner_code.bytesize end ensure $VERBOSE = verbose diff --git a/test/irb/test_color.rb b/test/irb/test_color.rb index 6ad64a0ae..73e9b389c 100644 --- a/test/irb/test_color.rb +++ b/test/irb/test_color.rb @@ -88,6 +88,8 @@ def test_colorize_code "foo(*%W(bar))" => "foo(*#{RED}#{BOLD}%W(#{CLEAR}#{RED}bar#{CLEAR}#{RED}#{BOLD})#{CLEAR})", "$stdout" => "#{GREEN}#{BOLD}$stdout#{CLEAR}", "__END__" => "#{GREEN}__END__#{CLEAR}", + "foo\n__END__\nbar" => "foo\n#{GREEN}__END__#{CLEAR}\nbar", + "foo\n< "foo\n#{RED}< Date: Thu, 11 Aug 2022 06:15:32 +0900 Subject: [PATCH 2/3] Update expected colorize result that were uncolored before --- test/irb/test_color.rb | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/test/irb/test_color.rb b/test/irb/test_color.rb index 73e9b389c..dc394f9d6 100644 --- a/test/irb/test_color.rb +++ b/test/irb/test_color.rb @@ -90,6 +90,7 @@ def test_colorize_code "__END__" => "#{GREEN}__END__#{CLEAR}", "foo\n__END__\nbar" => "foo\n#{GREEN}__END__#{CLEAR}\nbar", "foo\n< "foo\n#{RED}< "#{RED}< "#{GREEN}def#{CLEAR} #{BLUE}#{BOLD}req#{CLEAR}(#{RED}#{REVERSE}@a#{CLEAR}) #{GREEN}end#{CLEAR}", }) else - tests.merge!({ - "[1]]]\u0013" => "[1]]]^S", + if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('2.7.0') + tests.merge!({ + "[1]]]\u0013" => "[#{BLUE}#{BOLD}1#{CLEAR}]#{RED}#{REVERSE}]#{CLEAR}]^S", + "def req(true) end" => "#{GREEN}def#{CLEAR} #{BLUE}#{BOLD}req#{CLEAR}(#{RED}#{REVERSE}true#{CLEAR}) end", + }) + else + tests.merge!({ + "[1]]]\u0013" => "[#{BLUE}#{BOLD}1#{CLEAR}]]]^S", + "def req(true) end" => "#{GREEN}def#{CLEAR} #{BLUE}#{BOLD}req#{CLEAR}(#{CYAN}#{BOLD}true#{CLEAR}) end", }) + end tests.merge!({ - "def req(true) end" => "def req(true) end", "nil = 1" => "#{CYAN}#{BOLD}nil#{CLEAR} = #{BLUE}#{BOLD}1#{CLEAR}", "alias $x $1" => "#{GREEN}alias#{CLEAR} #{GREEN}#{BOLD}$x#{CLEAR} $1", "class bad; end" => "#{GREEN}class#{CLEAR} bad; #{GREEN}end#{CLEAR}", From da54e7f0813d88546091af5bf220d83a6bd10198 Mon Sep 17 00:00:00 2001 From: tompng Date: Mon, 19 Sep 2022 14:14:10 +0900 Subject: [PATCH 3/3] Rewrite on_scan proc to be more readable. --- lib/irb/color.rb | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/lib/irb/color.rb b/lib/irb/color.rb index 401966d18..7071696cb 100644 --- a/lib/irb/color.rb +++ b/lib/irb/color.rb @@ -176,14 +176,18 @@ def scan(code, allow_last_error:) end on_scan = proc do |elem| - str = elem.tok start_pos = line_positions[elem.pos[0] - 1] + elem.pos[1] - end_pos = start_pos + str.bytesize - next if start_pos < byte_pos - yield(nil, inner_code.byteslice(byte_pos...start_pos), nil) if byte_pos < start_pos - yield(elem.event, str, elem.state) - byte_pos = end_pos + # yield uncolorable code + if byte_pos < start_pos + yield(nil, inner_code.byteslice(byte_pos...start_pos), nil) + end + + if byte_pos <= start_pos + str = elem.tok + yield(elem.event, str, elem.state) + byte_pos = start_pos + str.bytesize + end end if lexer.respond_to?(:scan) # Ruby 2.7+