Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix warning in Lexer::Elem#[] #313

Merged
merged 1 commit into from
Dec 20, 2021
Merged

Fix warning in Lexer::Elem#[] #313

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion lib/irb/cmd/show_source.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ def find_end(file, first_line)
prev_tokens = []

# chunk with line number
tokens.chunk { |tok| tok[0][0] }.each do |lnum, chunk|
tokens.chunk { |tok| tok.pos[0] }.each do |lnum, chunk|
code = lines[0..lnum].join
prev_tokens.concat chunk
continue = lex.process_continue(prev_tokens)
Expand Down
126 changes: 63 additions & 63 deletions lib/irb/ruby-lex.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,8 +87,8 @@ def set_input(io, p = nil, context: nil, &block)
tokens.each do |t|
partial_tokens << t
unprocessed_tokens << t
if t[2].include?("\n")
t_str = t[2]
if t.tok.include?("\n")
t_str = t.tok
t_str.each_line("\n") do |s|
code << s << "\n"
ltype, indent, continue, code_block_open = check_state(code, partial_tokens, context: context)
Expand All @@ -97,7 +97,7 @@ def set_input(io, p = nil, context: nil, &block)
end
unprocessed_tokens = []
else
code << t[2]
code << t.tok
end
end
unless unprocessed_tokens.empty?
Expand Down Expand Up @@ -153,14 +153,14 @@ def self.ripper_lex_without_warning(code, context: nil)
pos_to_index = {}
lexer.scan.each do |t|
next if t.pos.first == 0
if pos_to_index.has_key?(t[0])
index = pos_to_index[t[0]]
if pos_to_index.has_key?(t.pos)
index = pos_to_index[t.pos]
found_tk = tokens[index]
if ERROR_TOKENS.include?(found_tk[1]) && !ERROR_TOKENS.include?(t[1])
if ERROR_TOKENS.include?(found_tk.event) && !ERROR_TOKENS.include?(t.event)
tokens[index] = t
end
else
pos_to_index[t[0]] = tokens.size
pos_to_index[t.pos] = tokens.size
tokens << t
end
end
Expand All @@ -175,17 +175,17 @@ def self.ripper_lex_without_warning(code, context: nil)

def find_prev_spaces(line_index)
return 0 if @tokens.size == 0
md = @tokens[0][2].match(/(\A +)/)
md = @tokens[0].tok.match(/(\A +)/)
prev_spaces = md.nil? ? 0 : md[1].count(' ')
line_count = 0
@tokens.each_with_index do |t, i|
if t[2].include?("\n")
line_count += t[2].count("\n")
if t.tok.include?("\n")
line_count += t.tok.count("\n")
if line_count >= line_index
return prev_spaces
end
if (@tokens.size - 1) > i
md = @tokens[i + 1][2].match(/(\A +)/)
md = @tokens[i + 1].tok.match(/(\A +)/)
prev_spaces = md.nil? ? 0 : md[1].count(' ')
end
end
Expand Down Expand Up @@ -295,18 +295,18 @@ def lex

def process_continue(tokens = @tokens)
# last token is always newline
if tokens.size >= 2 and tokens[-2][1] == :on_regexp_end
if tokens.size >= 2 and tokens[-2].event == :on_regexp_end
# end of regexp literal
return false
elsif tokens.size >= 2 and tokens[-2][1] == :on_semicolon
elsif tokens.size >= 2 and tokens[-2].event == :on_semicolon
return false
elsif tokens.size >= 2 and tokens[-2][1] == :on_kw and ['begin', 'else', 'ensure'].include?(tokens[-2][2])
elsif tokens.size >= 2 and tokens[-2].event == :on_kw and ['begin', 'else', 'ensure'].include?(tokens[-2].tok)
return false
elsif !tokens.empty? and tokens.last[2] == "\\\n"
elsif !tokens.empty? and tokens.last.tok == "\\\n"
return true
elsif tokens.size >= 1 and tokens[-1][1] == :on_heredoc_end # "EOH\n"
elsif tokens.size >= 1 and tokens[-1].event == :on_heredoc_end # "EOH\n"
return false
elsif tokens.size >= 2 and defined?(Ripper::EXPR_BEG) and tokens[-2][3].anybits?(Ripper::EXPR_BEG | Ripper::EXPR_FNAME) and tokens[-2][2] !~ /\A\.\.\.?\z/
elsif tokens.size >= 2 and defined?(Ripper::EXPR_BEG) and tokens[-2].state.anybits?(Ripper::EXPR_BEG | Ripper::EXPR_FNAME) and tokens[-2].tok !~ /\A\.\.\.?\z/
# end of literal except for regexp
# endless range at end of line is not a continue
return true
Expand All @@ -316,7 +316,7 @@ def process_continue(tokens = @tokens)

def check_code_block(code, tokens = @tokens)
return true if tokens.empty?
if tokens.last[1] == :on_heredoc_beg
if tokens.last.event == :on_heredoc_beg
return true
end

Expand Down Expand Up @@ -388,7 +388,7 @@ def check_code_block(code, tokens = @tokens)
end

if defined?(Ripper::EXPR_BEG)
last_lex_state = tokens.last[3]
last_lex_state = tokens.last.state
if last_lex_state.allbits?(Ripper::EXPR_BEG)
return false
elsif last_lex_state.allbits?(Ripper::EXPR_DOT)
Expand All @@ -413,14 +413,14 @@ def process_nesting_level(tokens = @tokens)
tokens.each_with_index { |t, index|
# detecting one-liner method definition
if in_oneliner_def.nil?
if t[3].allbits?(Ripper::EXPR_ENDFN)
if t.state.allbits?(Ripper::EXPR_ENDFN)
in_oneliner_def = :ENDFN
end
else
if t[3].allbits?(Ripper::EXPR_ENDFN)
if t.state.allbits?(Ripper::EXPR_ENDFN)
# continuing
elsif t[3].allbits?(Ripper::EXPR_BEG)
if t[2] == '='
elsif t.state.allbits?(Ripper::EXPR_BEG)
if t.tok == '='
in_oneliner_def = :BODY
end
else
Expand All @@ -432,22 +432,22 @@ def process_nesting_level(tokens = @tokens)
end
end

case t[1]
case t.event
when :on_lbracket, :on_lbrace, :on_lparen, :on_tlambeg
indent += 1
when :on_rbracket, :on_rbrace, :on_rparen
indent -= 1
when :on_kw
next if index > 0 and tokens[index - 1][3].allbits?(Ripper::EXPR_FNAME)
case t[2]
next if index > 0 and tokens[index - 1].state.allbits?(Ripper::EXPR_FNAME)
case t.tok
when 'do'
syntax_of_do = take_corresponding_syntax_to_kw_do(tokens, index)
indent += 1 if syntax_of_do == :method_calling
when 'def', 'case', 'for', 'begin', 'class', 'module'
indent += 1
when 'if', 'unless', 'while', 'until'
# postfix if/unless/while/until must be Ripper::EXPR_LABEL
indent += 1 unless t[3].allbits?(Ripper::EXPR_LABEL)
indent += 1 unless t.state.allbits?(Ripper::EXPR_LABEL)
when 'end'
indent -= 1
end
Expand All @@ -459,14 +459,14 @@ def process_nesting_level(tokens = @tokens)

def is_method_calling?(tokens, index)
tk = tokens[index]
if tk[3].anybits?(Ripper::EXPR_CMDARG) and tk[1] == :on_ident
if tk.state.anybits?(Ripper::EXPR_CMDARG) and tk.event == :on_ident
# The target method call to pass the block with "do".
return true
elsif tk[3].anybits?(Ripper::EXPR_ARG) and tk[1] == :on_ident
non_sp_index = tokens[0..(index - 1)].rindex{ |t| t[1] != :on_sp }
elsif tk.state.anybits?(Ripper::EXPR_ARG) and tk.event == :on_ident
non_sp_index = tokens[0..(index - 1)].rindex{ |t| t.event != :on_sp }
if non_sp_index
prev_tk = tokens[non_sp_index]
if prev_tk[3].anybits?(Ripper::EXPR_DOT) and prev_tk[1] == :on_period
if prev_tk.state.anybits?(Ripper::EXPR_DOT) and prev_tk.event == :on_period
# The target method call with receiver to pass the block with "do".
return true
end
Expand All @@ -481,17 +481,17 @@ def take_corresponding_syntax_to_kw_do(tokens, index)
index.downto(0) do |i|
tk = tokens[i]
# In "continue", the token isn't the corresponding syntax to "do".
non_sp_index = tokens[0..(i - 1)].rindex{ |t| t[1] != :on_sp }
non_sp_index = tokens[0..(i - 1)].rindex{ |t| t.event != :on_sp }
first_in_fomula = false
if non_sp_index.nil?
first_in_fomula = true
elsif [:on_ignored_nl, :on_nl, :on_comment].include?(tokens[non_sp_index][1])
elsif [:on_ignored_nl, :on_nl, :on_comment].include?(tokens[non_sp_index].event)
first_in_fomula = true
end
if is_method_calling?(tokens, i)
syntax_of_do = :method_calling
break if first_in_fomula
elsif tk[1] == :on_kw && %w{while until for}.include?(tk[2])
elsif tk.event == :on_kw && %w{while until for}.include?(tk.tok)
# A loop syntax in front of "do" found.
#
# while cond do # also "until" or "for"
Expand All @@ -512,14 +512,14 @@ def is_the_in_correspond_to_a_for(tokens, index)
index.downto(0) do |i|
tk = tokens[i]
# In "continue", the token isn't the corresponding syntax to "do".
non_sp_index = tokens[0..(i - 1)].rindex{ |t| t[1] != :on_sp }
non_sp_index = tokens[0..(i - 1)].rindex{ |t| t.event != :on_sp }
first_in_fomula = false
if non_sp_index.nil?
first_in_fomula = true
elsif [:on_ignored_nl, :on_nl, :on_comment].include?(tokens[non_sp_index][1])
elsif [:on_ignored_nl, :on_nl, :on_comment].include?(tokens[non_sp_index].event)
first_in_fomula = true
end
if tk[1] == :on_kw && tk[2] == 'for'
if tk.event == :on_kw && tk.tok == 'for'
# A loop syntax in front of "do" found.
#
# while cond do # also "until" or "for"
Expand All @@ -541,14 +541,14 @@ def check_newline_depth_difference
@tokens.each_with_index do |t, index|
# detecting one-liner method definition
if in_oneliner_def.nil?
if t[3].allbits?(Ripper::EXPR_ENDFN)
if t.state.allbits?(Ripper::EXPR_ENDFN)
in_oneliner_def = :ENDFN
end
else
if t[3].allbits?(Ripper::EXPR_ENDFN)
if t.state.allbits?(Ripper::EXPR_ENDFN)
# continuing
elsif t[3].allbits?(Ripper::EXPR_BEG)
if t[2] == '='
elsif t.state.allbits?(Ripper::EXPR_BEG)
if t.tok == '='
in_oneliner_def = :BODY
end
else
Expand All @@ -560,7 +560,7 @@ def check_newline_depth_difference
end
end

case t[1]
case t.event
when :on_ignored_nl, :on_nl, :on_comment
if index != (@tokens.size - 1) and in_oneliner_def != :BODY
depth_difference = 0
Expand All @@ -570,23 +570,23 @@ def check_newline_depth_difference
when :on_sp
next
end
case t[1]
case t.event
when :on_lbracket, :on_lbrace, :on_lparen, :on_tlambeg
depth_difference += 1
open_brace_on_line += 1
when :on_rbracket, :on_rbrace, :on_rparen
depth_difference -= 1 if open_brace_on_line > 0
when :on_kw
next if index > 0 and @tokens[index - 1][3].allbits?(Ripper::EXPR_FNAME)
case t[2]
next if index > 0 and @tokens[index - 1].state.allbits?(Ripper::EXPR_FNAME)
case t.tok
when 'do'
syntax_of_do = take_corresponding_syntax_to_kw_do(@tokens, index)
depth_difference += 1 if syntax_of_do == :method_calling
when 'def', 'case', 'for', 'begin', 'class', 'module'
depth_difference += 1
when 'if', 'unless', 'while', 'until', 'rescue'
# postfix if/unless/while/until/rescue must be Ripper::EXPR_LABEL
unless t[3].allbits?(Ripper::EXPR_LABEL)
unless t.state.allbits?(Ripper::EXPR_LABEL)
depth_difference += 1
end
when 'else', 'elsif', 'ensure', 'when'
Expand Down Expand Up @@ -619,14 +619,14 @@ def check_corresponding_token_depth(lines, line_index)
@tokens.each_with_index do |t, index|
# detecting one-liner method definition
if in_oneliner_def.nil?
if t[3].allbits?(Ripper::EXPR_ENDFN)
if t.state.allbits?(Ripper::EXPR_ENDFN)
in_oneliner_def = :ENDFN
end
else
if t[3].allbits?(Ripper::EXPR_ENDFN)
if t.state.allbits?(Ripper::EXPR_ENDFN)
# continuing
elsif t[3].allbits?(Ripper::EXPR_BEG)
if t[2] == '='
elsif t.state.allbits?(Ripper::EXPR_BEG)
if t.tok == '='
in_oneliner_def = :BODY
end
else
Expand All @@ -643,7 +643,7 @@ def check_corresponding_token_depth(lines, line_index)
end
end

case t[1]
case t.event
when :on_ignored_nl, :on_nl, :on_comment
if in_oneliner_def != :BODY
corresponding_token_depth = nil
Expand All @@ -654,11 +654,11 @@ def check_corresponding_token_depth(lines, line_index)
end
next
when :on_sp
spaces_at_line_head = t[2].count(' ') if is_first_spaces_of_line
spaces_at_line_head = t.tok.count(' ') if is_first_spaces_of_line
is_first_spaces_of_line = false
next
end
case t[1]
case t.event
when :on_lbracket, :on_lbrace, :on_lparen, :on_tlambeg
spaces_of_nest.push(spaces_at_line_head + open_brace_on_line * 2)
open_brace_on_line += 1
Expand All @@ -671,8 +671,8 @@ def check_corresponding_token_depth(lines, line_index)
end
open_brace_on_line -= 1
when :on_kw
next if index > 0 and @tokens[index - 1][3].allbits?(Ripper::EXPR_FNAME)
case t[2]
next if index > 0 and @tokens[index - 1].state.allbits?(Ripper::EXPR_FNAME)
case t.tok
when 'do'
syntax_of_do = take_corresponding_syntax_to_kw_do(@tokens, index)
if syntax_of_do == :method_calling
Expand All @@ -681,12 +681,12 @@ def check_corresponding_token_depth(lines, line_index)
when 'def', 'case', 'for', 'begin', 'class', 'module'
spaces_of_nest.push(spaces_at_line_head)
when 'rescue'
unless t[3].allbits?(Ripper::EXPR_LABEL)
unless t.state.allbits?(Ripper::EXPR_LABEL)
corresponding_token_depth = spaces_of_nest.last
end
when 'if', 'unless', 'while', 'until'
# postfix if/unless/while/until must be Ripper::EXPR_LABEL
unless t[3].allbits?(Ripper::EXPR_LABEL)
unless t.state.allbits?(Ripper::EXPR_LABEL)
spaces_of_nest.push(spaces_at_line_head)
end
when 'else', 'elsif', 'ensure', 'when'
Expand Down Expand Up @@ -716,7 +716,7 @@ def check_string_literal(tokens)
end_type = []
while i < tokens.size
t = tokens[i]
case t[1]
case t.event
when *end_type.last
start_token.pop
end_type.pop
Expand All @@ -729,7 +729,7 @@ def check_string_literal(tokens)
when :on_symbeg
acceptable_single_tokens = %i{on_ident on_const on_op on_cvar on_ivar on_gvar on_kw on_int on_backtick}
if (i + 1) < tokens.size
if acceptable_single_tokens.all?{ |st| tokens[i + 1][1] != st }
if acceptable_single_tokens.all?{ |st| tokens[i + 1].event != st }
start_token << t
end_type << :on_tstring_end
else
Expand All @@ -748,14 +748,14 @@ def check_string_literal(tokens)
end
i += 1
end
start_token.last.nil? ? '' : start_token.last
start_token.last.nil? ? nil : start_token.last
end

def process_literal_type(tokens = @tokens)
start_token = check_string_literal(tokens)
case start_token[1]
case start_token&.event
when :on_tstring_beg
case start_token[2]
case start_token&.tok
when ?" then ?"
when /^%.$/ then ?"
when /^%Q.$/ then ?"
Expand All @@ -770,7 +770,7 @@ def process_literal_type(tokens = @tokens)
when :on_qsymbols_beg then ?]
when :on_symbols_beg then ?]
when :on_heredoc_beg
start_token[2] =~ /<<[-~]?(['"`])[_a-zA-Z0-9]+\1/
start_token&.tok =~ /<<[-~]?(['"`])[_a-zA-Z0-9]+\1/
case $1
when ?" then ?"
when ?' then ?'
Expand Down
Loading