From b03705dbaf80ae445ab9fa38b52e7f7ac0b6829a Mon Sep 17 00:00:00 2001 From: tomoya ishida Date: Sun, 24 Mar 2024 19:45:28 +0900 Subject: [PATCH 001/211] [ruby/reline] Refactor key actor test (https://github.com/ruby/reline/pull/645) * Add assertion assert_cursor_line to test helper * Autofix key_actor test to use assert_cursor_line * Rename the assertion to assert_line_around_cursor and remove other assertions for line and cursor https://github.com/ruby/reline/commit/e4773800c6 --- test/reline/helper.rb | 30 +- test/reline/test_key_actor_emacs.rb | 1770 ++++++--------------------- test/reline/test_key_actor_vi.rb | 1160 ++++-------------- 3 files changed, 649 insertions(+), 2311 deletions(-) diff --git a/test/reline/helper.rb b/test/reline/helper.rb index 2180fdf5969abe..b7f849591b7d0d 100644 --- a/test/reline/helper.rb +++ b/test/reline/helper.rb @@ -136,9 +136,14 @@ def input_raw_keys(input, convert = true) end end - def assert_line(expected) - expected = convert_str(expected) - assert_equal(expected, @line_editor.line) + def assert_line_around_cursor(before, after) + before = convert_str(before) + after = convert_str(after) + line = @line_editor.line + byte_pointer = @line_editor.instance_variable_get(:@byte_pointer) + actual_before = line.byteslice(0, byte_pointer) + actual_after = line.byteslice(byte_pointer..) + assert_equal([before, after], [actual_before, actual_after]) end def assert_byte_pointer_size(expected) @@ -153,25 +158,6 @@ def assert_byte_pointer_size(expected) EOM end - def assert_cursor(expected) - # This test satisfies nothing because there is no `@cursor` anymore - # Test editor_cursor_position instead - cursor_x = @line_editor.instance_eval do - line_before_cursor = whole_lines[@line_index].byteslice(0, @byte_pointer) - Reline::Unicode.calculate_width(line_before_cursor) - end - assert_equal(expected, cursor_x) - end - - def assert_cursor_max(expected) - # This test satisfies nothing because there is no `@cursor_max` anymore - cursor_max = @line_editor.instance_eval do - line = whole_lines[@line_index] - Reline::Unicode.calculate_width(line) - end - assert_equal(expected, cursor_max) - end - def assert_line_index(expected) assert_equal(expected, @line_editor.instance_variable_get(:@line_index)) end diff --git a/test/reline/test_key_actor_emacs.rb b/test/reline/test_key_actor_emacs.rb index d5ddd40f8526cf..2311af0f5aad26 100644 --- a/test/reline/test_key_actor_emacs.rb +++ b/test/reline/test_key_actor_emacs.rb @@ -19,418 +19,238 @@ def teardown def test_ed_insert_one input_keys('a') - assert_line('a') - assert_byte_pointer_size('a') - assert_cursor(1) - assert_cursor_max(1) + assert_line_around_cursor('a', '') end def test_ed_insert_two input_keys('ab') - assert_line('ab') - assert_byte_pointer_size('ab') - assert_cursor(2) - assert_cursor_max(2) + assert_line_around_cursor('ab', '') end def test_ed_insert_mbchar_one input_keys('か') - assert_line('か') - assert_byte_pointer_size('か') - assert_cursor(2) - assert_cursor_max(2) + assert_line_around_cursor('か', '') end def test_ed_insert_mbchar_two input_keys('かき') - assert_line('かき') - assert_byte_pointer_size('かき') - assert_cursor(4) - assert_cursor_max(4) + assert_line_around_cursor('かき', '') end def test_ed_insert_for_mbchar_by_plural_code_points input_keys("か\u3099") - assert_line("か\u3099") - assert_byte_pointer_size("か\u3099") - assert_cursor(2) - assert_cursor_max(2) + assert_line_around_cursor("か\u3099", '') end def test_ed_insert_for_plural_mbchar_by_plural_code_points input_keys("か\u3099き\u3099") - assert_line("か\u3099き\u3099") - assert_byte_pointer_size("か\u3099き\u3099") - assert_cursor(4) - assert_cursor_max(4) + assert_line_around_cursor("か\u3099き\u3099", '') end def test_move_next_and_prev input_keys('abd') - assert_byte_pointer_size('abd') - assert_cursor(3) - assert_cursor_max(3) + assert_line_around_cursor('abd', '') input_keys("\C-b", false) - assert_byte_pointer_size('ab') - assert_cursor(2) - assert_cursor_max(3) + assert_line_around_cursor('ab', 'd') input_keys("\C-b", false) - assert_byte_pointer_size('a') - assert_cursor(1) - assert_cursor_max(3) + assert_line_around_cursor('a', 'bd') input_keys("\C-f", false) - assert_byte_pointer_size('ab') - assert_cursor(2) - assert_cursor_max(3) + assert_line_around_cursor('ab', 'd') input_keys('c') - assert_byte_pointer_size('abc') - assert_cursor(3) - assert_cursor_max(4) - assert_line('abcd') + assert_line_around_cursor('abc', 'd') end def test_move_next_and_prev_for_mbchar input_keys('かきけ') - assert_byte_pointer_size('かきけ') - assert_cursor(6) - assert_cursor_max(6) + assert_line_around_cursor('かきけ', '') input_keys("\C-b", false) - assert_byte_pointer_size('かき') - assert_cursor(4) - assert_cursor_max(6) + assert_line_around_cursor('かき', 'け') input_keys("\C-b", false) - assert_byte_pointer_size('か') - assert_cursor(2) - assert_cursor_max(6) + assert_line_around_cursor('か', 'きけ') input_keys("\C-f", false) - assert_byte_pointer_size('かき') - assert_cursor(4) - assert_cursor_max(6) + assert_line_around_cursor('かき', 'け') input_keys('く') - assert_byte_pointer_size('かきく') - assert_cursor(6) - assert_cursor_max(8) - assert_line('かきくけ') + assert_line_around_cursor('かきく', 'け') end def test_move_next_and_prev_for_mbchar_by_plural_code_points input_keys("か\u3099き\u3099け\u3099") - assert_byte_pointer_size("か\u3099き\u3099け\u3099") - assert_cursor(6) - assert_cursor_max(6) + assert_line_around_cursor("か\u3099き\u3099け\u3099", '') input_keys("\C-b", false) - assert_byte_pointer_size("か\u3099き\u3099") - assert_cursor(4) - assert_cursor_max(6) + assert_line_around_cursor("か\u3099き\u3099", "け\u3099") input_keys("\C-b", false) - assert_byte_pointer_size("か\u3099") - assert_cursor(2) - assert_cursor_max(6) + assert_line_around_cursor("か\u3099", "き\u3099け\u3099") input_keys("\C-f", false) - assert_byte_pointer_size("か\u3099き\u3099") - assert_cursor(4) - assert_cursor_max(6) + assert_line_around_cursor("か\u3099き\u3099", "け\u3099") input_keys("く\u3099") - assert_byte_pointer_size("か\u3099き\u3099く\u3099") - assert_cursor(6) - assert_cursor_max(8) - assert_line("か\u3099き\u3099く\u3099け\u3099") + assert_line_around_cursor("か\u3099き\u3099く\u3099", "け\u3099") end def test_move_to_beg_end input_keys('bcd') - assert_byte_pointer_size('bcd') - assert_cursor(3) - assert_cursor_max(3) + assert_line_around_cursor('bcd', '') input_keys("\C-a", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(3) + assert_line_around_cursor('', 'bcd') input_keys('a') - assert_byte_pointer_size('a') - assert_cursor(1) - assert_cursor_max(4) + assert_line_around_cursor('a', 'bcd') input_keys("\C-e", false) - assert_byte_pointer_size('abcd') - assert_cursor(4) - assert_cursor_max(4) + assert_line_around_cursor('abcd', '') input_keys('e') - assert_byte_pointer_size('abcde') - assert_cursor(5) - assert_cursor_max(5) - assert_line('abcde') + assert_line_around_cursor('abcde', '') end def test_ed_newline_with_cr input_keys('ab') - assert_byte_pointer_size('ab') - assert_cursor(2) - assert_cursor_max(2) + assert_line_around_cursor('ab', '') refute(@line_editor.finished?) input_keys("\C-m", false) - assert_line('ab') + assert_line_around_cursor('ab', '') assert(@line_editor.finished?) end def test_ed_newline_with_lf input_keys('ab') - assert_byte_pointer_size('ab') - assert_cursor(2) - assert_cursor_max(2) + assert_line_around_cursor('ab', '') refute(@line_editor.finished?) input_keys("\C-j", false) - assert_line('ab') + assert_line_around_cursor('ab', '') assert(@line_editor.finished?) end def test_em_delete_prev_char input_keys('ab') - assert_byte_pointer_size('ab') - assert_cursor(2) - assert_cursor_max(2) + assert_line_around_cursor('ab', '') input_keys("\C-h", false) - assert_byte_pointer_size('a') - assert_cursor(1) - assert_cursor_max(1) - assert_line('a') + assert_line_around_cursor('a', '') end def test_em_delete_prev_char_for_mbchar input_keys('かき') - assert_byte_pointer_size('かき') - assert_cursor(4) - assert_cursor_max(4) + assert_line_around_cursor('かき', '') input_keys("\C-h", false) - assert_byte_pointer_size('か') - assert_cursor(2) - assert_cursor_max(2) - assert_line('か') + assert_line_around_cursor('か', '') end def test_em_delete_prev_char_for_mbchar_by_plural_code_points input_keys("か\u3099き\u3099") - assert_byte_pointer_size("か\u3099き\u3099") - assert_cursor(4) - assert_cursor_max(4) + assert_line_around_cursor("か\u3099き\u3099", '') input_keys("\C-h", false) - assert_byte_pointer_size("か\u3099") - assert_cursor(2) - assert_cursor_max(2) - assert_line("か\u3099") + assert_line_around_cursor("か\u3099", '') end def test_ed_quoted_insert input_keys("ab\C-v\C-acd") - assert_line("ab\C-acd") - assert_byte_pointer_size("ab\C-acd") - assert_cursor(6) - assert_cursor_max(6) + assert_line_around_cursor("ab\C-acd", '') input_keys("\C-q\C-b") - assert_line("ab\C-acd\C-b") - assert_byte_pointer_size("ab\C-acd\C-b") - assert_cursor(8) - assert_cursor_max(8) + assert_line_around_cursor("ab\C-acd\C-b", '') end def test_ed_kill_line input_keys("\C-k", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(0) - assert_line('') + assert_line_around_cursor('', '') input_keys('abc') - assert_byte_pointer_size('abc') - assert_cursor(3) - assert_cursor_max(3) + assert_line_around_cursor('abc', '') input_keys("\C-k", false) - assert_byte_pointer_size('abc') - assert_cursor(3) - assert_cursor_max(3) - assert_line('abc') + assert_line_around_cursor('abc', '') input_keys("\C-b\C-k", false) - assert_byte_pointer_size('ab') - assert_cursor(2) - assert_cursor_max(2) - assert_line('ab') + assert_line_around_cursor('ab', '') end def test_em_kill_line @line_editor.input_key(Reline::Key.new(:em_kill_line, :em_kill_line, false)) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(0) - assert_line('') + assert_line_around_cursor('', '') input_keys('abc') @line_editor.input_key(Reline::Key.new(:em_kill_line, :em_kill_line, false)) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(0) - assert_line('') + assert_line_around_cursor('', '') input_keys('abc') input_keys("\C-b", false) @line_editor.input_key(Reline::Key.new(:em_kill_line, :em_kill_line, false)) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(0) - assert_line('') + assert_line_around_cursor('', '') input_keys('abc') input_keys("\C-a", false) @line_editor.input_key(Reline::Key.new(:em_kill_line, :em_kill_line, false)) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(0) - assert_line('') + assert_line_around_cursor('', '') end def test_ed_move_to_beg input_keys('abd') - assert_byte_pointer_size('abd') - assert_cursor(3) - assert_cursor_max(3) + assert_line_around_cursor('abd', '') input_keys("\C-b", false) - assert_byte_pointer_size('ab') - assert_cursor(2) - assert_cursor_max(3) + assert_line_around_cursor('ab', 'd') input_keys('c') - assert_byte_pointer_size('abc') - assert_cursor(3) - assert_cursor_max(4) + assert_line_around_cursor('abc', 'd') input_keys("\C-a", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(4) + assert_line_around_cursor('', 'abcd') input_keys('012') - assert_byte_pointer_size('012') - assert_cursor(3) - assert_cursor_max(7) - assert_line('012abcd') + assert_line_around_cursor('012', 'abcd') input_keys("\C-a", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(7) + assert_line_around_cursor('', '012abcd') input_keys('ABC') - assert_byte_pointer_size('ABC') - assert_cursor(3) - assert_cursor_max(10) - assert_line('ABC012abcd') + assert_line_around_cursor('ABC', '012abcd') input_keys("\C-f" * 10 + "\C-a", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(10) + assert_line_around_cursor('', 'ABC012abcd') input_keys('a') - assert_byte_pointer_size('a') - assert_cursor(1) - assert_cursor_max(11) - assert_line('aABC012abcd') + assert_line_around_cursor('a', 'ABC012abcd') end def test_ed_move_to_beg_with_blank input_keys(' abc') - assert_byte_pointer_size(' abc') - assert_cursor(5) - assert_cursor_max(5) + assert_line_around_cursor(' abc', '') input_keys("\C-a", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(5) + assert_line_around_cursor('', ' abc') end def test_ed_move_to_end input_keys('abd') - assert_byte_pointer_size('abd') - assert_cursor(3) - assert_cursor_max(3) + assert_line_around_cursor('abd', '') input_keys("\C-b", false) - assert_byte_pointer_size('ab') - assert_cursor(2) - assert_cursor_max(3) + assert_line_around_cursor('ab', 'd') input_keys('c') - assert_byte_pointer_size('abc') - assert_cursor(3) - assert_cursor_max(4) + assert_line_around_cursor('abc', 'd') input_keys("\C-e", false) - assert_byte_pointer_size('abcd') - assert_cursor(4) - assert_cursor_max(4) + assert_line_around_cursor('abcd', '') input_keys('012') - assert_byte_pointer_size('abcd012') - assert_cursor(7) - assert_cursor_max(7) - assert_line('abcd012') + assert_line_around_cursor('abcd012', '') input_keys("\C-e", false) - assert_byte_pointer_size('abcd012') - assert_cursor(7) - assert_cursor_max(7) + assert_line_around_cursor('abcd012', '') input_keys('ABC') - assert_byte_pointer_size('abcd012ABC') - assert_cursor(10) - assert_cursor_max(10) - assert_line('abcd012ABC') + assert_line_around_cursor('abcd012ABC', '') input_keys("\C-b" * 10 + "\C-e", false) - assert_byte_pointer_size('abcd012ABC') - assert_cursor(10) - assert_cursor_max(10) + assert_line_around_cursor('abcd012ABC', '') input_keys('a') - assert_byte_pointer_size('abcd012ABCa') - assert_cursor(11) - assert_cursor_max(11) - assert_line('abcd012ABCa') + assert_line_around_cursor('abcd012ABCa', '') end def test_em_delete input_keys('ab') - assert_byte_pointer_size('ab') - assert_cursor(2) - assert_cursor_max(2) + assert_line_around_cursor('ab', '') input_keys("\C-a", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(2) + assert_line_around_cursor('', 'ab') input_keys("\C-d", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(1) - assert_line('b') + assert_line_around_cursor('', 'b') end def test_em_delete_for_mbchar input_keys('かき') - assert_byte_pointer_size('かき') - assert_cursor(4) - assert_cursor_max(4) + assert_line_around_cursor('かき', '') input_keys("\C-a", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(4) + assert_line_around_cursor('', 'かき') input_keys("\C-d", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(2) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(2) - assert_line('き') + assert_line_around_cursor('', 'き') end def test_em_delete_for_mbchar_by_plural_code_points input_keys("か\u3099き\u3099") - assert_byte_pointer_size("か\u3099き\u3099") - assert_cursor(4) - assert_cursor_max(4) + assert_line_around_cursor("か\u3099き\u3099", '') input_keys("\C-a", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(4) + assert_line_around_cursor('', "か\u3099き\u3099") input_keys("\C-d", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(2) - assert_line("き\u3099") + assert_line_around_cursor('', "き\u3099") end def test_em_delete_ends_editing input_keys("\C-d") # quit from inputing - assert_line(nil) + assert_nil(@line_editor.line) assert(@line_editor.finished?) end @@ -444,834 +264,454 @@ def test_ed_clear_screen_with_inputed input_keys('abc') input_keys("\C-b", false) refute(@line_editor.instance_variable_get(:@cleared)) - assert_byte_pointer_size('ab') - assert_cursor(2) - assert_cursor_max(3) + assert_line_around_cursor('ab', 'c') input_keys("\C-l", false) assert(@line_editor.instance_variable_get(:@cleared)) - assert_byte_pointer_size('ab') - assert_cursor(2) - assert_cursor_max(3) - assert_line('abc') + assert_line_around_cursor('ab', 'c') end def test_key_delete input_keys('abc') - assert_cursor(3) - assert_cursor_max(3) + assert_line_around_cursor('abc', '') @line_editor.input_key(Reline::Key.new(:key_delete, :key_delete, false)) - assert_cursor(3) - assert_cursor_max(3) - assert_line('abc') + assert_line_around_cursor('abc', '') end def test_key_delete_does_not_end_editing @line_editor.input_key(Reline::Key.new(:key_delete, :key_delete, false)) - assert_cursor(0) - assert_cursor_max(0) - assert_line('') + assert_line_around_cursor('', '') refute(@line_editor.finished?) end def test_key_delete_preserves_cursor input_keys('abc') input_keys("\C-b", false) - assert_cursor(2) - assert_cursor_max(3) + assert_line_around_cursor('ab', 'c') @line_editor.input_key(Reline::Key.new(:key_delete, :key_delete, false)) - assert_cursor(2) - assert_cursor_max(2) - assert_line('ab') + assert_line_around_cursor('ab', '') end def test_em_next_word - assert_byte_pointer_size('') - assert_cursor(0) + assert_line_around_cursor('', '') input_keys('abc def{bbb}ccc') input_keys("\C-a\M-F", false) - assert_byte_pointer_size('abc') - assert_cursor(3) + assert_line_around_cursor('abc', ' def{bbb}ccc') input_keys("\M-F", false) - assert_byte_pointer_size('abc def') - assert_cursor(7) + assert_line_around_cursor('abc def', '{bbb}ccc') input_keys("\M-F", false) - assert_byte_pointer_size('abc def{bbb') - assert_cursor(11) + assert_line_around_cursor('abc def{bbb', '}ccc') input_keys("\M-F", false) - assert_byte_pointer_size('abc def{bbb}ccc') - assert_cursor(15) + assert_line_around_cursor('abc def{bbb}ccc', '') input_keys("\M-F", false) - assert_byte_pointer_size('abc def{bbb}ccc') - assert_cursor(15) + assert_line_around_cursor('abc def{bbb}ccc', '') end def test_em_next_word_for_mbchar - assert_cursor(0) + assert_line_around_cursor('', '') input_keys('あいう かきく{さしす}たちつ') input_keys("\C-a\M-F", false) - assert_byte_pointer_size('あいう') - assert_cursor(6) + assert_line_around_cursor('あいう', ' かきく{さしす}たちつ') input_keys("\M-F", false) - assert_byte_pointer_size('あいう かきく') - assert_cursor(13) + assert_line_around_cursor('あいう かきく', '{さしす}たちつ') input_keys("\M-F", false) - assert_byte_pointer_size('あいう かきく{さしす') - assert_cursor(20) + assert_line_around_cursor('あいう かきく{さしす', '}たちつ') input_keys("\M-F", false) - assert_byte_pointer_size('あいう かきく{さしす}たちつ') - assert_cursor(27) + assert_line_around_cursor('あいう かきく{さしす}たちつ', '') input_keys("\M-F", false) - assert_byte_pointer_size('あいう かきく{さしす}たちつ') - assert_cursor(27) + assert_line_around_cursor('あいう かきく{さしす}たちつ', '') end def test_em_next_word_for_mbchar_by_plural_code_points - assert_cursor(0) + assert_line_around_cursor("", "") input_keys("あいう か\u3099き\u3099く\u3099{さしす}たちつ") input_keys("\C-a\M-F", false) - assert_byte_pointer_size("あいう") - assert_cursor(6) + assert_line_around_cursor("あいう", " か\u3099き\u3099く\u3099{さしす}たちつ") input_keys("\M-F", false) - assert_byte_pointer_size("あいう か\u3099き\u3099く\u3099") - assert_cursor(13) + assert_line_around_cursor("あいう か\u3099き\u3099く\u3099", "{さしす}たちつ") input_keys("\M-F", false) - assert_byte_pointer_size("あいう か\u3099き\u3099く\u3099{さしす") - assert_cursor(20) + assert_line_around_cursor("あいう か\u3099き\u3099く\u3099{さしす", "}たちつ") input_keys("\M-F", false) - assert_byte_pointer_size("あいう か\u3099き\u3099く\u3099{さしす}たちつ") - assert_cursor(27) + assert_line_around_cursor("あいう か\u3099き\u3099く\u3099{さしす}たちつ", "") input_keys("\M-F", false) - assert_byte_pointer_size("あいう か\u3099き\u3099く\u3099{さしす}たちつ") - assert_cursor(27) + assert_line_around_cursor("あいう か\u3099き\u3099く\u3099{さしす}たちつ", "") end def test_em_prev_word input_keys('abc def{bbb}ccc') - assert_byte_pointer_size('abc def{bbb}ccc') - assert_cursor(15) + assert_line_around_cursor('abc def{bbb}ccc', '') input_keys("\M-B", false) - assert_byte_pointer_size('abc def{bbb}') - assert_cursor(12) + assert_line_around_cursor('abc def{bbb}', 'ccc') input_keys("\M-B", false) - assert_byte_pointer_size('abc def{') - assert_cursor(8) + assert_line_around_cursor('abc def{', 'bbb}ccc') input_keys("\M-B", false) - assert_byte_pointer_size('abc ') - assert_cursor(4) + assert_line_around_cursor('abc ', 'def{bbb}ccc') input_keys("\M-B", false) - assert_byte_pointer_size('') - assert_cursor(0) + assert_line_around_cursor('', 'abc def{bbb}ccc') input_keys("\M-B", false) - assert_byte_pointer_size('') - assert_cursor(0) + assert_line_around_cursor('', 'abc def{bbb}ccc') end def test_em_prev_word_for_mbchar input_keys('あいう かきく{さしす}たちつ') - assert_byte_pointer_size('あいう かきく{さしす}たちつ') - assert_cursor(27) + assert_line_around_cursor('あいう かきく{さしす}たちつ', '') input_keys("\M-B", false) - assert_byte_pointer_size('あいう かきく{さしす}') - assert_cursor(21) + assert_line_around_cursor('あいう かきく{さしす}', 'たちつ') input_keys("\M-B", false) - assert_byte_pointer_size('あいう かきく{') - assert_cursor(14) + assert_line_around_cursor('あいう かきく{', 'さしす}たちつ') input_keys("\M-B", false) - assert_byte_pointer_size('あいう ') - assert_cursor(7) + assert_line_around_cursor('あいう ', 'かきく{さしす}たちつ') input_keys("\M-B", false) - assert_byte_pointer_size('') - assert_cursor(0) + assert_line_around_cursor('', 'あいう かきく{さしす}たちつ') input_keys("\M-B", false) - assert_byte_pointer_size('') - assert_cursor(0) + assert_line_around_cursor('', 'あいう かきく{さしす}たちつ') end def test_em_prev_word_for_mbchar_by_plural_code_points input_keys("あいう か\u3099き\u3099く\u3099{さしす}たちつ") - assert_byte_pointer_size("あいう か\u3099き\u3099く\u3099{さしす}たちつ") - assert_cursor(27) + assert_line_around_cursor("あいう か\u3099き\u3099く\u3099{さしす}たちつ", "") input_keys("\M-B", false) - assert_byte_pointer_size("あいう か\u3099き\u3099く\u3099{さしす}") - assert_cursor(21) + assert_line_around_cursor("あいう か\u3099き\u3099く\u3099{さしす}", "たちつ") input_keys("\M-B", false) - assert_byte_pointer_size("あいう か\u3099き\u3099く\u3099{") - assert_cursor(14) + assert_line_around_cursor("あいう か\u3099き\u3099く\u3099{", "さしす}たちつ") input_keys("\M-B", false) - assert_byte_pointer_size('あいう ') - assert_cursor(7) + assert_line_around_cursor("あいう ", "か\u3099き\u3099く\u3099{さしす}たちつ") input_keys("\M-B", false) - assert_byte_pointer_size('') - assert_cursor(0) + assert_line_around_cursor("", "あいう か\u3099き\u3099く\u3099{さしす}たちつ") input_keys("\M-B", false) - assert_byte_pointer_size('') - assert_cursor(0) + assert_line_around_cursor("", "あいう か\u3099き\u3099く\u3099{さしす}たちつ") end def test_em_delete_next_word input_keys('abc def{bbb}ccc') input_keys("\C-a", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(15) + assert_line_around_cursor('', 'abc def{bbb}ccc') input_keys("\M-d", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(12) - assert_line(' def{bbb}ccc') + assert_line_around_cursor('', ' def{bbb}ccc') input_keys("\M-d", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(8) - assert_line('{bbb}ccc') + assert_line_around_cursor('', '{bbb}ccc') input_keys("\M-d", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(4) - assert_line('}ccc') + assert_line_around_cursor('', '}ccc') input_keys("\M-d", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(0) - assert_line('') + assert_line_around_cursor('', '') end def test_em_delete_next_word_for_mbchar input_keys('あいう かきく{さしす}たちつ') input_keys("\C-a", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(27) + assert_line_around_cursor('', 'あいう かきく{さしす}たちつ') input_keys("\M-d", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(21) - assert_line(' かきく{さしす}たちつ') + assert_line_around_cursor('', ' かきく{さしす}たちつ') input_keys("\M-d", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(14) - assert_line('{さしす}たちつ') + assert_line_around_cursor('', '{さしす}たちつ') input_keys("\M-d", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(7) - assert_line('}たちつ') + assert_line_around_cursor('', '}たちつ') input_keys("\M-d", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(0) - assert_line('') + assert_line_around_cursor('', '') end def test_em_delete_next_word_for_mbchar_by_plural_code_points input_keys("あいう か\u3099き\u3099く\u3099{さしす}たちつ") input_keys("\C-a", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(27) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(27) + assert_line_around_cursor('', "あいう か\u3099き\u3099く\u3099{さしす}たちつ") input_keys("\M-d", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(21) - assert_line(" か\u3099き\u3099く\u3099{さしす}たちつ") + assert_line_around_cursor('', " か\u3099き\u3099く\u3099{さしす}たちつ") input_keys("\M-d", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(14) - assert_line('{さしす}たちつ') + assert_line_around_cursor('', '{さしす}たちつ') input_keys("\M-d", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(7) - assert_line('}たちつ') + assert_line_around_cursor('', '}たちつ') input_keys("\M-d", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(0) - assert_line('') + assert_line_around_cursor('', '') end def test_ed_delete_prev_word input_keys('abc def{bbb}ccc') - assert_byte_pointer_size('abc def{bbb}ccc') - assert_cursor(15) - assert_cursor_max(15) + assert_line_around_cursor('abc def{bbb}ccc', '') input_keys("\M-\C-H", false) - assert_byte_pointer_size('abc def{bbb}') - assert_cursor(12) - assert_cursor_max(12) - assert_line('abc def{bbb}') + assert_line_around_cursor('abc def{bbb}', '') input_keys("\M-\C-H", false) - assert_byte_pointer_size('abc def{') - assert_cursor(8) - assert_cursor_max(8) - assert_line('abc def{') + assert_line_around_cursor('abc def{', '') input_keys("\M-\C-H", false) - assert_byte_pointer_size('abc ') - assert_cursor(4) - assert_cursor_max(4) - assert_line('abc ') + assert_line_around_cursor('abc ', '') input_keys("\M-\C-H", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(0) - assert_line('') + assert_line_around_cursor('', '') end def test_ed_delete_prev_word_for_mbchar input_keys('あいう かきく{さしす}たちつ') - assert_byte_pointer_size('あいう かきく{さしす}たちつ') - assert_cursor(27) - assert_cursor_max(27) + assert_line_around_cursor('あいう かきく{さしす}たちつ', '') input_keys("\M-\C-H", false) - assert_byte_pointer_size('あいう かきく{さしす}') - assert_cursor(21) - assert_cursor_max(21) - assert_line('あいう かきく{さしす}') + assert_line_around_cursor('あいう かきく{さしす}', '') input_keys("\M-\C-H", false) - assert_byte_pointer_size('あいう かきく{') - assert_cursor(14) - assert_cursor_max(14) - assert_line('あいう かきく{') + assert_line_around_cursor('あいう かきく{', '') input_keys("\M-\C-H", false) - assert_byte_pointer_size('あいう ') - assert_cursor(7) - assert_cursor_max(7) - assert_line('あいう ') + assert_line_around_cursor('あいう ', '') input_keys("\M-\C-H", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(0) - assert_line('') + assert_line_around_cursor('', '') end def test_ed_delete_prev_word_for_mbchar_by_plural_code_points input_keys("あいう か\u3099き\u3099く\u3099{さしす}たちつ") - assert_byte_pointer_size("あいう か\u3099き\u3099く\u3099{さしす}たちつ") - assert_cursor(27) - assert_cursor_max(27) + assert_line_around_cursor("あいう か\u3099き\u3099く\u3099{さしす}たちつ", '') input_keys("\M-\C-H", false) - assert_byte_pointer_size("あいう か\u3099き\u3099く\u3099{さしす}") - assert_cursor(21) - assert_cursor_max(21) - assert_line("あいう か\u3099き\u3099く\u3099{さしす}") + assert_line_around_cursor("あいう か\u3099き\u3099く\u3099{さしす}", '') input_keys("\M-\C-H", false) - assert_byte_pointer_size("あいう か\u3099き\u3099く\u3099{") - assert_cursor(14) - assert_cursor_max(14) - assert_line("あいう か\u3099き\u3099く\u3099{") + assert_line_around_cursor("あいう か\u3099き\u3099く\u3099{", '') input_keys("\M-\C-H", false) - assert_byte_pointer_size("あいう ") - assert_cursor(7) - assert_cursor_max(7) - assert_line('あいう ') + assert_line_around_cursor('あいう ', '') input_keys("\M-\C-H", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(0) - assert_line('') + assert_line_around_cursor('', '') end def test_ed_transpose_chars input_keys('abc') input_keys("\C-a", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(3) + assert_line_around_cursor('', 'abc') input_keys("\C-t", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(3) - assert_line('abc') + assert_line_around_cursor('', 'abc') input_keys("\C-f\C-t", false) - assert_byte_pointer_size('ba') - assert_cursor(2) - assert_cursor_max(3) - assert_line('bac') + assert_line_around_cursor('ba', 'c') input_keys("\C-t", false) - assert_byte_pointer_size('bca') - assert_cursor(3) - assert_cursor_max(3) - assert_line('bca') + assert_line_around_cursor('bca', '') input_keys("\C-t", false) - assert_byte_pointer_size('bac') - assert_cursor(3) - assert_cursor_max(3) - assert_line('bac') + assert_line_around_cursor('bac', '') end def test_ed_transpose_chars_for_mbchar input_keys('あかさ') input_keys("\C-a", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(6) + assert_line_around_cursor('', 'あかさ') input_keys("\C-t", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(6) - assert_line('あかさ') + assert_line_around_cursor('', 'あかさ') input_keys("\C-f\C-t", false) - assert_byte_pointer_size('かあ') - assert_cursor(4) - assert_cursor_max(6) - assert_line('かあさ') + assert_line_around_cursor('かあ', 'さ') input_keys("\C-t", false) - assert_byte_pointer_size('かさあ') - assert_cursor(6) - assert_cursor_max(6) - assert_line('かさあ') + assert_line_around_cursor('かさあ', '') input_keys("\C-t", false) - assert_byte_pointer_size('かあさ') - assert_cursor(6) - assert_cursor_max(6) - assert_line('かあさ') + assert_line_around_cursor('かあさ', '') end def test_ed_transpose_chars_for_mbchar_by_plural_code_points input_keys("あか\u3099さ") input_keys("\C-a", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(6) + assert_line_around_cursor('', "あか\u3099さ") input_keys("\C-t", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(6) - assert_line("あか\u3099さ") + assert_line_around_cursor('', "あか\u3099さ") input_keys("\C-f\C-t", false) - assert_byte_pointer_size("か\u3099あ") - assert_cursor(4) - assert_cursor_max(6) - assert_line("か\u3099あさ") + assert_line_around_cursor("か\u3099あ", 'さ') input_keys("\C-t", false) - assert_byte_pointer_size("か\u3099さあ") - assert_cursor(6) - assert_cursor_max(6) - assert_line("か\u3099さあ") + assert_line_around_cursor("か\u3099さあ", '') input_keys("\C-t", false) - assert_byte_pointer_size("か\u3099あさ") - assert_cursor(6) - assert_cursor_max(6) - assert_line("か\u3099あさ") + assert_line_around_cursor("か\u3099あさ", '') end def test_ed_transpose_words input_keys('abc def') - assert_line('abc def') - assert_byte_pointer_size('abc def') - assert_cursor(7) - assert_cursor_max(7) + assert_line_around_cursor('abc def', '') input_keys("\M-t", false) - assert_line('def abc') - assert_byte_pointer_size('def abc') - assert_cursor(7) - assert_cursor_max(7) + assert_line_around_cursor('def abc', '') input_keys("\C-a\C-k", false) input_keys(' abc def ') input_keys("\C-b" * 4, false) - assert_line(' abc def ') - assert_byte_pointer_size(' abc de') - assert_cursor(8) - assert_cursor_max(12) + assert_line_around_cursor(' abc de', 'f ') input_keys("\M-t", false) - assert_line(' def abc ') - assert_byte_pointer_size(' def abc') - assert_cursor(9) - assert_cursor_max(12) + assert_line_around_cursor(' def abc', ' ') input_keys("\C-a\C-k", false) input_keys(' abc def ') input_keys("\C-b" * 6, false) - assert_line(' abc def ') - assert_byte_pointer_size(' abc ') - assert_cursor(6) - assert_cursor_max(12) + assert_line_around_cursor(' abc ', 'def ') input_keys("\M-t", false) - assert_line(' def abc ') - assert_byte_pointer_size(' def abc') - assert_cursor(9) - assert_cursor_max(12) + assert_line_around_cursor(' def abc', ' ') input_keys("\M-t", false) - assert_line(' abc def') - assert_byte_pointer_size(' abc def') - assert_cursor(12) - assert_cursor_max(12) + assert_line_around_cursor(' abc def', '') end def test_ed_transpose_words_for_mbchar input_keys('あいう かきく') - assert_line('あいう かきく') - assert_byte_pointer_size('あいう かきく') - assert_cursor(13) - assert_cursor_max(13) + assert_line_around_cursor('あいう かきく', '') input_keys("\M-t", false) - assert_line('かきく あいう') - assert_byte_pointer_size('かきく あいう') - assert_cursor(13) - assert_cursor_max(13) + assert_line_around_cursor('かきく あいう', '') input_keys("\C-a\C-k", false) input_keys(' あいう かきく ') input_keys("\C-b" * 4, false) - assert_line(' あいう かきく ') - assert_byte_pointer_size(' あいう かき') - assert_cursor(13) - assert_cursor_max(18) + assert_line_around_cursor(' あいう かき', 'く ') input_keys("\M-t", false) - assert_line(' かきく あいう ') - assert_byte_pointer_size(' かきく あいう') - assert_cursor(15) - assert_cursor_max(18) + assert_line_around_cursor(' かきく あいう', ' ') input_keys("\C-a\C-k", false) input_keys(' あいう かきく ') input_keys("\C-b" * 6, false) - assert_line(' あいう かきく ') - assert_byte_pointer_size(' あいう ') - assert_cursor(9) - assert_cursor_max(18) + assert_line_around_cursor(' あいう ', 'かきく ') input_keys("\M-t", false) - assert_line(' かきく あいう ') - assert_byte_pointer_size(' かきく あいう') - assert_cursor(15) - assert_cursor_max(18) + assert_line_around_cursor(' かきく あいう', ' ') input_keys("\M-t", false) - assert_line(' あいう かきく') - assert_byte_pointer_size(' あいう かきく') - assert_cursor(18) - assert_cursor_max(18) + assert_line_around_cursor(' あいう かきく', '') end def test_ed_transpose_words_with_one_word input_keys('abc ') - assert_line('abc ') - assert_byte_pointer_size('abc ') - assert_cursor(5) - assert_cursor_max(5) + assert_line_around_cursor('abc ', '') input_keys("\M-t", false) - assert_line('abc ') - assert_byte_pointer_size('abc ') - assert_cursor(5) - assert_cursor_max(5) + assert_line_around_cursor('abc ', '') input_keys("\C-b", false) - assert_line('abc ') - assert_byte_pointer_size('abc ') - assert_cursor(4) - assert_cursor_max(5) + assert_line_around_cursor('abc ', ' ') input_keys("\M-t", false) - assert_line('abc ') - assert_byte_pointer_size('abc ') - assert_cursor(4) - assert_cursor_max(5) + assert_line_around_cursor('abc ', ' ') input_keys("\C-b" * 2, false) - assert_line('abc ') - assert_byte_pointer_size('ab') - assert_cursor(2) - assert_cursor_max(5) + assert_line_around_cursor('ab', 'c ') input_keys("\M-t", false) - assert_line('abc ') - assert_byte_pointer_size('ab') - assert_cursor(2) - assert_cursor_max(5) + assert_line_around_cursor('ab', 'c ') input_keys("\M-t", false) - assert_line('abc ') - assert_byte_pointer_size('ab') - assert_cursor(2) - assert_cursor_max(5) + assert_line_around_cursor('ab', 'c ') end def test_ed_transpose_words_with_one_word_for_mbchar input_keys('あいう ') - assert_line('あいう ') - assert_byte_pointer_size('あいう ') - assert_cursor(8) - assert_cursor_max(8) + assert_line_around_cursor('あいう ', '') input_keys("\M-t", false) - assert_line('あいう ') - assert_byte_pointer_size('あいう ') - assert_cursor(8) - assert_cursor_max(8) + assert_line_around_cursor('あいう ', '') input_keys("\C-b", false) - assert_line('あいう ') - assert_byte_pointer_size('あいう ') - assert_cursor(7) - assert_cursor_max(8) + assert_line_around_cursor('あいう ', ' ') input_keys("\M-t", false) - assert_line('あいう ') - assert_byte_pointer_size('あいう ') - assert_cursor(7) - assert_cursor_max(8) + assert_line_around_cursor('あいう ', ' ') input_keys("\C-b" * 2, false) - assert_line('あいう ') - assert_byte_pointer_size('あい') - assert_cursor(4) - assert_cursor_max(8) + assert_line_around_cursor('あい', 'う ') input_keys("\M-t", false) - assert_line('あいう ') - assert_byte_pointer_size('あい') - assert_cursor(4) - assert_cursor_max(8) + assert_line_around_cursor('あい', 'う ') input_keys("\M-t", false) - assert_line('あいう ') - assert_byte_pointer_size('あい') - assert_cursor(4) - assert_cursor_max(8) + assert_line_around_cursor('あい', 'う ') end def test_ed_digit input_keys('0123') - assert_byte_pointer_size('0123') - assert_cursor(4) - assert_cursor_max(4) - assert_line('0123') + assert_line_around_cursor('0123', '') end def test_ed_next_and_prev_char input_keys('abc') - assert_byte_pointer_size('abc') - assert_cursor(3) - assert_cursor_max(3) + assert_line_around_cursor('abc', '') input_keys("\C-b", false) - assert_byte_pointer_size('ab') - assert_cursor(2) - assert_cursor_max(3) + assert_line_around_cursor('ab', 'c') input_keys("\C-b", false) - assert_byte_pointer_size('a') - assert_cursor(1) - assert_cursor_max(3) + assert_line_around_cursor('a', 'bc') input_keys("\C-b", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(3) + assert_line_around_cursor('', 'abc') input_keys("\C-b", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(3) + assert_line_around_cursor('', 'abc') input_keys("\C-f", false) - assert_byte_pointer_size('a') - assert_cursor(1) - assert_cursor_max(3) + assert_line_around_cursor('a', 'bc') input_keys("\C-f", false) - assert_byte_pointer_size('ab') - assert_cursor(2) - assert_cursor_max(3) + assert_line_around_cursor('ab', 'c') input_keys("\C-f", false) - assert_byte_pointer_size('abc') - assert_cursor(3) - assert_cursor_max(3) + assert_line_around_cursor('abc', '') input_keys("\C-f", false) - assert_byte_pointer_size('abc') - assert_cursor(3) - assert_cursor_max(3) + assert_line_around_cursor('abc', '') end def test_ed_next_and_prev_char_for_mbchar input_keys('あいう') - assert_byte_pointer_size('あいう') - assert_cursor(6) - assert_cursor_max(6) + assert_line_around_cursor('あいう', '') input_keys("\C-b", false) - assert_byte_pointer_size('あい') - assert_cursor(4) - assert_cursor_max(6) + assert_line_around_cursor('あい', 'う') input_keys("\C-b", false) - assert_byte_pointer_size('あ') - assert_cursor(2) - assert_cursor_max(6) + assert_line_around_cursor('あ', 'いう') input_keys("\C-b", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(6) + assert_line_around_cursor('', 'あいう') input_keys("\C-b", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(6) + assert_line_around_cursor('', 'あいう') input_keys("\C-f", false) - assert_byte_pointer_size('あ') - assert_cursor(2) - assert_cursor_max(6) + assert_line_around_cursor('あ', 'いう') input_keys("\C-f", false) - assert_byte_pointer_size('あい') - assert_cursor(4) - assert_cursor_max(6) + assert_line_around_cursor('あい', 'う') input_keys("\C-f", false) - assert_byte_pointer_size('あいう') - assert_cursor(6) - assert_cursor_max(6) + assert_line_around_cursor('あいう', '') input_keys("\C-f", false) - assert_byte_pointer_size('あいう') - assert_cursor(6) - assert_cursor_max(6) + assert_line_around_cursor('あいう', '') end def test_ed_next_and_prev_char_for_mbchar_by_plural_code_points input_keys("か\u3099き\u3099く\u3099") - assert_byte_pointer_size("か\u3099き\u3099く\u3099") - assert_cursor(6) - assert_cursor_max(6) + assert_line_around_cursor("か\u3099き\u3099く\u3099", '') input_keys("\C-b", false) - assert_byte_pointer_size("か\u3099き\u3099") - assert_cursor(4) - assert_cursor_max(6) + assert_line_around_cursor("か\u3099き\u3099", "く\u3099") input_keys("\C-b", false) - assert_byte_pointer_size("か\u3099") - assert_cursor(2) - assert_cursor_max(6) + assert_line_around_cursor("か\u3099", "き\u3099く\u3099") input_keys("\C-b", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(6) + assert_line_around_cursor('', "か\u3099き\u3099く\u3099") input_keys("\C-b", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(6) + assert_line_around_cursor('', "か\u3099き\u3099く\u3099") input_keys("\C-f", false) - assert_byte_pointer_size("か\u3099") - assert_cursor(2) - assert_cursor_max(6) + assert_line_around_cursor("か\u3099", "き\u3099く\u3099") input_keys("\C-f", false) - assert_byte_pointer_size("か\u3099き\u3099") - assert_cursor(4) - assert_cursor_max(6) + assert_line_around_cursor("か\u3099き\u3099", "く\u3099") input_keys("\C-f", false) - assert_byte_pointer_size("か\u3099き\u3099く\u3099") - assert_cursor(6) - assert_cursor_max(6) + assert_line_around_cursor("か\u3099き\u3099く\u3099", '') input_keys("\C-f", false) - assert_byte_pointer_size("か\u3099き\u3099く\u3099") - assert_cursor(6) - assert_cursor_max(6) + assert_line_around_cursor("か\u3099き\u3099く\u3099", '') end def test_em_capitol_case input_keys('abc def{bbb}ccc') input_keys("\C-a\M-c", false) - assert_byte_pointer_size('Abc') - assert_cursor(3) - assert_cursor_max(15) - assert_line('Abc def{bbb}ccc') + assert_line_around_cursor('Abc', ' def{bbb}ccc') input_keys("\M-c", false) - assert_byte_pointer_size('Abc Def') - assert_cursor(7) - assert_cursor_max(15) - assert_line('Abc Def{bbb}ccc') + assert_line_around_cursor('Abc Def', '{bbb}ccc') input_keys("\M-c", false) - assert_byte_pointer_size('Abc Def{Bbb') - assert_cursor(11) - assert_cursor_max(15) - assert_line('Abc Def{Bbb}ccc') + assert_line_around_cursor('Abc Def{Bbb', '}ccc') input_keys("\M-c", false) - assert_byte_pointer_size('Abc Def{Bbb}Ccc') - assert_cursor(15) - assert_cursor_max(15) - assert_line('Abc Def{Bbb}Ccc') + assert_line_around_cursor('Abc Def{Bbb}Ccc', '') end def test_em_capitol_case_with_complex_example input_keys('{}#* AaA!!!cCc ') input_keys("\C-a\M-c", false) - assert_byte_pointer_size('{}#* Aaa') - assert_cursor(11) - assert_cursor_max(20) - assert_line('{}#* Aaa!!!cCc ') + assert_line_around_cursor('{}#* Aaa', '!!!cCc ') input_keys("\M-c", false) - assert_byte_pointer_size('{}#* Aaa!!!Ccc') - assert_cursor(17) - assert_cursor_max(20) - assert_line('{}#* Aaa!!!Ccc ') + assert_line_around_cursor('{}#* Aaa!!!Ccc', ' ') input_keys("\M-c", false) - assert_byte_pointer_size('{}#* Aaa!!!Ccc ') - assert_cursor(20) - assert_cursor_max(20) - assert_line('{}#* Aaa!!!Ccc ') + assert_line_around_cursor('{}#* Aaa!!!Ccc ', '') end def test_em_lower_case input_keys('AbC def{bBb}CCC') input_keys("\C-a\M-l", false) - assert_byte_pointer_size('abc') - assert_cursor(3) - assert_cursor_max(15) - assert_line('abc def{bBb}CCC') + assert_line_around_cursor('abc', ' def{bBb}CCC') input_keys("\M-l", false) - assert_byte_pointer_size('abc def') - assert_cursor(7) - assert_cursor_max(15) - assert_line('abc def{bBb}CCC') + assert_line_around_cursor('abc def', '{bBb}CCC') input_keys("\M-l", false) - assert_byte_pointer_size('abc def{bbb') - assert_cursor(11) - assert_cursor_max(15) - assert_line('abc def{bbb}CCC') + assert_line_around_cursor('abc def{bbb', '}CCC') input_keys("\M-l", false) - assert_byte_pointer_size('abc def{bbb}ccc') - assert_cursor(15) - assert_cursor_max(15) - assert_line('abc def{bbb}ccc') + assert_line_around_cursor('abc def{bbb}ccc', '') end def test_em_lower_case_with_complex_example input_keys('{}#* AaA!!!cCc ') input_keys("\C-a\M-l", false) - assert_byte_pointer_size('{}#* aaa') - assert_cursor(11) - assert_cursor_max(20) - assert_line('{}#* aaa!!!cCc ') + assert_line_around_cursor('{}#* aaa', '!!!cCc ') input_keys("\M-l", false) - assert_byte_pointer_size('{}#* aaa!!!ccc') - assert_cursor(17) - assert_cursor_max(20) - assert_line('{}#* aaa!!!ccc ') + assert_line_around_cursor('{}#* aaa!!!ccc', ' ') input_keys("\M-l", false) - assert_byte_pointer_size('{}#* aaa!!!ccc ') - assert_cursor(20) - assert_cursor_max(20) - assert_line('{}#* aaa!!!ccc ') + assert_line_around_cursor('{}#* aaa!!!ccc ', '') end def test_em_upper_case input_keys('AbC def{bBb}CCC') input_keys("\C-a\M-u", false) - assert_byte_pointer_size('ABC') - assert_cursor(3) - assert_cursor_max(15) - assert_line('ABC def{bBb}CCC') + assert_line_around_cursor('ABC', ' def{bBb}CCC') input_keys("\M-u", false) - assert_byte_pointer_size('ABC DEF') - assert_cursor(7) - assert_cursor_max(15) - assert_line('ABC DEF{bBb}CCC') + assert_line_around_cursor('ABC DEF', '{bBb}CCC') input_keys("\M-u", false) - assert_byte_pointer_size('ABC DEF{BBB') - assert_cursor(11) - assert_cursor_max(15) - assert_line('ABC DEF{BBB}CCC') + assert_line_around_cursor('ABC DEF{BBB', '}CCC') input_keys("\M-u", false) - assert_byte_pointer_size('ABC DEF{BBB}CCC') - assert_cursor(15) - assert_cursor_max(15) - assert_line('ABC DEF{BBB}CCC') + assert_line_around_cursor('ABC DEF{BBB}CCC', '') end def test_em_upper_case_with_complex_example input_keys('{}#* AaA!!!cCc ') input_keys("\C-a\M-u", false) - assert_byte_pointer_size('{}#* AAA') - assert_cursor(11) - assert_cursor_max(20) - assert_line('{}#* AAA!!!cCc ') + assert_line_around_cursor('{}#* AAA', '!!!cCc ') input_keys("\M-u", false) - assert_byte_pointer_size('{}#* AAA!!!CCC') - assert_cursor(17) - assert_cursor_max(20) - assert_line('{}#* AAA!!!CCC ') + assert_line_around_cursor('{}#* AAA!!!CCC', ' ') input_keys("\M-u", false) - assert_byte_pointer_size('{}#* AAA!!!CCC ') - assert_cursor(20) - assert_cursor_max(20) - assert_line('{}#* AAA!!!CCC ') + assert_line_around_cursor('{}#* AAA!!!CCC ', '') end def test_em_delete_or_list @@ -1286,28 +726,16 @@ def test_em_delete_or_list } } input_keys('fooo') - assert_byte_pointer_size('fooo') - assert_cursor(4) - assert_cursor_max(4) - assert_line('fooo') + assert_line_around_cursor('fooo', '') assert_equal(nil, @line_editor.instance_variable_get(:@menu_info)) input_keys("\C-b", false) - assert_byte_pointer_size('foo') - assert_cursor(3) - assert_cursor_max(4) - assert_line('fooo') + assert_line_around_cursor('foo', 'o') assert_equal(nil, @line_editor.instance_variable_get(:@menu_info)) @line_editor.input_key(Reline::Key.new(:em_delete_or_list, :em_delete_or_list, false)) - assert_byte_pointer_size('foo') - assert_cursor(3) - assert_cursor_max(3) - assert_line('foo') + assert_line_around_cursor('foo', '') assert_equal(nil, @line_editor.instance_variable_get(:@menu_info)) @line_editor.input_key(Reline::Key.new(:em_delete_or_list, :em_delete_or_list, false)) - assert_byte_pointer_size('foo') - assert_cursor(3) - assert_cursor_max(3) - assert_line('foo') + assert_line_around_cursor('foo', '') assert_equal(%w{foo_foo foo_bar foo_baz}, @line_editor.instance_variable_get(:@menu_info).list) end @@ -1322,22 +750,13 @@ def test_completion_duplicated_list } } input_keys('foo_') - assert_byte_pointer_size('foo_') - assert_cursor(4) - assert_cursor_max(4) - assert_line('foo_') + assert_line_around_cursor('foo_', '') assert_equal(nil, @line_editor.instance_variable_get(:@menu_info)) input_keys("\C-i", false) - assert_byte_pointer_size('foo_') - assert_cursor(4) - assert_cursor_max(4) - assert_line('foo_') + assert_line_around_cursor('foo_', '') assert_equal(nil, @line_editor.instance_variable_get(:@menu_info)) input_keys("\C-i", false) - assert_byte_pointer_size('foo_') - assert_cursor(4) - assert_cursor_max(4) - assert_line('foo_') + assert_line_around_cursor('foo_', '') assert_equal(%w{foo_foo foo_bar}, @line_editor.instance_variable_get(:@menu_info).list) end @@ -1353,36 +772,21 @@ def test_completion } } input_keys('fo') - assert_byte_pointer_size('fo') - assert_cursor(2) - assert_cursor_max(2) - assert_line('fo') + assert_line_around_cursor('fo', '') assert_equal(nil, @line_editor.instance_variable_get(:@menu_info)) input_keys("\C-i", false) - assert_byte_pointer_size('foo_') - assert_cursor(4) - assert_cursor_max(4) - assert_line('foo_') + assert_line_around_cursor('foo_', '') assert_equal(nil, @line_editor.instance_variable_get(:@menu_info)) input_keys("\C-i", false) - assert_byte_pointer_size('foo_') - assert_cursor(4) - assert_cursor_max(4) - assert_line('foo_') + assert_line_around_cursor('foo_', '') assert_equal(%w{foo_foo foo_bar foo_baz}, @line_editor.instance_variable_get(:@menu_info).list) input_keys('a') input_keys("\C-i", false) - assert_byte_pointer_size('foo_a') - assert_cursor(5) - assert_cursor_max(5) - assert_line('foo_a') + assert_line_around_cursor('foo_a', '') input_keys("\C-h", false) input_keys('b') input_keys("\C-i", false) - assert_byte_pointer_size('foo_ba') - assert_cursor(6) - assert_cursor_max(6) - assert_line('foo_ba') + assert_line_around_cursor('foo_ba', '') end def test_completion_with_indent @@ -1397,22 +801,13 @@ def test_completion_with_indent } } input_keys(' fo') - assert_byte_pointer_size(' fo') - assert_cursor(4) - assert_cursor_max(4) - assert_line(' fo') + assert_line_around_cursor(' fo', '') assert_equal(nil, @line_editor.instance_variable_get(:@menu_info)) input_keys("\C-i", false) - assert_byte_pointer_size(' foo_') - assert_cursor(6) - assert_cursor_max(6) - assert_line(' foo_') + assert_line_around_cursor(' foo_', '') assert_equal(nil, @line_editor.instance_variable_get(:@menu_info)) input_keys("\C-i", false) - assert_byte_pointer_size(' foo_') - assert_cursor(6) - assert_cursor_max(6) - assert_line(' foo_') + assert_line_around_cursor(' foo_', '') assert_equal(%w{foo_foo foo_bar foo_baz}, @line_editor.instance_variable_get(:@menu_info).list) end @@ -1428,22 +823,13 @@ def test_completion_with_indent_and_completer_quote_characters } } input_keys(' "".fo') - assert_byte_pointer_size(' "".fo') - assert_cursor(7) - assert_cursor_max(7) - assert_line(' "".fo') + assert_line_around_cursor(' "".fo', '') assert_equal(nil, @line_editor.instance_variable_get(:@menu_info)) input_keys("\C-i", false) - assert_byte_pointer_size(' "".foo_') - assert_cursor(9) - assert_cursor_max(9) - assert_line(' "".foo_') + assert_line_around_cursor(' "".foo_', '') assert_equal(nil, @line_editor.instance_variable_get(:@menu_info)) input_keys("\C-i", false) - assert_byte_pointer_size(' "".foo_') - assert_cursor(9) - assert_cursor_max(9) - assert_line(' "".foo_') + assert_line_around_cursor(' "".foo_', '') assert_equal(%w{"".foo_foo "".foo_bar "".foo_baz}, @line_editor.instance_variable_get(:@menu_info).list) end @@ -1461,54 +847,33 @@ def test_completion_with_perfect_match matched = m } input_keys('fo') - assert_byte_pointer_size('fo') - assert_cursor(2) - assert_cursor_max(2) - assert_line('fo') + assert_line_around_cursor('fo', '') assert_equal(Reline::LineEditor::CompletionState::NORMAL, @line_editor.instance_variable_get(:@completion_state)) assert_equal(nil, matched) input_keys("\C-i", false) - assert_byte_pointer_size('foo') - assert_cursor(3) - assert_cursor_max(3) - assert_line('foo') + assert_line_around_cursor('foo', '') assert_equal(Reline::LineEditor::CompletionState::MENU_WITH_PERFECT_MATCH, @line_editor.instance_variable_get(:@completion_state)) assert_equal(nil, matched) input_keys("\C-i", false) - assert_byte_pointer_size('foo') - assert_cursor(3) - assert_cursor_max(3) - assert_line('foo') + assert_line_around_cursor('foo', '') assert_equal(Reline::LineEditor::CompletionState::PERFECT_MATCH, @line_editor.instance_variable_get(:@completion_state)) assert_equal(nil, matched) input_keys("\C-i", false) - assert_byte_pointer_size('foo') - assert_cursor(3) - assert_cursor_max(3) - assert_line('foo') + assert_line_around_cursor('foo', '') assert_equal(Reline::LineEditor::CompletionState::PERFECT_MATCH, @line_editor.instance_variable_get(:@completion_state)) assert_equal('foo', matched) matched = nil input_keys('_') input_keys("\C-i", false) - assert_byte_pointer_size('foo_bar') - assert_cursor(7) - assert_cursor_max(7) - assert_line('foo_bar') + assert_line_around_cursor('foo_bar', '') assert_equal(Reline::LineEditor::CompletionState::MENU_WITH_PERFECT_MATCH, @line_editor.instance_variable_get(:@completion_state)) assert_equal(nil, matched) input_keys("\C-i", false) - assert_byte_pointer_size('foo_bar') - assert_cursor(7) - assert_cursor_max(7) - assert_line('foo_bar') + assert_line_around_cursor('foo_bar', '') assert_equal(Reline::LineEditor::CompletionState::PERFECT_MATCH, @line_editor.instance_variable_get(:@completion_state)) assert_equal(nil, matched) input_keys("\C-i", false) - assert_byte_pointer_size('foo_bar') - assert_cursor(7) - assert_cursor_max(7) - assert_line('foo_bar') + assert_line_around_cursor('foo_bar', '') assert_equal(Reline::LineEditor::CompletionState::PERFECT_MATCH, @line_editor.instance_variable_get(:@completion_state)) assert_equal('foo_bar', matched) end @@ -1525,43 +890,25 @@ def test_completion_with_completion_ignore_case } } input_keys('fo') - assert_byte_pointer_size('fo') - assert_cursor(2) - assert_cursor_max(2) - assert_line('fo') + assert_line_around_cursor('fo', '') assert_equal(nil, @line_editor.instance_variable_get(:@menu_info)) input_keys("\C-i", false) - assert_byte_pointer_size('foo_') - assert_cursor(4) - assert_cursor_max(4) - assert_line('foo_') + assert_line_around_cursor('foo_', '') assert_equal(nil, @line_editor.instance_variable_get(:@menu_info)) input_keys("\C-i", false) - assert_byte_pointer_size('foo_') - assert_cursor(4) - assert_cursor_max(4) - assert_line('foo_') + assert_line_around_cursor('foo_', '') assert_equal(%w{foo_foo foo_bar}, @line_editor.instance_variable_get(:@menu_info).list) @config.completion_ignore_case = true input_keys("\C-i", false) - assert_byte_pointer_size('foo_') - assert_cursor(4) - assert_cursor_max(4) - assert_line('foo_') + assert_line_around_cursor('foo_', '') assert_equal(%w{foo_foo foo_bar Foo_baz}, @line_editor.instance_variable_get(:@menu_info).list) input_keys('a') input_keys("\C-i", false) - assert_byte_pointer_size('foo_a') - assert_cursor(5) - assert_cursor_max(5) - assert_line('foo_a') + assert_line_around_cursor('foo_a', '') input_keys("\C-h", false) input_keys('b') input_keys("\C-i", false) - assert_byte_pointer_size('foo_ba') - assert_cursor(6) - assert_cursor_max(6) - assert_line('foo_ba') + assert_line_around_cursor('foo_ba', '') end def test_completion_in_middle_of_line @@ -1576,17 +923,11 @@ def test_completion_in_middle_of_line } } input_keys('abcde fo ABCDE') - assert_line('abcde fo ABCDE') + assert_line_around_cursor('abcde fo ABCDE', '') input_keys("\C-b" * 6 + "\C-i", false) - assert_byte_pointer_size('abcde foo_') - assert_cursor(10) - assert_cursor_max(16) - assert_line('abcde foo_ ABCDE') + assert_line_around_cursor('abcde foo_', ' ABCDE') input_keys("\C-b" * 2 + "\C-i", false) - assert_byte_pointer_size('abcde foo_') - assert_cursor(10) - assert_cursor_max(18) - assert_line('abcde foo_o_ ABCDE') + assert_line_around_cursor('abcde foo_', 'o_ ABCDE') end def test_completion_with_nil_value @@ -1602,125 +943,65 @@ def test_completion_with_nil_value } @config.completion_ignore_case = true input_keys('fo') - assert_byte_pointer_size('fo') - assert_cursor(2) - assert_cursor_max(2) - assert_line('fo') + assert_line_around_cursor('fo', '') assert_equal(nil, @line_editor.instance_variable_get(:@menu_info)) input_keys("\C-i", false) - assert_byte_pointer_size('foo_') - assert_cursor(4) - assert_cursor_max(4) - assert_line('foo_') + assert_line_around_cursor('foo_', '') assert_equal(nil, @line_editor.instance_variable_get(:@menu_info)) input_keys("\C-i", false) - assert_byte_pointer_size('foo_') - assert_cursor(4) - assert_cursor_max(4) - assert_line('foo_') + assert_line_around_cursor('foo_', '') assert_equal(%w{foo_foo foo_bar Foo_baz}, @line_editor.instance_variable_get(:@menu_info).list) input_keys('a') input_keys("\C-i", false) - assert_byte_pointer_size('foo_a') - assert_cursor(5) - assert_cursor_max(5) - assert_line('foo_a') + assert_line_around_cursor('foo_a', '') input_keys("\C-h", false) input_keys('b') input_keys("\C-i", false) - assert_byte_pointer_size('foo_ba') - assert_cursor(6) - assert_cursor_max(6) - assert_line('foo_ba') + assert_line_around_cursor('foo_ba', '') end def test_em_kill_region input_keys('abc def{bbb}ccc ddd ') - assert_byte_pointer_size('abc def{bbb}ccc ddd ') - assert_cursor(26) - assert_cursor_max(26) - assert_line('abc def{bbb}ccc ddd ') + assert_line_around_cursor('abc def{bbb}ccc ddd ', '') input_keys("\C-w", false) - assert_byte_pointer_size('abc def{bbb}ccc ') - assert_cursor(20) - assert_cursor_max(20) - assert_line('abc def{bbb}ccc ') + assert_line_around_cursor('abc def{bbb}ccc ', '') input_keys("\C-w", false) - assert_byte_pointer_size('abc ') - assert_cursor(6) - assert_cursor_max(6) - assert_line('abc ') + assert_line_around_cursor('abc ', '') input_keys("\C-w", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(0) - assert_line('') + assert_line_around_cursor('', '') input_keys("\C-w", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(0) - assert_line('') + assert_line_around_cursor('', '') end def test_em_kill_region_mbchar input_keys('あ い う{う}う ') - assert_byte_pointer_size('あ い う{う}う ') - assert_cursor(21) - assert_cursor_max(21) - assert_line('あ い う{う}う ') + assert_line_around_cursor('あ い う{う}う ', '') input_keys("\C-w", false) - assert_byte_pointer_size('あ い ') - assert_cursor(10) - assert_cursor_max(10) - assert_line('あ い ') + assert_line_around_cursor('あ い ', '') input_keys("\C-w", false) - assert_byte_pointer_size('あ ') - assert_cursor(5) - assert_cursor_max(5) - assert_line('あ ') + assert_line_around_cursor('あ ', '') input_keys("\C-w", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(0) - assert_line('') + assert_line_around_cursor('', '') end def test_vi_search_prev Reline::HISTORY.concat(%w{abc 123 AAA}) - assert_line('') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(0) + assert_line_around_cursor('', '') input_keys("\C-ra\C-j") - assert_line('abc') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(3) + assert_line_around_cursor('', 'abc') end def test_larger_histories_than_history_size history_size = @config.history_size @config.history_size = 2 Reline::HISTORY.concat(%w{abc 123 AAA}) - assert_line('') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(0) + assert_line_around_cursor('', '') input_keys("\C-p") - assert_line('AAA') - assert_byte_pointer_size('AAA') - assert_cursor(3) - assert_cursor_max(3) + assert_line_around_cursor('AAA', '') input_keys("\C-p") - assert_line('123') - assert_byte_pointer_size('123') - assert_cursor(3) - assert_cursor_max(3) + assert_line_around_cursor('123', '') input_keys("\C-p") - assert_line('123') - assert_byte_pointer_size('123') - assert_cursor(3) - assert_cursor_max(3) + assert_line_around_cursor('123', '') ensure @config.history_size = history_size end @@ -1731,25 +1012,13 @@ def test_search_history_to_back '12aa', '1234' # new ]) - assert_line('') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(0) + assert_line_around_cursor('', '') input_keys("\C-r123") - assert_line('1234') - assert_byte_pointer_size('1234') - assert_cursor(4) - assert_cursor_max(4) + assert_line_around_cursor('1234', '') input_keys("\C-ha") - assert_line('12aa') - assert_byte_pointer_size('12aa') - assert_cursor(4) - assert_cursor_max(4) + assert_line_around_cursor('12aa', '') input_keys("\C-h3") - assert_line('1235') - assert_byte_pointer_size('1235') - assert_cursor(4) - assert_cursor_max(4) + assert_line_around_cursor('1235', '') end def test_search_history_to_front @@ -1758,25 +1027,13 @@ def test_search_history_to_front '12aa', '1234' # new ]) - assert_line('') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(0) + assert_line_around_cursor('', '') input_keys("\C-s123") - assert_line('1235') - assert_byte_pointer_size('1235') - assert_cursor(4) - assert_cursor_max(4) + assert_line_around_cursor('1235', '') input_keys("\C-ha") - assert_line('12aa') - assert_byte_pointer_size('12aa') - assert_cursor(4) - assert_cursor_max(4) + assert_line_around_cursor('12aa', '') input_keys("\C-h3") - assert_line('1234') - assert_byte_pointer_size('1234') - assert_cursor(4) - assert_cursor_max(4) + assert_line_around_cursor('1234', '') end def test_search_history_front_and_back @@ -1785,30 +1042,15 @@ def test_search_history_front_and_back '12aa', '1234' # new ]) - assert_line('') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(0) + assert_line_around_cursor('', '') input_keys("\C-s12") - assert_line('1235') - assert_byte_pointer_size('1235') - assert_cursor(4) - assert_cursor_max(4) + assert_line_around_cursor('1235', '') input_keys("\C-s") - assert_line('12aa') - assert_byte_pointer_size('12aa') - assert_cursor(4) - assert_cursor_max(4) + assert_line_around_cursor('12aa', '') input_keys("\C-r") - assert_line('12aa') - assert_byte_pointer_size('12aa') - assert_cursor(4) - assert_cursor_max(4) + assert_line_around_cursor('12aa', '') input_keys("\C-r") - assert_line('1235') - assert_byte_pointer_size('1235') - assert_cursor(4) - assert_cursor_max(4) + assert_line_around_cursor('1235', '') end def test_search_history_back_and_front @@ -1817,30 +1059,15 @@ def test_search_history_back_and_front '12aa', '1234' # new ]) - assert_line('') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(0) + assert_line_around_cursor('', '') input_keys("\C-r12") - assert_line('1234') - assert_byte_pointer_size('1234') - assert_cursor(4) - assert_cursor_max(4) + assert_line_around_cursor('1234', '') input_keys("\C-r") - assert_line('12aa') - assert_byte_pointer_size('12aa') - assert_cursor(4) - assert_cursor_max(4) + assert_line_around_cursor('12aa', '') input_keys("\C-s") - assert_line('12aa') - assert_byte_pointer_size('12aa') - assert_cursor(4) - assert_cursor_max(4) + assert_line_around_cursor('12aa', '') input_keys("\C-s") - assert_line('1234') - assert_byte_pointer_size('1234') - assert_cursor(4) - assert_cursor_max(4) + assert_line_around_cursor('1234', '') end def test_search_history_to_back_in_the_middle_of_histories @@ -1849,20 +1076,11 @@ def test_search_history_to_back_in_the_middle_of_histories '12aa', '1234' # new ]) - assert_line('') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(0) + assert_line_around_cursor('', '') input_keys("\C-p\C-p") - assert_line('12aa') - assert_byte_pointer_size('12aa') - assert_cursor(4) - assert_cursor_max(4) + assert_line_around_cursor('12aa', '') input_keys("\C-r123") - assert_line('1235') - assert_byte_pointer_size('1235') - assert_cursor(4) - assert_cursor_max(4) + assert_line_around_cursor('1235', '') end def test_search_history_twice @@ -1871,20 +1089,11 @@ def test_search_history_twice '12aa', '1234' # new ]) - assert_line('') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(0) + assert_line_around_cursor('', '') input_keys("\C-r123") - assert_line('1234') - assert_byte_pointer_size('1234') - assert_cursor(4) - assert_cursor_max(4) + assert_line_around_cursor('1234', '') input_keys("\C-r") - assert_line('1235') - assert_byte_pointer_size('1235') - assert_cursor(4) - assert_cursor_max(4) + assert_line_around_cursor('1235', '') end def test_search_history_by_last_determined @@ -1893,35 +1102,17 @@ def test_search_history_by_last_determined '12aa', '1234' # new ]) - assert_line('') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(0) + assert_line_around_cursor('', '') input_keys("\C-r123") - assert_line('1234') - assert_byte_pointer_size('1234') - assert_cursor(4) - assert_cursor_max(4) + assert_line_around_cursor('1234', '') input_keys("\C-j") - assert_line('1234') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(4) + assert_line_around_cursor('', '1234') input_keys("\C-k") # delete - assert_line('') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(0) + assert_line_around_cursor('', '') input_keys("\C-r") - assert_line('') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(0) + assert_line_around_cursor('', '') input_keys("\C-r") - assert_line('1235') - assert_byte_pointer_size('1235') - assert_cursor(4) - assert_cursor_max(4) + assert_line_around_cursor('1235', '') end def test_search_history_with_isearch_terminator @@ -1933,76 +1124,40 @@ def test_search_history_with_isearch_terminator '12aa', '1234' # new ]) - assert_line('') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(0) + assert_line_around_cursor('', '') input_keys("\C-r12a") - assert_line('12aa') - assert_byte_pointer_size('12aa') - assert_cursor(4) - assert_cursor_max(4) + assert_line_around_cursor('12aa', '') input_keys('Y') - assert_line('12aa') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(4) + assert_line_around_cursor('', '12aa') input_keys('x') - assert_line('x12aa') - assert_byte_pointer_size('x') - assert_cursor(1) - assert_cursor_max(5) + assert_line_around_cursor('x', '12aa') end def test_em_set_mark_and_em_exchange_mark input_keys('aaa bbb ccc ddd') - assert_byte_pointer_size('aaa bbb ccc ddd') - assert_cursor(15) - assert_cursor_max(15) - assert_line('aaa bbb ccc ddd') + assert_line_around_cursor('aaa bbb ccc ddd', '') input_keys("\C-a\M-F\M-F", false) - assert_byte_pointer_size('aaa bbb') - assert_cursor(7) - assert_cursor_max(15) - assert_line('aaa bbb ccc ddd') + assert_line_around_cursor('aaa bbb', ' ccc ddd') assert_equal(nil, @line_editor.instance_variable_get(:@mark_pointer)) input_keys("\x00", false) # C-Space - assert_byte_pointer_size('aaa bbb') - assert_cursor(7) - assert_cursor_max(15) - assert_line('aaa bbb ccc ddd') + assert_line_around_cursor('aaa bbb', ' ccc ddd') assert_equal([7, 0], @line_editor.instance_variable_get(:@mark_pointer)) input_keys("\C-a", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(15) - assert_line('aaa bbb ccc ddd') + assert_line_around_cursor('', 'aaa bbb ccc ddd') assert_equal([7, 0], @line_editor.instance_variable_get(:@mark_pointer)) input_key_by_symbol(:em_exchange_mark) - assert_byte_pointer_size('aaa bbb') - assert_cursor(7) - assert_cursor_max(15) - assert_line('aaa bbb ccc ddd') + assert_line_around_cursor('aaa bbb', ' ccc ddd') assert_equal([0, 0], @line_editor.instance_variable_get(:@mark_pointer)) end def test_em_exchange_mark_without_mark input_keys('aaa bbb ccc ddd') - assert_byte_pointer_size('aaa bbb ccc ddd') - assert_cursor(15) - assert_cursor_max(15) - assert_line('aaa bbb ccc ddd') + assert_line_around_cursor('aaa bbb ccc ddd', '') input_keys("\C-a\M-f", false) - assert_byte_pointer_size('aaa') - assert_cursor(3) - assert_cursor_max(15) - assert_line('aaa bbb ccc ddd') + assert_line_around_cursor('aaa', ' bbb ccc ddd') assert_equal(nil, @line_editor.instance_variable_get(:@mark_pointer)) input_key_by_symbol(:em_exchange_mark) - assert_byte_pointer_size('aaa') - assert_cursor(3) - assert_cursor_max(15) - assert_line('aaa bbb ccc ddd') + assert_line_around_cursor('aaa', ' bbb ccc ddd') assert_equal(nil, @line_editor.instance_variable_get(:@mark_pointer)) end @@ -2031,20 +1186,11 @@ def test_ed_search_prev_history input_keys('123') # The ed_search_prev_history doesn't have default binding @line_editor.__send__(:ed_search_prev_history, "\C-p".ord) - assert_byte_pointer_size('123') - assert_cursor(3) - assert_cursor_max(5) - assert_line('12345') + assert_line_around_cursor('123', '45') @line_editor.__send__(:ed_search_prev_history, "\C-p".ord) - assert_byte_pointer_size('123') - assert_cursor(3) - assert_cursor_max(5) - assert_line('12356') + assert_line_around_cursor('123', '56') @line_editor.__send__(:ed_search_prev_history, "\C-p".ord) - assert_byte_pointer_size('123') - assert_cursor(3) - assert_cursor_max(5) - assert_line('12356') + assert_line_around_cursor('123', '56') end def test_ed_search_prev_history_with_empty @@ -2055,25 +1201,13 @@ def test_ed_search_prev_history_with_empty ]) # The ed_search_prev_history doesn't have default binding @line_editor.__send__(:ed_search_prev_history, "\C-p".ord) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(5) - assert_line('12345') + assert_line_around_cursor('', '12345') @line_editor.__send__(:ed_search_prev_history, "\C-p".ord) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(5) - assert_line('12aaa') + assert_line_around_cursor('', '12aaa') @line_editor.__send__(:ed_search_prev_history, "\C-p".ord) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(5) - assert_line('12356') + assert_line_around_cursor('', '12356') @line_editor.__send__(:ed_search_prev_history, "\C-p".ord) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(5) - assert_line('12356') + assert_line_around_cursor('', '12356') end def test_ed_search_prev_history_without_match @@ -2085,10 +1219,7 @@ def test_ed_search_prev_history_without_match input_keys('ABC') # The ed_search_prev_history doesn't have default binding @line_editor.__send__(:ed_search_prev_history, "\C-p".ord) - assert_byte_pointer_size('ABC') - assert_cursor(3) - assert_cursor_max(3) - assert_line('ABC') + assert_line_around_cursor('ABC', '') end def test_ed_search_next_history @@ -2100,30 +1231,15 @@ def test_ed_search_next_history input_keys('123') # The ed_search_prev_history and ed_search_next_history doesn't have default binding @line_editor.__send__(:ed_search_prev_history, "\C-p".ord) - assert_byte_pointer_size('123') - assert_cursor(3) - assert_cursor_max(5) - assert_line('12345') + assert_line_around_cursor('123', '45') @line_editor.__send__(:ed_search_prev_history, "\C-p".ord) - assert_byte_pointer_size('123') - assert_cursor(3) - assert_cursor_max(5) - assert_line('12356') + assert_line_around_cursor('123', '56') @line_editor.__send__(:ed_search_prev_history, "\C-p".ord) - assert_byte_pointer_size('123') - assert_cursor(3) - assert_cursor_max(5) - assert_line('12356') + assert_line_around_cursor('123', '56') @line_editor.__send__(:ed_search_next_history, "\C-n".ord) - assert_byte_pointer_size('123') - assert_cursor(3) - assert_cursor_max(5) - assert_line('12345') + assert_line_around_cursor('123', '45') @line_editor.__send__(:ed_search_next_history, "\C-n".ord) - assert_byte_pointer_size('123') - assert_cursor(3) - assert_cursor_max(5) - assert_line('12345') + assert_line_around_cursor('123', '45') end def test_ed_search_next_history_with_empty @@ -2134,35 +1250,17 @@ def test_ed_search_next_history_with_empty ]) # The ed_search_prev_history and ed_search_next_history doesn't have default binding @line_editor.__send__(:ed_search_prev_history, "\C-p".ord) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(5) - assert_line('12345') + assert_line_around_cursor('', '12345') @line_editor.__send__(:ed_search_prev_history, "\C-p".ord) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(5) - assert_line('12aaa') + assert_line_around_cursor('', '12aaa') @line_editor.__send__(:ed_search_prev_history, "\C-p".ord) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(5) - assert_line('12356') + assert_line_around_cursor('', '12356') @line_editor.__send__(:ed_search_next_history, "\C-n".ord) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(5) - assert_line('12aaa') + assert_line_around_cursor('', '12aaa') @line_editor.__send__(:ed_search_next_history, "\C-n".ord) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(5) - assert_line('12345') + assert_line_around_cursor('', '12345') @line_editor.__send__(:ed_search_next_history, "\C-n".ord) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(0) - assert_line('') + assert_line_around_cursor('', '') end # Unicode emoji test @@ -2170,97 +1268,49 @@ def test_ed_insert_for_include_zwj_emoji omit "This test is for UTF-8 but the locale is #{Reline.core.encoding}" if Reline.core.encoding != Encoding::UTF_8 # U+1F468 U+200D U+1F469 U+200D U+1F467 U+200D U+1F466 is family: man, woman, girl, boy "👨‍👩‍👧‍👦" input_keys("\u{1F468}") # U+1F468 is man "👨" - assert_line("\u{1F468}") - assert_byte_pointer_size("\u{1F468}") - assert_cursor(2) - assert_cursor_max(2) + assert_line_around_cursor('👨', '') input_keys("\u200D") # U+200D is ZERO WIDTH JOINER - assert_line("\u{1F468 200D}") - assert_byte_pointer_size("\u{1F468 200D}") - assert_cursor(2) - assert_cursor_max(2) + assert_line_around_cursor('👨‍', '') input_keys("\u{1F469}") # U+1F469 is woman "👩" - assert_line("\u{1F468 200D 1F469}") - assert_byte_pointer_size("\u{1F468 200D 1F469}") - assert_cursor(2) - assert_cursor_max(2) + assert_line_around_cursor('👨‍👩', '') input_keys("\u200D") # U+200D is ZERO WIDTH JOINER - assert_line("\u{1F468 200D 1F469 200D}") - assert_byte_pointer_size("\u{1F468 200D 1F469 200D}") - assert_cursor(2) - assert_cursor_max(2) + assert_line_around_cursor('👨‍👩‍', '') input_keys("\u{1F467}") # U+1F467 is girl "👧" - assert_line("\u{1F468 200D 1F469 200D 1F467}") - assert_byte_pointer_size("\u{1F468 200D 1F469 200D 1F467}") - assert_cursor(2) - assert_cursor_max(2) + assert_line_around_cursor('👨‍👩‍👧', '') input_keys("\u200D") # U+200D is ZERO WIDTH JOINER - assert_line("\u{1F468 200D 1F469 200D 1F467 200D}") - assert_byte_pointer_size("\u{1F468 200D 1F469 200D 1F467 200D}") - assert_cursor(2) - assert_cursor_max(2) + assert_line_around_cursor('👨‍👩‍👧‍', '') input_keys("\u{1F466}") # U+1F466 is boy "👦" - assert_line("\u{1F468 200D 1F469 200D 1F467 200D 1F466}") - assert_byte_pointer_size("\u{1F468 200D 1F469 200D 1F467 200D 1F466}") - assert_cursor(2) - assert_cursor_max(2) + assert_line_around_cursor('👨‍👩‍👧‍👦', '') # U+1F468 U+200D U+1F469 U+200D U+1F467 U+200D U+1F466 is family: man, woman, girl, boy "👨‍👩‍👧‍👦" input_keys("\u{1F468 200D 1F469 200D 1F467 200D 1F466}") - assert_line("\u{1F468 200D 1F469 200D 1F467 200D 1F466 1F468 200D 1F469 200D 1F467 200D 1F466}") - assert_byte_pointer_size("\u{1F468 200D 1F469 200D 1F467 200D 1F466 1F468 200D 1F469 200D 1F467 200D 1F466}") - assert_cursor(4) - assert_cursor_max(4) + assert_line_around_cursor('👨‍👩‍👧‍👦👨‍👩‍👧‍👦', '') end def test_ed_insert_for_include_valiation_selector omit "This test is for UTF-8 but the locale is #{Reline.core.encoding}" if Reline.core.encoding != Encoding::UTF_8 # U+0030 U+FE00 is DIGIT ZERO + VARIATION SELECTOR-1 "0︀" input_keys("\u0030") # U+0030 is DIGIT ZERO - assert_line("\u0030") - assert_byte_pointer_size("\u0030") - assert_cursor(1) - assert_cursor_max(1) + assert_line_around_cursor('0', '') input_keys("\uFE00") # U+FE00 is VARIATION SELECTOR-1 - assert_line("\u{0030 FE00}") - assert_byte_pointer_size("\u{0030 FE00}") - assert_cursor(1) - assert_cursor_max(1) + assert_line_around_cursor('0︀', '') end def test_em_yank_pop input_keys("def hoge\C-w\C-b\C-f\C-w", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(0) - assert_line('') + assert_line_around_cursor('', '') input_keys("\C-y", false) - assert_byte_pointer_size('def ') - assert_cursor(4) - assert_cursor_max(4) - assert_line('def ') + assert_line_around_cursor('def ', '') input_keys("\M-\C-y", false) - assert_byte_pointer_size('hoge') - assert_cursor(4) - assert_cursor_max(4) - assert_line('hoge') + assert_line_around_cursor('hoge', '') end def test_em_kill_region_with_kill_ring input_keys("def hoge\C-b\C-b\C-b\C-b", false) - assert_byte_pointer_size('def ') - assert_cursor(4) - assert_cursor_max(8) - assert_line('def hoge') + assert_line_around_cursor('def ', 'hoge') input_keys("\C-k\C-w", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(0) - assert_line('') + assert_line_around_cursor('', '') input_keys("\C-y", false) - assert_byte_pointer_size('def hoge') - assert_cursor(8) - assert_cursor_max(8) - assert_line('def hoge') + assert_line_around_cursor('def hoge', '') end def test_ed_search_prev_next_history_in_multibyte @@ -2276,104 +1326,60 @@ def test_ed_search_prev_next_history_in_multibyte assert_whole_lines(['def foo', ' 12345', 'end']) assert_line_index(1) assert_whole_lines(['def foo', ' 12345', 'end']) - assert_byte_pointer_size(' 123') - assert_cursor(5) - assert_cursor_max(7) - assert_line(' 12345') + assert_line_around_cursor(' 123', '45') @line_editor.__send__(:ed_search_prev_history, "\C-p".ord) assert_line_index(2) assert_whole_lines(['def hoge', ' 67890', ' 12345', 'end']) - assert_byte_pointer_size(' 123') - assert_cursor(5) - assert_cursor_max(7) - assert_line(' 12345') + assert_line_around_cursor(' 123', '45') @line_editor.__send__(:ed_search_prev_history, "\C-p".ord) assert_line_index(2) assert_whole_lines(['def hoge', ' 67890', ' 12345', 'end']) - assert_byte_pointer_size(' 123') - assert_cursor(5) - assert_cursor_max(7) - assert_line(' 12345') + assert_line_around_cursor(' 123', '45') @line_editor.__send__(:ed_search_next_history, "\C-n".ord) assert_line_index(1) assert_whole_lines(['def foo', ' 12345', 'end']) - assert_byte_pointer_size(' 123') - assert_cursor(5) - assert_cursor_max(7) - assert_line(' 12345') + assert_line_around_cursor(' 123', '45') @line_editor.__send__(:ed_search_next_history, "\C-n".ord) assert_line_index(1) assert_whole_lines(['def foo', ' 12345', 'end']) - assert_byte_pointer_size(' 123') - assert_cursor(5) - assert_cursor_max(7) - assert_line(' 12345') + assert_line_around_cursor(' 123', '45') end def test_ignore_NUL_by_ed_quoted_insert input_keys(%Q{"\C-v\C-@"}, false) - assert_byte_pointer_size('""') - assert_cursor(2) - assert_cursor_max(2) + assert_line_around_cursor('""', '') end def test_ed_argument_digit_by_meta_num input_keys('abcdef') - assert_byte_pointer_size('abcdef') - assert_cursor(6) - assert_cursor_max(6) - assert_line('abcdef') + assert_line_around_cursor('abcdef', '') input_keys("\M-2", false) input_keys("\C-h", false) - assert_byte_pointer_size('abcd') - assert_cursor(4) - assert_cursor_max(4) - assert_line('abcd') + assert_line_around_cursor('abcd', '') end def test_halfwidth_kana_width_dakuten input_raw_keys('ｶﾞｷﾞｹﾞｺﾞ') - assert_byte_pointer_size('ｶﾞｷﾞｹﾞｺﾞ') - assert_cursor(8) - assert_cursor_max(8) + assert_line_around_cursor('ｶﾞｷﾞｹﾞｺﾞ', '') input_keys("\C-b\C-b", false) - assert_byte_pointer_size('ｶﾞｷﾞ') - assert_cursor(4) - assert_cursor_max(8) + assert_line_around_cursor('ｶﾞｷﾞ', 'ｹﾞｺﾞ') input_raw_keys('ｸﾞ', false) - assert_byte_pointer_size('ｶﾞｷﾞｸﾞ') - assert_cursor(6) - assert_cursor_max(10) - assert_line('ｶﾞｷﾞｸﾞｹﾞｺﾞ') + assert_line_around_cursor('ｶﾞｷﾞｸﾞ', 'ｹﾞｺﾞ') end def test_input_unknown_char input_keys('͸') # U+0378 (unassigned) - assert_line('͸') - assert_byte_pointer_size('͸') - assert_cursor(1) - assert_cursor_max(1) + assert_line_around_cursor('͸', '') end def test_unix_line_discard input_keys("\C-u", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(0) - assert_line('') + assert_line_around_cursor('', '') input_keys('abc') - assert_byte_pointer_size('abc') - assert_cursor(3) - assert_cursor_max(3) + assert_line_around_cursor('abc', '') input_keys("\C-b\C-u", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(1) - assert_line('c') + assert_line_around_cursor('', 'c') input_keys("\C-f\C-u", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(0) - assert_line('') + assert_line_around_cursor('', '') end end diff --git a/test/reline/test_key_actor_vi.rb b/test/reline/test_key_actor_vi.rb index 7c924d19633bf8..0db24c6be7685d 100644 --- a/test/reline/test_key_actor_vi.rb +++ b/test/reline/test_key_actor_vi.rb @@ -25,950 +25,513 @@ def test_vi_command_mode def test_vi_command_mode_with_input input_keys("abc\C-[") assert_instance_of(Reline::KeyActor::ViCommand, @config.editing_mode) - assert_line('abc') + assert_line_around_cursor('ab', 'c') end def test_vi_insert assert_instance_of(Reline::KeyActor::ViInsert, @config.editing_mode) input_keys('i') - assert_line('i') - assert_cursor(1) + assert_line_around_cursor('i', '') assert_instance_of(Reline::KeyActor::ViInsert, @config.editing_mode) input_keys("\C-[") - assert_line('i') - assert_cursor(0) + assert_line_around_cursor('', 'i') assert_instance_of(Reline::KeyActor::ViCommand, @config.editing_mode) input_keys('i') - assert_line('i') - assert_cursor(0) + assert_line_around_cursor('', 'i') assert_instance_of(Reline::KeyActor::ViInsert, @config.editing_mode) end def test_vi_add assert_instance_of(Reline::KeyActor::ViInsert, @config.editing_mode) input_keys('a') - assert_line('a') - assert_cursor(1) + assert_line_around_cursor('a', '') assert_instance_of(Reline::KeyActor::ViInsert, @config.editing_mode) input_keys("\C-[") - assert_line('a') - assert_cursor(0) + assert_line_around_cursor('', 'a') assert_instance_of(Reline::KeyActor::ViCommand, @config.editing_mode) input_keys('a') - assert_line('a') - assert_cursor(1) + assert_line_around_cursor('a', '') assert_instance_of(Reline::KeyActor::ViInsert, @config.editing_mode) end def test_vi_insert_at_bol input_keys('I') - assert_line('I') + assert_line_around_cursor('I', '') assert_instance_of(Reline::KeyActor::ViInsert, @config.editing_mode) input_keys("12345\C-[hh") - assert_line('I12345') - assert_byte_pointer_size('I12') - assert_cursor(3) - assert_cursor_max(6) + assert_line_around_cursor('I12', '345') assert_instance_of(Reline::KeyActor::ViCommand, @config.editing_mode) input_keys('I') - assert_line('I12345') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(6) + assert_line_around_cursor('', 'I12345') assert_instance_of(Reline::KeyActor::ViInsert, @config.editing_mode) end def test_vi_add_at_eol input_keys('A') - assert_line('A') + assert_line_around_cursor('A', '') assert_instance_of(Reline::KeyActor::ViInsert, @config.editing_mode) input_keys("12345\C-[hh") - assert_line('A12345') - assert_byte_pointer_size('A12') - assert_cursor(3) - assert_cursor_max(6) + assert_line_around_cursor('A12', '345') assert_instance_of(Reline::KeyActor::ViCommand, @config.editing_mode) input_keys('A') - assert_line('A12345') - assert_byte_pointer_size('A12345') - assert_cursor(6) - assert_cursor_max(6) + assert_line_around_cursor('A12345', '') assert_instance_of(Reline::KeyActor::ViInsert, @config.editing_mode) end def test_ed_insert_one input_keys('a') - assert_line('a') - assert_byte_pointer_size('a') - assert_cursor(1) - assert_cursor_max(1) + assert_line_around_cursor('a', '') end def test_ed_insert_two input_keys('ab') - assert_line('ab') - assert_byte_pointer_size('ab') - assert_cursor(2) - assert_cursor_max(2) + assert_line_around_cursor('ab', '') end def test_ed_insert_mbchar_one input_keys('か') - assert_line('か') - assert_byte_pointer_size('か') - assert_cursor(2) - assert_cursor_max(2) + assert_line_around_cursor('か', '') end def test_ed_insert_mbchar_two input_keys('かき') - assert_line('かき') - assert_byte_pointer_size('かき') - assert_cursor(4) - assert_cursor_max(4) + assert_line_around_cursor('かき', '') end def test_ed_insert_for_mbchar_by_plural_code_points input_keys("か\u3099") - assert_line("か\u3099") - assert_byte_pointer_size("か\u3099") - assert_cursor(2) - assert_cursor_max(2) + assert_line_around_cursor("か\u3099", '') end def test_ed_insert_for_plural_mbchar_by_plural_code_points input_keys("か\u3099き\u3099") - assert_line("か\u3099き\u3099") - assert_byte_pointer_size("か\u3099き\u3099") - assert_cursor(4) - assert_cursor_max(4) + assert_line_around_cursor("か\u3099き\u3099", '') end def test_ed_next_char input_keys("abcdef\C-[0") - assert_line('abcdef') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(6) + assert_line_around_cursor('', 'abcdef') input_keys('l') - assert_line('abcdef') - assert_byte_pointer_size('a') - assert_cursor(1) - assert_cursor_max(6) + assert_line_around_cursor('a', 'bcdef') input_keys('2l') - assert_line('abcdef') - assert_byte_pointer_size('abc') - assert_cursor(3) - assert_cursor_max(6) + assert_line_around_cursor('abc', 'def') end def test_ed_prev_char input_keys("abcdef\C-[") - assert_line('abcdef') - assert_byte_pointer_size('abcde') - assert_cursor(5) - assert_cursor_max(6) + assert_line_around_cursor('abcde', 'f') input_keys('h') - assert_line('abcdef') - assert_byte_pointer_size('abcd') - assert_cursor(4) - assert_cursor_max(6) + assert_line_around_cursor('abcd', 'ef') input_keys('2h') - assert_line('abcdef') - assert_byte_pointer_size('ab') - assert_cursor(2) - assert_cursor_max(6) + assert_line_around_cursor('ab', 'cdef') end def test_history Reline::HISTORY.concat(%w{abc 123 AAA}) input_keys("\C-[") - assert_line('') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(0) + assert_line_around_cursor('', '') input_keys('k') - assert_line('AAA') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(3) + assert_line_around_cursor('', 'AAA') input_keys('2k') - assert_line('abc') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(3) + assert_line_around_cursor('', 'abc') input_keys('j') - assert_line('123') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(3) + assert_line_around_cursor('', '123') input_keys('2j') - assert_line('') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(0) + assert_line_around_cursor('', '') end def test_vi_paste_prev input_keys("abcde\C-[3h") - assert_line('abcde') - assert_byte_pointer_size('a') - assert_cursor(1) - assert_cursor_max(5) + assert_line_around_cursor('a', 'bcde') input_keys('P') - assert_line('abcde') - assert_byte_pointer_size('a') - assert_cursor(1) - assert_cursor_max(5) + assert_line_around_cursor('a', 'bcde') input_keys('d$') - assert_line('a') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(1) + assert_line_around_cursor('', 'a') input_keys('P') - assert_line('bcdea') - assert_byte_pointer_size('bcd') - assert_cursor(3) - assert_cursor_max(5) + assert_line_around_cursor('bcd', 'ea') input_keys('2P') - assert_line('bcdbcdbcdeeea') - assert_byte_pointer_size('bcdbcdbcd') - assert_cursor(9) - assert_cursor_max(13) + assert_line_around_cursor('bcdbcdbcd', 'eeea') end def test_vi_paste_next input_keys("abcde\C-[3h") - assert_line('abcde') - assert_byte_pointer_size('a') - assert_cursor(1) - assert_cursor_max(5) + assert_line_around_cursor('a', 'bcde') input_keys('p') - assert_line('abcde') - assert_byte_pointer_size('a') - assert_cursor(1) - assert_cursor_max(5) + assert_line_around_cursor('a', 'bcde') input_keys('d$') - assert_line('a') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(1) + assert_line_around_cursor('', 'a') input_keys('p') - assert_line('abcde') - assert_byte_pointer_size('abcd') - assert_cursor(4) - assert_cursor_max(5) + assert_line_around_cursor('abcd', 'e') input_keys('2p') - assert_line('abcdebcdebcde') - assert_byte_pointer_size('abcdebcdebcd') - assert_cursor(12) - assert_cursor_max(13) + assert_line_around_cursor('abcdebcdebcd', 'e') end def test_vi_paste_prev_for_mbchar input_keys("あいうえお\C-[3h") - assert_line('あいうえお') - assert_byte_pointer_size('あ') - assert_cursor(2) - assert_cursor_max(10) + assert_line_around_cursor('あ', 'いうえお') input_keys('P') - assert_line('あいうえお') - assert_byte_pointer_size('あ') - assert_cursor(2) - assert_cursor_max(10) + assert_line_around_cursor('あ', 'いうえお') input_keys('d$') - assert_line('あ') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(2) + assert_line_around_cursor('', 'あ') input_keys('P') - assert_line('いうえおあ') - assert_byte_pointer_size('いうえ') - assert_cursor(6) - assert_cursor_max(10) + assert_line_around_cursor('いうえ', 'おあ') input_keys('2P') - assert_line('いうえいうえいうえおおおあ') - assert_byte_pointer_size('いうえいうえいうえ') - assert_cursor(18) - assert_cursor_max(26) + assert_line_around_cursor('いうえいうえいうえ', 'おおおあ') end def test_vi_paste_next_for_mbchar input_keys("あいうえお\C-[3h") - assert_line('あいうえお') - assert_byte_pointer_size('あ') - assert_cursor(2) - assert_cursor_max(10) + assert_line_around_cursor('あ', 'いうえお') input_keys('p') - assert_line('あいうえお') - assert_byte_pointer_size('あ') - assert_cursor(2) - assert_cursor_max(10) + assert_line_around_cursor('あ', 'いうえお') input_keys('d$') - assert_line('あ') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(2) + assert_line_around_cursor('', 'あ') input_keys('p') - assert_line('あいうえお') - assert_byte_pointer_size('あいうえ') - assert_cursor(8) - assert_cursor_max(10) + assert_line_around_cursor('あいうえ', 'お') input_keys('2p') - assert_line('あいうえおいうえおいうえお') - assert_byte_pointer_size('あいうえおいうえおいうえ') - assert_cursor(24) - assert_cursor_max(26) + assert_line_around_cursor('あいうえおいうえおいうえ', 'お') end def test_vi_paste_prev_for_mbchar_by_plural_code_points input_keys("か\u3099き\u3099く\u3099け\u3099こ\u3099\C-[3h") - assert_line("か\u3099き\u3099く\u3099け\u3099こ\u3099") - assert_byte_pointer_size("か\u3099") - assert_cursor(2) - assert_cursor_max(10) + assert_line_around_cursor("か\u3099", "き\u3099く\u3099け\u3099こ\u3099") input_keys('P') - assert_line("か\u3099き\u3099く\u3099け\u3099こ\u3099") - assert_byte_pointer_size("か\u3099") - assert_cursor(2) - assert_cursor_max(10) + assert_line_around_cursor("か\u3099", "き\u3099く\u3099け\u3099こ\u3099") input_keys('d$') - assert_line("か\u3099") - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(2) + assert_line_around_cursor('', "か\u3099") input_keys('P') - assert_line("き\u3099く\u3099け\u3099こ\u3099か\u3099") - assert_byte_pointer_size("き\u3099く\u3099け\u3099") - assert_cursor(6) - assert_cursor_max(10) + assert_line_around_cursor("き\u3099く\u3099け\u3099", "こ\u3099か\u3099") input_keys('2P') - assert_line("き\u3099く\u3099け\u3099き\u3099く\u3099け\u3099き\u3099く\u3099け\u3099こ\u3099こ\u3099こ\u3099か\u3099") - assert_byte_pointer_size("き\u3099く\u3099け\u3099き\u3099く\u3099け\u3099き\u3099く\u3099け\u3099") - assert_cursor(18) - assert_cursor_max(26) + assert_line_around_cursor("き\u3099く\u3099け\u3099き\u3099く\u3099け\u3099き\u3099く\u3099け\u3099", "こ\u3099こ\u3099こ\u3099か\u3099") end def test_vi_paste_next_for_mbchar_by_plural_code_points input_keys("か\u3099き\u3099く\u3099け\u3099こ\u3099\C-[3h") - assert_line("か\u3099き\u3099く\u3099け\u3099こ\u3099") - assert_byte_pointer_size("か\u3099") - assert_cursor(2) - assert_cursor_max(10) + assert_line_around_cursor("か\u3099", "き\u3099く\u3099け\u3099こ\u3099") input_keys('p') - assert_line("か\u3099き\u3099く\u3099け\u3099こ\u3099") - assert_byte_pointer_size("か\u3099") - assert_cursor(2) - assert_cursor_max(10) + assert_line_around_cursor("か\u3099", "き\u3099く\u3099け\u3099こ\u3099") input_keys('d$') - assert_line("か\u3099") - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(2) + assert_line_around_cursor('', "か\u3099") input_keys('p') - assert_line("か\u3099き\u3099く\u3099け\u3099こ\u3099") - assert_byte_pointer_size("か\u3099き\u3099く\u3099け\u3099") - assert_cursor(8) - assert_cursor_max(10) + assert_line_around_cursor("か\u3099き\u3099く\u3099け\u3099", "こ\u3099") input_keys('2p') - assert_line("か\u3099き\u3099く\u3099け\u3099こ\u3099き\u3099く\u3099け\u3099こ\u3099き\u3099く\u3099け\u3099こ\u3099") - assert_byte_pointer_size("か\u3099き\u3099く\u3099け\u3099こ\u3099き\u3099く\u3099け\u3099こ\u3099き\u3099く\u3099け\u3099") - assert_cursor(24) - assert_cursor_max(26) + assert_line_around_cursor("か\u3099き\u3099く\u3099け\u3099こ\u3099き\u3099く\u3099け\u3099こ\u3099き\u3099く\u3099け\u3099", "こ\u3099") end def test_vi_prev_next_word input_keys("aaa b{b}b ccc\C-[0") - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(13) + assert_line_around_cursor('', 'aaa b{b}b ccc') input_keys('w') - assert_byte_pointer_size('aaa ') - assert_cursor(4) - assert_cursor_max(13) + assert_line_around_cursor('aaa ', 'b{b}b ccc') input_keys('w') - assert_byte_pointer_size('aaa b') - assert_cursor(5) - assert_cursor_max(13) + assert_line_around_cursor('aaa b', '{b}b ccc') input_keys('w') - assert_byte_pointer_size('aaa b{') - assert_cursor(6) - assert_cursor_max(13) + assert_line_around_cursor('aaa b{', 'b}b ccc') input_keys('w') - assert_byte_pointer_size('aaa b{b') - assert_cursor(7) - assert_cursor_max(13) + assert_line_around_cursor('aaa b{b', '}b ccc') input_keys('w') - assert_byte_pointer_size('aaa b{b}') - assert_cursor(8) - assert_cursor_max(13) + assert_line_around_cursor('aaa b{b}', 'b ccc') input_keys('w') - assert_byte_pointer_size('aaa b{b}b ') - assert_cursor(10) - assert_cursor_max(13) + assert_line_around_cursor('aaa b{b}b ', 'ccc') input_keys('w') - assert_byte_pointer_size('aaa b{b}b cc') - assert_cursor(12) - assert_cursor_max(13) + assert_line_around_cursor('aaa b{b}b cc', 'c') input_keys('b') - assert_byte_pointer_size('aaa b{b}b ') - assert_cursor(10) - assert_cursor_max(13) + assert_line_around_cursor('aaa b{b}b ', 'ccc') input_keys('b') - assert_byte_pointer_size('aaa b{b}') - assert_cursor(8) - assert_cursor_max(13) + assert_line_around_cursor('aaa b{b}', 'b ccc') input_keys('b') - assert_byte_pointer_size('aaa b{b') - assert_cursor(7) - assert_cursor_max(13) + assert_line_around_cursor('aaa b{b', '}b ccc') input_keys('b') - assert_byte_pointer_size('aaa b{') - assert_cursor(6) - assert_cursor_max(13) + assert_line_around_cursor('aaa b{', 'b}b ccc') input_keys('b') - assert_byte_pointer_size('aaa b') - assert_cursor(5) - assert_cursor_max(13) + assert_line_around_cursor('aaa b', '{b}b ccc') input_keys('b') - assert_byte_pointer_size('aaa ') - assert_cursor(4) - assert_cursor_max(13) + assert_line_around_cursor('aaa ', 'b{b}b ccc') input_keys('b') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(13) + assert_line_around_cursor('', 'aaa b{b}b ccc') input_keys('3w') - assert_byte_pointer_size('aaa b{') - assert_cursor(6) - assert_cursor_max(13) + assert_line_around_cursor('aaa b{', 'b}b ccc') input_keys('3w') - assert_byte_pointer_size('aaa b{b}b ') - assert_cursor(10) - assert_cursor_max(13) + assert_line_around_cursor('aaa b{b}b ', 'ccc') input_keys('3w') - assert_byte_pointer_size('aaa b{b}b cc') - assert_cursor(12) - assert_cursor_max(13) + assert_line_around_cursor('aaa b{b}b cc', 'c') input_keys('3b') - assert_byte_pointer_size('aaa b{b') - assert_cursor(7) - assert_cursor_max(13) + assert_line_around_cursor('aaa b{b', '}b ccc') input_keys('3b') - assert_byte_pointer_size('aaa ') - assert_cursor(4) - assert_cursor_max(13) + assert_line_around_cursor('aaa ', 'b{b}b ccc') input_keys('3b') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(13) + assert_line_around_cursor('', 'aaa b{b}b ccc') end def test_vi_end_word input_keys("aaa b{b}}}b ccc\C-[0") - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(19) + assert_line_around_cursor('', 'aaa b{b}}}b ccc') input_keys('e') - assert_byte_pointer_size('aa') - assert_cursor(2) - assert_cursor_max(19) + assert_line_around_cursor('aa', 'a b{b}}}b ccc') input_keys('e') - assert_byte_pointer_size('aaa ') - assert_cursor(6) - assert_cursor_max(19) + assert_line_around_cursor('aaa ', 'b{b}}}b ccc') input_keys('e') - assert_byte_pointer_size('aaa b') - assert_cursor(7) - assert_cursor_max(19) + assert_line_around_cursor('aaa b', '{b}}}b ccc') input_keys('e') - assert_byte_pointer_size('aaa b{') - assert_cursor(8) - assert_cursor_max(19) + assert_line_around_cursor('aaa b{', 'b}}}b ccc') input_keys('e') - assert_byte_pointer_size('aaa b{b}}') - assert_cursor(11) - assert_cursor_max(19) + assert_line_around_cursor('aaa b{b}}', '}b ccc') input_keys('e') - assert_byte_pointer_size('aaa b{b}}}') - assert_cursor(12) - assert_cursor_max(19) + assert_line_around_cursor('aaa b{b}}}', 'b ccc') input_keys('e') - assert_byte_pointer_size('aaa b{b}}}b cc') - assert_cursor(18) - assert_cursor_max(19) + assert_line_around_cursor('aaa b{b}}}b cc', 'c') input_keys('e') - assert_byte_pointer_size('aaa b{b}}}b cc') - assert_cursor(18) - assert_cursor_max(19) + assert_line_around_cursor('aaa b{b}}}b cc', 'c') input_keys('03e') - assert_byte_pointer_size('aaa b') - assert_cursor(7) - assert_cursor_max(19) + assert_line_around_cursor('aaa b', '{b}}}b ccc') input_keys('3e') - assert_byte_pointer_size('aaa b{b}}}') - assert_cursor(12) - assert_cursor_max(19) + assert_line_around_cursor('aaa b{b}}}', 'b ccc') input_keys('3e') - assert_byte_pointer_size('aaa b{b}}}b cc') - assert_cursor(18) - assert_cursor_max(19) + assert_line_around_cursor('aaa b{b}}}b cc', 'c') end def test_vi_prev_next_big_word input_keys("aaa b{b}b ccc\C-[0") - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(13) + assert_line_around_cursor('', 'aaa b{b}b ccc') input_keys('W') - assert_byte_pointer_size('aaa ') - assert_cursor(4) - assert_cursor_max(13) + assert_line_around_cursor('aaa ', 'b{b}b ccc') input_keys('W') - assert_byte_pointer_size('aaa b{b}b ') - assert_cursor(10) - assert_cursor_max(13) + assert_line_around_cursor('aaa b{b}b ', 'ccc') input_keys('W') - assert_byte_pointer_size('aaa b{b}b cc') - assert_cursor(12) - assert_cursor_max(13) + assert_line_around_cursor('aaa b{b}b cc', 'c') input_keys('B') - assert_byte_pointer_size('aaa b{b}b ') - assert_cursor(10) - assert_cursor_max(13) + assert_line_around_cursor('aaa b{b}b ', 'ccc') input_keys('B') - assert_byte_pointer_size('aaa ') - assert_cursor(4) - assert_cursor_max(13) + assert_line_around_cursor('aaa ', 'b{b}b ccc') input_keys('B') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(13) + assert_line_around_cursor('', 'aaa b{b}b ccc') input_keys('2W') - assert_byte_pointer_size('aaa b{b}b ') - assert_cursor(10) - assert_cursor_max(13) + assert_line_around_cursor('aaa b{b}b ', 'ccc') input_keys('2W') - assert_byte_pointer_size('aaa b{b}b cc') - assert_cursor(12) - assert_cursor_max(13) + assert_line_around_cursor('aaa b{b}b cc', 'c') input_keys('2B') - assert_byte_pointer_size('aaa ') - assert_cursor(4) - assert_cursor_max(13) + assert_line_around_cursor('aaa ', 'b{b}b ccc') input_keys('2B') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(13) + assert_line_around_cursor('', 'aaa b{b}b ccc') end def test_vi_end_big_word input_keys("aaa b{b}}}b ccc\C-[0") - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(19) + assert_line_around_cursor('', 'aaa b{b}}}b ccc') input_keys('E') - assert_byte_pointer_size('aa') - assert_cursor(2) - assert_cursor_max(19) + assert_line_around_cursor('aa', 'a b{b}}}b ccc') input_keys('E') - assert_byte_pointer_size('aaa b{b}}}') - assert_cursor(12) - assert_cursor_max(19) + assert_line_around_cursor('aaa b{b}}}', 'b ccc') input_keys('E') - assert_byte_pointer_size('aaa b{b}}}b cc') - assert_cursor(18) - assert_cursor_max(19) + assert_line_around_cursor('aaa b{b}}}b cc', 'c') input_keys('E') - assert_byte_pointer_size('aaa b{b}}}b cc') - assert_cursor(18) - assert_cursor_max(19) + assert_line_around_cursor('aaa b{b}}}b cc', 'c') end def test_ed_quoted_insert input_keys("ab\C-v\C-acd") - assert_line("ab\C-acd") - assert_byte_pointer_size("ab\C-acd") - assert_cursor(6) - assert_cursor_max(6) + assert_line_around_cursor("ab\C-acd", '') end def test_ed_quoted_insert_with_vi_arg input_keys("ab\C-[3\C-v\C-aacd") - assert_line("a\C-a\C-a\C-abcd") - assert_byte_pointer_size("a\C-a\C-a\C-abcd") - assert_cursor(10) - assert_cursor_max(10) + assert_line_around_cursor("a\C-a\C-a\C-abcd", '') end def test_vi_replace_char input_keys("abcdef\C-[03l") - assert_line('abcdef') - assert_byte_pointer_size('abc') - assert_cursor(3) - assert_cursor_max(6) + assert_line_around_cursor('abc', 'def') input_keys('rz') - assert_line('abczef') - assert_byte_pointer_size('abc') - assert_cursor(3) - assert_cursor_max(6) + assert_line_around_cursor('abc', 'zef') input_keys('2rx') - assert_line('abcxxf') - assert_byte_pointer_size('abcxx') - assert_cursor(5) - assert_cursor_max(6) + assert_line_around_cursor('abcxx', 'f') end def test_vi_replace_char_with_mbchar input_keys("あいうえお\C-[0l") - assert_line('あいうえお') - assert_byte_pointer_size('あ') - assert_cursor(2) - assert_cursor_max(10) + assert_line_around_cursor('あ', 'いうえお') input_keys('rx') - assert_line('あxうえお') - assert_byte_pointer_size('あ') - assert_cursor(2) - assert_cursor_max(9) + assert_line_around_cursor('あ', 'xうえお') input_keys('l2ry') - assert_line('あxyyお') - assert_byte_pointer_size('あxyy') - assert_cursor(5) - assert_cursor_max(7) + assert_line_around_cursor('あxyy', 'お') end def test_vi_next_char input_keys("abcdef\C-[0") - assert_line('abcdef') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(6) + assert_line_around_cursor('', 'abcdef') input_keys('fz') - assert_line('abcdef') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(6) + assert_line_around_cursor('', 'abcdef') input_keys('fe') - assert_line('abcdef') - assert_byte_pointer_size('abcd') - assert_cursor(4) - assert_cursor_max(6) + assert_line_around_cursor('abcd', 'ef') end def test_vi_to_next_char input_keys("abcdef\C-[0") - assert_line('abcdef') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(6) + assert_line_around_cursor('', 'abcdef') input_keys('tz') - assert_line('abcdef') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(6) + assert_line_around_cursor('', 'abcdef') input_keys('te') - assert_line('abcdef') - assert_byte_pointer_size('abc') - assert_cursor(3) - assert_cursor_max(6) + assert_line_around_cursor('abc', 'def') end def test_vi_prev_char input_keys("abcdef\C-[") - assert_line('abcdef') - assert_byte_pointer_size('abcde') - assert_cursor(5) - assert_cursor_max(6) + assert_line_around_cursor('abcde', 'f') input_keys('Fz') - assert_line('abcdef') - assert_byte_pointer_size('abcde') - assert_cursor(5) - assert_cursor_max(6) + assert_line_around_cursor('abcde', 'f') input_keys('Fa') - assert_line('abcdef') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(6) + assert_line_around_cursor('', 'abcdef') end def test_vi_to_prev_char input_keys("abcdef\C-[") - assert_line('abcdef') - assert_byte_pointer_size('abcde') - assert_cursor(5) - assert_cursor_max(6) + assert_line_around_cursor('abcde', 'f') input_keys('Tz') - assert_line('abcdef') - assert_byte_pointer_size('abcde') - assert_cursor(5) - assert_cursor_max(6) + assert_line_around_cursor('abcde', 'f') input_keys('Ta') - assert_line('abcdef') - assert_byte_pointer_size('a') - assert_cursor(1) - assert_cursor_max(6) + assert_line_around_cursor('a', 'bcdef') end def test_vi_delete_next_char input_keys("abc\C-[h") - assert_byte_pointer_size('a') - assert_cursor(1) - assert_cursor_max(3) - assert_line('abc') + assert_line_around_cursor('a', 'bc') input_keys('x') - assert_byte_pointer_size('a') - assert_cursor(1) - assert_cursor_max(2) - assert_line('ac') + assert_line_around_cursor('a', 'c') input_keys('x') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(1) - assert_line('a') + assert_line_around_cursor('', 'a') input_keys('x') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(0) - assert_line('') + assert_line_around_cursor('', '') input_keys('x') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(0) - assert_line('') + assert_line_around_cursor('', '') end def test_vi_delete_next_char_for_mbchar input_keys("あいう\C-[h") - assert_byte_pointer_size('あ') - assert_cursor(2) - assert_cursor_max(6) - assert_line('あいう') + assert_line_around_cursor('あ', 'いう') input_keys('x') - assert_byte_pointer_size('あ') - assert_cursor(2) - assert_cursor_max(4) - assert_line('あう') + assert_line_around_cursor('あ', 'う') input_keys('x') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(2) - assert_line('あ') + assert_line_around_cursor('', 'あ') input_keys('x') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(0) - assert_line('') + assert_line_around_cursor('', '') input_keys('x') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(0) - assert_line('') + assert_line_around_cursor('', '') end def test_vi_delete_next_char_for_mbchar_by_plural_code_points input_keys("か\u3099き\u3099く\u3099\C-[h") - assert_byte_pointer_size("か\u3099") - assert_cursor(2) - assert_cursor_max(6) - assert_line("か\u3099き\u3099く\u3099") + assert_line_around_cursor("か\u3099", "き\u3099く\u3099") input_keys('x') - assert_byte_pointer_size("か\u3099") - assert_cursor(2) - assert_cursor_max(4) - assert_line("か\u3099く\u3099") + assert_line_around_cursor("か\u3099", "く\u3099") input_keys('x') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(2) - assert_line("か\u3099") + assert_line_around_cursor('', "か\u3099") input_keys('x') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(0) - assert_line('') + assert_line_around_cursor('', '') input_keys('x') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(0) - assert_line('') + assert_line_around_cursor('', '') end def test_vi_delete_prev_char input_keys('ab') - assert_byte_pointer_size('ab') - assert_cursor(2) - assert_cursor_max(2) + assert_line_around_cursor('ab', '') input_keys("\C-h") - assert_byte_pointer_size('a') - assert_cursor(1) - assert_cursor_max(1) - assert_line('a') + assert_line_around_cursor('a', '') end def test_vi_delete_prev_char_for_mbchar input_keys('かき') - assert_byte_pointer_size('かき') - assert_cursor(4) - assert_cursor_max(4) + assert_line_around_cursor('かき', '') input_keys("\C-h") - assert_byte_pointer_size('か') - assert_cursor(2) - assert_cursor_max(2) - assert_line('か') + assert_line_around_cursor('か', '') end def test_vi_delete_prev_char_for_mbchar_by_plural_code_points input_keys("か\u3099き\u3099") - assert_byte_pointer_size("か\u3099き\u3099") - assert_cursor(4) - assert_cursor_max(4) + assert_line_around_cursor("か\u3099き\u3099", '') input_keys("\C-h") - assert_byte_pointer_size("か\u3099") - assert_cursor(2) - assert_cursor_max(2) - assert_line("か\u3099") + assert_line_around_cursor("か\u3099", '') end def test_ed_delete_prev_char input_keys("abcdefg\C-[h") - assert_byte_pointer_size('abcde') - assert_cursor(5) - assert_cursor_max(7) - assert_line('abcdefg') + assert_line_around_cursor('abcde', 'fg') input_keys('X') - assert_byte_pointer_size('abcd') - assert_cursor(4) - assert_cursor_max(6) - assert_line('abcdfg') + assert_line_around_cursor('abcd', 'fg') input_keys('3X') - assert_byte_pointer_size('a') - assert_cursor(1) - assert_cursor_max(3) - assert_line('afg') + assert_line_around_cursor('a', 'fg') input_keys('p') - assert_byte_pointer_size('abcd') - assert_cursor(4) - assert_cursor_max(6) - assert_line('afbcdg') + assert_line_around_cursor('afbc', 'dg') end def test_ed_delete_prev_word input_keys('abc def{bbb}ccc') - assert_byte_pointer_size('abc def{bbb}ccc') - assert_cursor(15) - assert_cursor_max(15) + assert_line_around_cursor('abc def{bbb}ccc', '') input_keys("\C-w") - assert_byte_pointer_size('abc def{bbb}') - assert_cursor(12) - assert_cursor_max(12) - assert_line('abc def{bbb}') + assert_line_around_cursor('abc def{bbb}', '') input_keys("\C-w") - assert_byte_pointer_size('abc def{') - assert_cursor(8) - assert_cursor_max(8) - assert_line('abc def{') + assert_line_around_cursor('abc def{', '') input_keys("\C-w") - assert_byte_pointer_size('abc ') - assert_cursor(4) - assert_cursor_max(4) - assert_line('abc ') + assert_line_around_cursor('abc ', '') input_keys("\C-w") - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(0) - assert_line('') + assert_line_around_cursor('', '') end def test_ed_delete_prev_word_for_mbchar input_keys('あいう かきく{さしす}たちつ') - assert_byte_pointer_size('あいう かきく{さしす}たちつ') - assert_cursor(27) - assert_cursor_max(27) + assert_line_around_cursor('あいう かきく{さしす}たちつ', '') input_keys("\C-w") - assert_byte_pointer_size('あいう かきく{さしす}') - assert_cursor(21) - assert_cursor_max(21) - assert_line('あいう かきく{さしす}') + assert_line_around_cursor('あいう かきく{さしす}', '') input_keys("\C-w") - assert_byte_pointer_size('あいう かきく{') - assert_cursor(14) - assert_cursor_max(14) - assert_line('あいう かきく{') + assert_line_around_cursor('あいう かきく{', '') input_keys("\C-w") - assert_byte_pointer_size('あいう ') - assert_cursor(7) - assert_cursor_max(7) - assert_line('あいう ') + assert_line_around_cursor('あいう ', '') input_keys("\C-w") - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(0) - assert_line('') + assert_line_around_cursor('', '') end def test_ed_delete_prev_word_for_mbchar_by_plural_code_points input_keys("あいう か\u3099き\u3099く\u3099{さしす}たちつ") - assert_byte_pointer_size("あいう か\u3099き\u3099く\u3099{さしす}たちつ") - assert_cursor(27) - assert_cursor_max(27) + assert_line_around_cursor("あいう か\u3099き\u3099く\u3099{さしす}たちつ", '') input_keys("\C-w") - assert_byte_pointer_size("あいう か\u3099き\u3099く\u3099{さしす}") - assert_cursor(21) - assert_cursor_max(21) - assert_line("あいう か\u3099き\u3099く\u3099{さしす}") + assert_line_around_cursor("あいう か\u3099き\u3099く\u3099{さしす}", '') input_keys("\C-w") - assert_byte_pointer_size("あいう か\u3099き\u3099く\u3099{") - assert_cursor(14) - assert_cursor_max(14) - assert_line("あいう か\u3099き\u3099く\u3099{") + assert_line_around_cursor("あいう か\u3099き\u3099く\u3099{", '') input_keys("\C-w") - assert_byte_pointer_size('あいう ') - assert_cursor(7) - assert_cursor_max(7) - assert_line('あいう ') + assert_line_around_cursor('あいう ', '') input_keys("\C-w") - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(0) - assert_line('') + assert_line_around_cursor('', '') end def test_ed_newline_with_cr input_keys('ab') - assert_byte_pointer_size('ab') - assert_cursor(2) - assert_cursor_max(2) + assert_line_around_cursor('ab', '') refute(@line_editor.finished?) input_keys("\C-m") - assert_line('ab') + assert_line_around_cursor('ab', '') assert(@line_editor.finished?) end def test_ed_newline_with_lf input_keys('ab') - assert_byte_pointer_size('ab') - assert_cursor(2) - assert_cursor_max(2) + assert_line_around_cursor('ab', '') refute(@line_editor.finished?) input_keys("\C-j") - assert_line('ab') + assert_line_around_cursor('ab', '') assert(@line_editor.finished?) end def test_vi_list_or_eof input_keys("\C-d") # quit from inputing - assert_line(nil) + assert_nil(@line_editor.line) assert(@line_editor.finished?) end def test_vi_list_or_eof_with_non_empty_line input_keys('ab') - assert_byte_pointer_size('ab') - assert_cursor(2) - assert_cursor_max(2) + assert_line_around_cursor('ab', '') refute(@line_editor.finished?) input_keys("\C-d") - assert_line('ab') + assert_line_around_cursor('ab', '') assert(@line_editor.finished?) end @@ -982,40 +545,19 @@ def test_completion_journey } } input_keys('foo') - assert_byte_pointer_size('foo') - assert_cursor(3) - assert_cursor_max(3) - assert_line('foo') + assert_line_around_cursor('foo', '') input_keys("\C-n") - assert_byte_pointer_size('foo_bar') - assert_cursor(7) - assert_cursor_max(7) - assert_line('foo_bar') + assert_line_around_cursor('foo_bar', '') input_keys("\C-n") - assert_byte_pointer_size('foo_bar_baz') - assert_cursor(11) - assert_cursor_max(11) - assert_line('foo_bar_baz') + assert_line_around_cursor('foo_bar_baz', '') input_keys("\C-n") - assert_byte_pointer_size('foo') - assert_cursor(3) - assert_cursor_max(3) - assert_line('foo') + assert_line_around_cursor('foo', '') input_keys("\C-n") - assert_byte_pointer_size('foo_bar') - assert_cursor(7) - assert_cursor_max(7) - assert_line('foo_bar') + assert_line_around_cursor('foo_bar', '') input_keys("_\C-n") - assert_byte_pointer_size('foo_bar_baz') - assert_cursor(11) - assert_cursor_max(11) - assert_line('foo_bar_baz') + assert_line_around_cursor('foo_bar_baz', '') input_keys("\C-n") - assert_byte_pointer_size('foo_bar_') - assert_cursor(8) - assert_cursor_max(8) - assert_line('foo_bar_') + assert_line_around_cursor('foo_bar_', '') end def test_completion_journey_reverse @@ -1028,40 +570,19 @@ def test_completion_journey_reverse } } input_keys('foo') - assert_byte_pointer_size('foo') - assert_cursor(3) - assert_cursor_max(3) - assert_line('foo') + assert_line_around_cursor('foo', '') input_keys("\C-p") - assert_byte_pointer_size('foo_bar_baz') - assert_cursor(11) - assert_cursor_max(11) - assert_line('foo_bar_baz') + assert_line_around_cursor('foo_bar_baz', '') input_keys("\C-p") - assert_byte_pointer_size('foo_bar') - assert_cursor(7) - assert_cursor_max(7) - assert_line('foo_bar') + assert_line_around_cursor('foo_bar', '') input_keys("\C-p") - assert_byte_pointer_size('foo') - assert_cursor(3) - assert_cursor_max(3) - assert_line('foo') + assert_line_around_cursor('foo', '') input_keys("\C-p") - assert_byte_pointer_size('foo_bar_baz') - assert_cursor(11) - assert_cursor_max(11) - assert_line('foo_bar_baz') + assert_line_around_cursor('foo_bar_baz', '') input_keys("\C-h\C-p") - assert_byte_pointer_size('foo_bar_baz') - assert_cursor(11) - assert_cursor_max(11) - assert_line('foo_bar_baz') + assert_line_around_cursor('foo_bar_baz', '') input_keys("\C-p") - assert_byte_pointer_size('foo_bar_ba') - assert_cursor(10) - assert_cursor_max(10) - assert_line('foo_bar_ba') + assert_line_around_cursor('foo_bar_ba', '') end def test_completion_journey_in_middle_of_line @@ -1074,42 +595,21 @@ def test_completion_journey_in_middle_of_line } } input_keys('abcde fo ABCDE') - assert_line('abcde fo ABCDE') + assert_line_around_cursor('abcde fo ABCDE', '') input_keys("\C-[" + 'h' * 5 + "i\C-n") - assert_byte_pointer_size('abcde foo_bar') - assert_cursor(13) - assert_cursor_max(19) - assert_line('abcde foo_bar ABCDE') + assert_line_around_cursor('abcde foo_bar', ' ABCDE') input_keys("\C-n") - assert_byte_pointer_size('abcde foo_bar_baz') - assert_cursor(17) - assert_cursor_max(23) - assert_line('abcde foo_bar_baz ABCDE') + assert_line_around_cursor('abcde foo_bar_baz', ' ABCDE') input_keys("\C-n") - assert_byte_pointer_size('abcde fo') - assert_cursor(8) - assert_cursor_max(14) - assert_line('abcde fo ABCDE') + assert_line_around_cursor('abcde fo', ' ABCDE') input_keys("\C-n") - assert_byte_pointer_size('abcde foo_bar') - assert_cursor(13) - assert_cursor_max(19) - assert_line('abcde foo_bar ABCDE') + assert_line_around_cursor('abcde foo_bar', ' ABCDE') input_keys("_\C-n") - assert_byte_pointer_size('abcde foo_bar_baz') - assert_cursor(17) - assert_cursor_max(23) - assert_line('abcde foo_bar_baz ABCDE') + assert_line_around_cursor('abcde foo_bar_baz', ' ABCDE') input_keys("\C-n") - assert_byte_pointer_size('abcde foo_bar_') - assert_cursor(14) - assert_cursor_max(20) - assert_line('abcde foo_bar_ ABCDE') + assert_line_around_cursor('abcde foo_bar_', ' ABCDE') input_keys("\C-n") - assert_byte_pointer_size('abcde foo_bar_baz') - assert_cursor(17) - assert_cursor_max(23) - assert_line('abcde foo_bar_baz ABCDE') + assert_line_around_cursor('abcde foo_bar_baz', ' ABCDE') end def test_completion @@ -1122,15 +622,9 @@ def test_completion } } input_keys('foo') - assert_byte_pointer_size('foo') - assert_cursor(3) - assert_cursor_max(3) - assert_line('foo') + assert_line_around_cursor('foo', '') input_keys("\C-i") - assert_byte_pointer_size('foo_bar') - assert_cursor(7) - assert_cursor_max(7) - assert_line('foo_bar') + assert_line_around_cursor('foo_bar', '') end def test_completion_with_disable_completion @@ -1144,247 +638,130 @@ def test_completion_with_disable_completion } } input_keys('foo') - assert_byte_pointer_size('foo') - assert_cursor(3) - assert_cursor_max(3) - assert_line('foo') + assert_line_around_cursor('foo', '') input_keys("\C-i") - assert_byte_pointer_size('foo') - assert_cursor(3) - assert_cursor_max(3) - assert_line('foo') + assert_line_around_cursor('foo', '') end def test_vi_first_print input_keys("abcde\C-[^") - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(5) + assert_line_around_cursor('', 'abcde') input_keys("0\C-ki") input_keys(" abcde\C-[^") - assert_byte_pointer_size(' ') - assert_cursor(1) - assert_cursor_max(6) + assert_line_around_cursor(' ', 'abcde') input_keys("0\C-ki") input_keys(" abcde ABCDE \C-[^") - assert_byte_pointer_size(' ') - assert_cursor(3) - assert_cursor_max(17) + assert_line_around_cursor(' ', 'abcde ABCDE ') end def test_ed_move_to_beg input_keys("abcde\C-[0") - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(5) + assert_line_around_cursor('', 'abcde') input_keys("0\C-ki") input_keys(" abcde\C-[0") - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(6) + assert_line_around_cursor('', ' abcde') input_keys("0\C-ki") input_keys(" abcde ABCDE \C-[0") - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(17) + assert_line_around_cursor('', ' abcde ABCDE ') end def test_vi_delete_meta input_keys("aaa bbb ccc ddd eee\C-[02w") - assert_byte_pointer_size('aaa bbb ') - assert_cursor(8) - assert_cursor_max(19) - assert_line('aaa bbb ccc ddd eee') + assert_line_around_cursor('aaa bbb ', 'ccc ddd eee') input_keys('dw') - assert_byte_pointer_size('aaa bbb ') - assert_cursor(8) - assert_cursor_max(15) - assert_line('aaa bbb ddd eee') + assert_line_around_cursor('aaa bbb ', 'ddd eee') input_keys('db') - assert_byte_pointer_size('aaa ') - assert_cursor(4) - assert_cursor_max(11) - assert_line('aaa ddd eee') + assert_line_around_cursor('aaa ', 'ddd eee') end def test_vi_delete_meta_with_vi_next_word_at_eol input_keys("foo bar\C-[0w") - assert_byte_pointer_size('foo ') - assert_cursor(4) - assert_cursor_max(7) - assert_line('foo bar') + assert_line_around_cursor('foo ', 'bar') input_keys('w') - assert_byte_pointer_size('foo ba') - assert_cursor(6) - assert_cursor_max(7) - assert_line('foo bar') + assert_line_around_cursor('foo ba', 'r') input_keys('0dw') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(3) - assert_line('bar') + assert_line_around_cursor('', 'bar') input_keys('dw') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(0) - assert_line('') + assert_line_around_cursor('', '') end def test_vi_delete_meta_with_vi_next_char input_keys("aaa bbb ccc ___ ddd\C-[02w") - assert_byte_pointer_size('aaa bbb ') - assert_cursor(8) - assert_cursor_max(19) - assert_line('aaa bbb ccc ___ ddd') + assert_line_around_cursor('aaa bbb ', 'ccc ___ ddd') input_keys('df_') - assert_byte_pointer_size('aaa bbb ') - assert_cursor(8) - assert_cursor_max(14) - assert_line('aaa bbb __ ddd') + assert_line_around_cursor('aaa bbb ', '__ ddd') end def test_vi_delete_meta_with_arg input_keys("aaa bbb ccc\C-[02w") - assert_byte_pointer_size('aaa bbb ') - assert_cursor(8) - assert_cursor_max(11) - assert_line('aaa bbb ccc') + assert_line_around_cursor('aaa bbb ', 'ccc') input_keys('2dl') - assert_byte_pointer_size('aaa bbb ') - assert_cursor(8) - assert_cursor_max(9) - assert_line('aaa bbb c') + assert_line_around_cursor('aaa bbb ', 'c') end def test_vi_change_meta input_keys("aaa bbb ccc ddd eee\C-[02w") - assert_byte_pointer_size('aaa bbb ') - assert_cursor(8) - assert_cursor_max(19) - assert_line('aaa bbb ccc ddd eee') + assert_line_around_cursor('aaa bbb ', 'ccc ddd eee') input_keys('cwaiueo') - assert_byte_pointer_size('aaa bbb aiueo') - assert_cursor(13) - assert_cursor_max(21) - assert_line('aaa bbb aiueo ddd eee') + assert_line_around_cursor('aaa bbb aiueo', ' ddd eee') input_keys("\C-[") - assert_byte_pointer_size('aaa bbb aiue') - assert_cursor(12) - assert_cursor_max(21) - assert_line('aaa bbb aiueo ddd eee') + assert_line_around_cursor('aaa bbb aiue', 'o ddd eee') input_keys('cb') - assert_byte_pointer_size('aaa bbb ') - assert_cursor(8) - assert_cursor_max(17) - assert_line('aaa bbb o ddd eee') + assert_line_around_cursor('aaa bbb ', 'o ddd eee') end def test_vi_change_meta_with_vi_next_word input_keys("foo bar baz\C-[0w") - assert_byte_pointer_size('foo ') - assert_cursor(5) - assert_cursor_max(13) - assert_line('foo bar baz') + assert_line_around_cursor('foo ', 'bar baz') input_keys('cwhoge') - assert_byte_pointer_size('foo hoge') - assert_cursor(9) - assert_cursor_max(14) - assert_line('foo hoge baz') + assert_line_around_cursor('foo hoge', ' baz') input_keys("\C-[") - assert_byte_pointer_size('foo hog') - assert_cursor(8) - assert_cursor_max(14) - assert_line('foo hoge baz') + assert_line_around_cursor('foo hog', 'e baz') end def test_unimplemented_vi_command_should_be_no_op input_keys("abc\C-[h") - assert_byte_pointer_size('a') - assert_cursor(1) - assert_cursor_max(3) - assert_line('abc') + assert_line_around_cursor('a', 'bc') input_keys('@') - assert_byte_pointer_size('a') - assert_cursor(1) - assert_cursor_max(3) - assert_line('abc') + assert_line_around_cursor('a', 'bc') end def test_vi_yank input_keys("foo bar\C-[0") - assert_line('foo bar') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(7) + assert_line_around_cursor('', 'foo bar') input_keys('y3l') - assert_line('foo bar') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(7) + assert_line_around_cursor('', 'foo bar') input_keys('P') - assert_line('foofoo bar') - assert_byte_pointer_size('fo') - assert_cursor(2) - assert_cursor_max(10) + assert_line_around_cursor('fo', 'ofoo bar') end def test_vi_end_word_with_operator input_keys("foo bar\C-[0") - assert_line('foo bar') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(7) + assert_line_around_cursor('', 'foo bar') input_keys('de') - assert_line(' bar') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(4) + assert_line_around_cursor('', ' bar') input_keys('de') - assert_line('') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(0) + assert_line_around_cursor('', '') input_keys('de') - assert_line('') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(0) + assert_line_around_cursor('', '') end def test_vi_end_big_word_with_operator input_keys("aaa b{b}}}b\C-[0") - assert_line('aaa b{b}}}b') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(13) + assert_line_around_cursor('', 'aaa b{b}}}b') input_keys('dE') - assert_line(' b{b}}}b') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(10) + assert_line_around_cursor('', ' b{b}}}b') input_keys('dE') - assert_line('') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(0) + assert_line_around_cursor('', '') input_keys('dE') - assert_line('') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(0) + assert_line_around_cursor('', '') end def test_vi_next_char_with_operator input_keys("foo bar\C-[0") - assert_line('foo bar') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(7) + assert_line_around_cursor('', 'foo bar') input_keys('df ') - assert_line('bar') - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(3) + assert_line_around_cursor('', 'bar') end def test_pasting @@ -1392,10 +769,7 @@ def test_pasting input_keys('ab') finish_pasting input_keys('c') - assert_line('abc') - assert_byte_pointer_size('abc') - assert_cursor(3) - assert_cursor_max(3) + assert_line_around_cursor('abc', '') end def test_pasting_fullwidth @@ -1403,65 +777,37 @@ def test_pasting_fullwidth input_keys('あ') finish_pasting input_keys('い') - assert_line('あい') - assert_byte_pointer_size('あい') - assert_cursor(4) - assert_cursor_max(4) + assert_line_around_cursor('あい', '') end def test_ed_delete_next_char_at_eol input_keys('"あ"') - assert_line('"あ"') - assert_byte_pointer_size('"あ"') - assert_cursor(4) - assert_cursor_max(4) + assert_line_around_cursor('"あ"', '') input_keys("\C-[") - assert_line('"あ"') - assert_byte_pointer_size('"あ') - assert_cursor(3) - assert_cursor_max(4) + assert_line_around_cursor('"あ', '"') input_keys('xa"') - assert_line('"あ"') - assert_byte_pointer_size('"あ"') - assert_cursor(4) - assert_cursor_max(4) + assert_line_around_cursor('"あ"', '') end def test_vi_kill_line_prev input_keys("\C-u", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(0) - assert_line('') + assert_line_around_cursor('', '') input_keys('abc') - assert_byte_pointer_size('abc') - assert_cursor(3) - assert_cursor_max(3) + assert_line_around_cursor('abc', '') input_keys("\C-u", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(0) - assert_line('') + assert_line_around_cursor('', '') input_keys('abc') input_keys("\C-[\C-u", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(1) - assert_line('c') + assert_line_around_cursor('', 'c') input_keys("\C-u", false) - assert_byte_pointer_size('') - assert_cursor(0) - assert_cursor_max(1) - assert_line('c') + assert_line_around_cursor('', 'c') end def test_vi_change_to_eol input_keys("abcdef\C-[2hC") - assert_line("abc") + assert_line_around_cursor('abc', '') input_keys("\C-[0C") - assert_line("") - assert_cursor(0) - assert_cursor_max(0) + assert_line_around_cursor('', '') assert_instance_of(Reline::KeyActor::ViInsert, @config.editing_mode) end From b0eda83ee0c34d265277635a1e13591d6367bd01 Mon Sep 17 00:00:00 2001 From: tomoya ishida Date: Sun, 24 Mar 2024 19:47:18 +0900 Subject: [PATCH 002/211] [ruby/reline] Add mode_string to prompt calculation dependencies (https://github.com/ruby/reline/pull/658) * Add mode_string to prompt calculation dependencies * Update vi show-mode-in-prompt test https://github.com/ruby/reline/commit/a0cee06ec5 --- lib/reline/line_editor.rb | 8 +++----- test/reline/yamatanooroti/test_rendering.rb | 5 ++++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/lib/reline/line_editor.rb b/lib/reline/line_editor.rb index 5a04e5650f6c87..d202ba02d27a76 100644 --- a/lib/reline/line_editor.rb +++ b/lib/reline/line_editor.rb @@ -85,7 +85,7 @@ def set_pasting_state(in_pasting) end end - private def check_multiline_prompt(buffer) + private def check_multiline_prompt(buffer, mode_string) if @vi_arg prompt = "(arg: #{@vi_arg}) " elsif @searching_prompt @@ -97,7 +97,6 @@ def set_pasting_state(in_pasting) prompt_list = @prompt_proc.(buffer).map { |pr| pr.gsub("\n", "\\n") } prompt_list.map!{ prompt } if @vi_arg or @searching_prompt prompt_list = [prompt] if prompt_list.empty? - mode_string = check_mode_string prompt_list = prompt_list.map{ |pr| mode_string + pr } if mode_string prompt = prompt_list[@line_index] prompt = prompt_list[0] if prompt.nil? @@ -109,7 +108,6 @@ def set_pasting_state(in_pasting) end prompt_list else - mode_string = check_mode_string prompt = mode_string + prompt if mode_string [prompt] * buffer.size end @@ -319,8 +317,8 @@ def modified_lines end def prompt_list - with_cache(__method__, whole_lines, @vi_arg, @searching_prompt) do |lines| - check_multiline_prompt(lines) + with_cache(__method__, whole_lines, check_mode_string, @vi_arg, @searching_prompt) do |lines, mode_string| + check_multiline_prompt(lines, mode_string) end end diff --git a/test/reline/yamatanooroti/test_rendering.rb b/test/reline/yamatanooroti/test_rendering.rb index 527da519c1c78c..efb1e01562b905 100644 --- a/test/reline/yamatanooroti/test_rendering.rb +++ b/test/reline/yamatanooroti/test_rendering.rb @@ -197,9 +197,12 @@ def test_mode_string_vi LINES start_terminal(5, 30, %W{ruby -I#{@pwd}/lib #{@pwd}/test/reline/yamatanooroti/multiline_repl}, startup_message: 'Multiline REPL.') write(":a\n\C-[k") + write("i\n:a") + write("\C-[h") close assert_screen(<<~EOC) - Multiline REPL. + (ins)prompt> :a + => :a (ins)prompt> :a => :a (cmd)prompt> :a From 82f4cff1f39d5f2c762f8cf61f079e318066e44e Mon Sep 17 00:00:00 2001 From: tomoya ishida Date: Sun, 24 Mar 2024 21:54:33 +0900 Subject: [PATCH 003/211] [ruby/irb] Fix indent test for new reline (https://github.com/ruby/irb/pull/908) https://github.com/ruby/irb/commit/7c16ce033e --- test/irb/test_irb.rb | 7 ++----- test/irb/yamatanooroti/test_rendering.rb | 26 +++++++++++++----------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/test/irb/test_irb.rb b/test/irb/test_irb.rb index 8c4fb5ddee3d7a..966c840135c74f 100644 --- a/test/irb/test_irb.rb +++ b/test/irb/test_irb.rb @@ -70,11 +70,8 @@ def test_empty_input_echoing_behaviour type "exit" end - # Input cramped together due to how Reline's Reline::GeneralIO works - assert_include( - output, - "irb(main):001> irb(main):002> irb(main):002> irb(main):002> => nil\r\n" - ) + assert_not_match(/irb\(main\):001> (\r*\n)?=> nil/, output) + assert_match(/irb\(main\):002> (\r*\n)?=> nil/, output) end end diff --git a/test/irb/yamatanooroti/test_rendering.rb b/test/irb/yamatanooroti/test_rendering.rb index 511df58ea6c930..df4ec01a5cafa6 100644 --- a/test/irb/yamatanooroti/test_rendering.rb +++ b/test/irb/yamatanooroti/test_rendering.rb @@ -137,6 +137,7 @@ def b; true; end a .a .b + .itself EOC close assert_screen(<<~EOC) @@ -153,9 +154,10 @@ def b; true; end irb(main):008> irb(main):009> a irb(main):010> .a - irb(main):011> .b + irb(main):011> .b + irb(main):012> .itself => true - irb(main):012> + irb(main):013> EOC end @@ -181,7 +183,6 @@ def c; true; end (a) &.b() - class A def b; self; end; def c; true; end; end; a = A.new a @@ -190,6 +191,7 @@ class A def b; self; end; def c; true; end; end; .c (a) &.b() + .itself EOC close assert_screen(<<~EOC) @@ -214,17 +216,17 @@ class A def b; self; end; def c; true; end; end; irb(main):015> &.b() => # irb(main):016> - irb(main):017> - irb(main):018> class A def b; self; end; def c; true; end; end; - irb(main):019> a = A.new + irb(main):017> class A def b; self; end; def c; true; end; end; + irb(main):018> a = A.new => # - irb(main):020> a - irb(main):021> .b - irb(main):022> # aaa - irb(main):023> .c + irb(main):019> a + irb(main):020> .b + irb(main):021> # aaa + irb(main):022> .c => true - irb(main):024> (a) - irb(main):025> &.b() + irb(main):023> (a) + irb(main):024> &.b() + irb(main):025> .itself => # irb(main):026> EOC From 74593613ea399518d6f72cb9ca330efbfcf719cb Mon Sep 17 00:00:00 2001 From: tomoya ishida Date: Sun, 24 Mar 2024 22:04:30 +0900 Subject: [PATCH 004/211] [ruby/reline] Disable dialog proc if TERM=dumb (https://github.com/ruby/reline/pull/663) https://github.com/ruby/reline/commit/4928e06a24 --- lib/reline.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/reline.rb b/lib/reline.rb index 713d1a8b60c78c..f3fd28b627736a 100644 --- a/lib/reline.rb +++ b/lib/reline.rb @@ -331,8 +331,10 @@ def readline(prompt = '', add_hist = false) line_editor.auto_indent_proc = auto_indent_proc line_editor.dig_perfect_match_proc = dig_perfect_match_proc pre_input_hook&.call - @dialog_proc_list.each_pair do |name_sym, d| - line_editor.add_dialog_proc(name_sym, d.dialog_proc, d.context) + unless Reline::IOGate == Reline::GeneralIO + @dialog_proc_list.each_pair do |name_sym, d| + line_editor.add_dialog_proc(name_sym, d.dialog_proc, d.context) + end end unless config.test_mode From 7c015c3b30ca3618f9c02c03ff4184ca8587b2c1 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Sun, 24 Mar 2024 22:35:09 +0900 Subject: [PATCH 005/211] Remove never used macros related to RJIT [ci skip] These macros have not been used since the commit "Stop exporting symbols for MJIT" 233ddfac541749a0da80ea27913dc1ef4ea700bb, and renamed as RJIT. --- common.mk | 2 -- template/Doxyfile.tmpl | 4 ---- 2 files changed, 6 deletions(-) diff --git a/common.mk b/common.mk index 9401127651b672..21ceaa3f01279b 100644 --- a/common.mk +++ b/common.mk @@ -471,8 +471,6 @@ ruby.imp: $(COMMONOBJS) $(Q){ \ $(NM) -Pgp $(COMMONOBJS) | \ awk 'BEGIN{print "#!"}; $$2~/^[A-TV-Z]$$/&&$$1!~/^$(SYMBOL_PREFIX)(Init_|InitVM_|ruby_static_id_|.*_threadptr_|rb_ec_)|^\./{print $$1}'; \ - ($(CHDIR) $(srcdir) && \ - exec sed -n '/^RJIT_FUNC_EXPORTED/!d;N;s/.*\n\(rb_[a-zA-Z_0-9]*\).*/$(SYMBOL_PREFIX)\1/p' cont.c gc.c thread*c vm*.c) \ } | \ sort -u -o $@ diff --git a/template/Doxyfile.tmpl b/template/Doxyfile.tmpl index c04f14977de323..cea28d04c7d278 100644 --- a/template/Doxyfile.tmpl +++ b/template/Doxyfile.tmpl @@ -2359,10 +2359,6 @@ PREDEFINED += DEPRECATED(_)=_ PREDEFINED += DEPRECATED_BY(__,_)=_ PREDEFINED += ENUM_OVER_INT=1 PREDEFINED += ERRORFUNC(__,_)=_ -PREDEFINED += RJIT_FUNC_EXPORTED= -PREDEFINED += RJIT_STATIC=extern -PREDEFINED += RJIT_SYMBOL_EXPORT_BEGIN= -PREDEFINED += RJIT_SYMBOL_EXPORT_END= PREDEFINED += NOINLINE(_)=_ PREDEFINED += NORETURN(_)=_ PREDEFINED += PACKED_STRUCT_UNALIGNED(_)=_ From 5e4b4d667441461f6384b2afb47df855c06d2e20 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Sun, 24 Mar 2024 20:23:57 +0900 Subject: [PATCH 006/211] [rubygems/rubygems] Remove typo name It is more 2 years since #5109, probably enough time to warn. https://github.com/rubygems/rubygems/commit/1e9433e77f --- lib/rubygems/exceptions.rb | 6 ------ 1 file changed, 6 deletions(-) diff --git a/lib/rubygems/exceptions.rb b/lib/rubygems/exceptions.rb index 65caaab8b1f3c0..0308b4687f9f49 100644 --- a/lib/rubygems/exceptions.rb +++ b/lib/rubygems/exceptions.rb @@ -292,9 +292,3 @@ def version @dependency.requirement end end - -## -# Backwards compatible typo'd exception class for early RubyGems 2.0.x - -Gem::UnsatisfiableDepedencyError = Gem::UnsatisfiableDependencyError # :nodoc: -Gem.deprecate_constant :UnsatisfiableDepedencyError From e86a6191660c613e7879e62b564502eaf5456495 Mon Sep 17 00:00:00 2001 From: yui-knk Date: Sun, 24 Mar 2024 14:33:04 +0900 Subject: [PATCH 007/211] Lrama v0.6.5 --- tool/lrama/NEWS.md | 40 ++++++ tool/lrama/lib/lrama/grammar/rule_builder.rb | 3 +- tool/lrama/lib/lrama/lexer/token/user_code.rb | 2 + tool/lrama/lib/lrama/parser.rb | 133 +++++++++--------- tool/lrama/lib/lrama/version.rb | 2 +- 5 files changed, 111 insertions(+), 69 deletions(-) diff --git a/tool/lrama/NEWS.md b/tool/lrama/NEWS.md index 3fcb0ee88a91e3..96aaaf94f57d5f 100644 --- a/tool/lrama/NEWS.md +++ b/tool/lrama/NEWS.md @@ -1,5 +1,45 @@ # NEWS for Lrama +## Lrama 0.6.5 (2024-03-25) + +### Typed Midrule Actions + +User can specify the type of mid rule action by tag (``) instead of specifying it with in an action. + +``` +primary: k_case expr_value terms? + { + $$ = p->case_labels; + p->case_labels = Qnil; + } + case_body + k_end + { + ... + } +``` + +can be written as + +``` +primary: k_case expr_value terms? + { + $$ = p->case_labels; + p->case_labels = Qnil; + } + case_body + k_end + { + ... + } +``` + +`%destructor` for midrule action is invoked only when tag is specified by Typed Midrule Actions. + +Difference from Bison's Typed Midrule Actions is that tag is postposed in Lrama however it's preposed in Bison. + +Bison supports this feature from 3.1. + ## Lrama 0.6.4 (2024-03-22) ### Parameterizing rules (preceded, terminated, delimited) diff --git a/tool/lrama/lib/lrama/grammar/rule_builder.rb b/tool/lrama/lib/lrama/grammar/rule_builder.rb index 24020db6519c91..b2ccc3e2437b8c 100644 --- a/tool/lrama/lib/lrama/grammar/rule_builder.rb +++ b/tool/lrama/lib/lrama/grammar/rule_builder.rb @@ -128,10 +128,11 @@ def process_rhs(parameterizing_rule_resolver) end when Lrama::Lexer::Token::UserCode prefix = token.referred ? "@" : "$@" + tag = token.tag || lhs_tag new_token = Lrama::Lexer::Token::Ident.new(s_value: prefix + @midrule_action_counter.increment.to_s) @replaced_rhs << new_token - rule_builder = RuleBuilder.new(@rule_counter, @midrule_action_counter, i, lhs_tag: lhs_tag, skip_preprocess_references: true) + rule_builder = RuleBuilder.new(@rule_counter, @midrule_action_counter, i, lhs_tag: tag, skip_preprocess_references: true) rule_builder.lhs = new_token rule_builder.user_code = token rule_builder.complete_input diff --git a/tool/lrama/lib/lrama/lexer/token/user_code.rb b/tool/lrama/lib/lrama/lexer/token/user_code.rb index 14c69f3de6b423..4d487bf01cee4e 100644 --- a/tool/lrama/lib/lrama/lexer/token/user_code.rb +++ b/tool/lrama/lib/lrama/lexer/token/user_code.rb @@ -4,6 +4,8 @@ module Lrama class Lexer class Token class UserCode < Token + attr_accessor :tag + def references @references ||= _references end diff --git a/tool/lrama/lib/lrama/parser.rb b/tool/lrama/lib/lrama/parser.rb index 404b7c27b2bca0..0a46f759c0c787 100644 --- a/tool/lrama/lib/lrama/parser.rb +++ b/tool/lrama/lib/lrama/parser.rb @@ -658,7 +658,7 @@ def token_to_str(t) module Lrama class Parser < Racc::Parser -module_eval(<<'...end parser.y/module_eval...', 'parser.y', 528) +module_eval(<<'...end parser.y/module_eval...', 'parser.y', 529) include Lrama::Report::Duration @@ -759,7 +759,7 @@ def raise_parse_error(error_message, location) 126, 127, 128, 129, 130, 133, 137, 138, 139, 142, 143, 144, 146, 161, 163, 164, 165, 166, 167, 168, 169, 142, 171, 179, 180, 189, 194, 195, 197, 202, - 189, 94, 194, 216, 218, 94, 223, 94 ] + 189, 94, 194, 216, 218, 94, 194, 224, 94 ] racc_action_check = [ 48, 141, 48, 141, 140, 141, 170, 188, 170, 188, @@ -789,7 +789,7 @@ def raise_parse_error(error_message, location) 102, 103, 104, 105, 106, 110, 118, 119, 120, 121, 122, 123, 125, 145, 147, 148, 149, 150, 151, 152, 153, 154, 156, 160, 162, 168, 173, 177, 187, 190, - 197, 198, 203, 206, 211, 216, 222, 223 ] + 197, 198, 203, 206, 211, 216, 220, 222, 224 ] racc_action_pointer = [ nil, 20, 9, 26, 97, nil, nil, 23, nil, 32, @@ -814,10 +814,10 @@ def raise_parse_error(error_message, location) 229, 166, 163, nil, nil, nil, nil, 225, 221, -16, nil, 183, 188, 264, 189, nil, 253, 9, nil, nil, 194, 272, nil, 172, nil, nil, 225, 173, nil, nil, - nil, nil, 256, 227, nil ] + 268, nil, 257, nil, 228, nil ] racc_action_default = [ - -2, -136, -8, -136, -136, -3, -4, -136, 225, -136, + -2, -136, -8, -136, -136, -3, -4, -136, 226, -136, -9, -10, -11, -136, -136, -136, -136, -136, -136, -136, -23, -24, -136, -28, -136, -136, -136, -136, -136, -136, -136, -136, -136, -136, -136, -136, -136, -136, -136, -136, @@ -839,58 +839,56 @@ def raise_parse_error(error_message, location) -92, -136, -114, -105, -135, -108, -130, -60, -118, -92, -65, -136, -136, -134, -136, -116, -136, -59, -62, -63, -136, -136, -68, -136, -106, -115, -118, -136, -66, -117, - -109, -64, -136, -118, -67 ] + -134, -64, -136, -109, -118, -67 ] racc_goto_table = [ - 93, 51, 73, 68, 116, 75, 108, 173, 193, 1, - 188, 196, 2, 191, 117, 196, 196, 141, 4, 71, - 41, 83, 83, 83, 83, 42, 79, 84, 85, 86, - 52, 54, 55, 181, 185, 186, 89, 5, 214, 207, - 109, 116, 205, 114, 213, 113, 75, 108, 135, 209, - 170, 39, 217, 119, 10, 71, 71, 90, 11, 116, - 12, 48, 95, 125, 162, 102, 147, 83, 83, 108, - 103, 148, 104, 149, 105, 150, 106, 151, 131, 67, - 72, 134, 110, 132, 75, 136, 113, 187, 211, 222, - 123, 160, 100, 145, 71, 140, 71, 177, 206, 120, - nil, nil, 83, nil, 83, nil, 113, nil, nil, nil, - nil, nil, 172, 157, nil, nil, nil, nil, 71, nil, - nil, nil, 83, nil, nil, nil, nil, nil, nil, nil, - nil, 178, nil, nil, nil, nil, nil, nil, nil, nil, - nil, nil, 157, 192, nil, nil, nil, nil, nil, nil, - nil, nil, nil, 208, nil, nil, nil, nil, nil, nil, - 198, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, 220, nil, 212, 192, nil, 215, nil, 224, 198, - nil, nil, 192 ] + 93, 75, 51, 68, 73, 193, 116, 108, 191, 173, + 196, 1, 117, 2, 196, 196, 141, 4, 42, 41, + 71, 89, 83, 83, 83, 83, 188, 79, 84, 85, + 86, 52, 54, 55, 5, 214, 181, 185, 186, 213, + 109, 113, 75, 116, 205, 114, 135, 217, 108, 170, + 90, 209, 223, 39, 119, 207, 71, 71, 10, 11, + 12, 116, 48, 95, 125, 162, 102, 147, 83, 83, + 108, 103, 148, 104, 149, 105, 150, 106, 131, 151, + 75, 67, 113, 134, 72, 110, 132, 136, 187, 211, + 222, 123, 160, 100, 145, 71, 140, 71, 177, 206, + 120, nil, 113, 83, nil, 83, nil, nil, nil, 157, + nil, nil, 172, nil, nil, nil, nil, nil, nil, 71, + nil, nil, nil, 83, nil, nil, nil, 178, nil, nil, + nil, nil, nil, nil, nil, nil, nil, nil, 157, 192, + nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, nil, 208, nil, nil, 198, nil, nil, nil, + nil, nil, nil, nil, nil, nil, nil, nil, nil, 212, + 192, 220, 215, nil, nil, 198, nil, nil, 192, 225 ] racc_goto_check = [ - 41, 34, 46, 32, 53, 40, 33, 42, 59, 1, - 39, 63, 2, 43, 52, 63, 63, 58, 3, 34, - 4, 34, 34, 34, 34, 54, 31, 31, 31, 31, - 14, 14, 14, 20, 20, 20, 5, 6, 59, 39, - 32, 53, 42, 46, 43, 40, 40, 33, 52, 42, - 58, 7, 43, 8, 9, 34, 34, 54, 10, 53, - 11, 12, 13, 15, 16, 17, 18, 34, 34, 33, - 21, 22, 23, 24, 25, 26, 27, 28, 32, 29, - 30, 46, 35, 36, 40, 37, 40, 38, 44, 45, - 48, 49, 50, 51, 34, 57, 34, 60, 61, 62, - nil, nil, 34, nil, 34, nil, 40, nil, nil, nil, - nil, nil, 41, 40, nil, nil, nil, nil, 34, nil, - nil, nil, 34, nil, nil, nil, nil, nil, nil, nil, - nil, 40, nil, nil, nil, nil, nil, nil, nil, nil, - nil, nil, 40, 40, nil, nil, nil, nil, nil, nil, - nil, nil, nil, 41, nil, nil, nil, nil, nil, nil, - 40, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, 41, nil, 40, 40, nil, 40, nil, 41, 40, - nil, nil, 40 ] + 41, 40, 34, 32, 46, 59, 53, 33, 43, 42, + 63, 1, 52, 2, 63, 63, 58, 3, 54, 4, + 34, 5, 34, 34, 34, 34, 39, 31, 31, 31, + 31, 14, 14, 14, 6, 59, 20, 20, 20, 43, + 32, 40, 40, 53, 42, 46, 52, 43, 33, 58, + 54, 42, 59, 7, 8, 39, 34, 34, 9, 10, + 11, 53, 12, 13, 15, 16, 17, 18, 34, 34, + 33, 21, 22, 23, 24, 25, 26, 27, 32, 28, + 40, 29, 40, 46, 30, 35, 36, 37, 38, 44, + 45, 48, 49, 50, 51, 34, 57, 34, 60, 61, + 62, nil, 40, 34, nil, 34, nil, nil, nil, 40, + nil, nil, 41, nil, nil, nil, nil, nil, nil, 34, + nil, nil, nil, 34, nil, nil, nil, 40, nil, nil, + nil, nil, nil, nil, nil, nil, nil, nil, 40, 40, + nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, nil, 41, nil, nil, 40, nil, nil, nil, + nil, nil, nil, nil, nil, nil, nil, nil, nil, 40, + 40, 41, 40, nil, nil, 40, nil, nil, 40, 41 ] racc_goto_pointer = [ - nil, 9, 12, 16, 11, -5, 35, 45, -35, 50, - 54, 56, 47, 14, 15, -38, -82, 8, -60, nil, - -132, 12, -56, 13, -55, 14, -54, 15, -53, 47, - 47, -8, -29, -62, -13, 11, -27, -33, -81, -158, - -28, -45, -150, -158, -112, -129, -31, nil, -9, -53, - 39, -31, -67, -76, 16, nil, nil, -26, -104, -165, - -61, -97, 8, -170 ] + nil, 11, 13, 15, 10, -20, 32, 47, -34, 54, + 55, 56, 48, 15, 16, -37, -81, 9, -59, nil, + -129, 13, -55, 14, -54, 15, -53, 16, -51, 49, + 51, -7, -29, -61, -12, 14, -24, -31, -80, -142, + -32, -45, -148, -163, -111, -128, -29, nil, -8, -52, + 40, -30, -69, -74, 9, nil, nil, -25, -105, -168, + -60, -96, 9, -171 ] racc_goto_default = [ nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, @@ -1011,7 +1009,7 @@ def raise_parse_error(error_message, location) 6, 111, :_reduce_106, 0, 113, :_reduce_107, 0, 114, :_reduce_108, - 7, 111, :_reduce_109, + 8, 111, :_reduce_109, 3, 111, :_reduce_110, 1, 95, :_reduce_111, 1, 95, :_reduce_112, @@ -1041,7 +1039,7 @@ def raise_parse_error(error_message, location) racc_reduce_n = 136 -racc_shift_n = 225 +racc_shift_n = 226 racc_token_table = { false => 0, @@ -2066,6 +2064,7 @@ def _reduce_108(val, _values, result) def _reduce_109(val, _values, result) user_code = val[3] user_code.alias_name = val[6] + user_code.tag = val[7] builder = val[0] builder.user_code = user_code result = builder @@ -2074,7 +2073,7 @@ def _reduce_109(val, _values, result) end .,., -module_eval(<<'.,.,', 'parser.y', 474) +module_eval(<<'.,.,', 'parser.y', 475) def _reduce_110(val, _values, result) sym = @grammar.find_symbol_by_id!(val[2]) @prec_seen = true @@ -2086,49 +2085,49 @@ def _reduce_110(val, _values, result) end .,., -module_eval(<<'.,.,', 'parser.y', 481) +module_eval(<<'.,.,', 'parser.y', 482) def _reduce_111(val, _values, result) result = "option" result end .,., -module_eval(<<'.,.,', 'parser.y', 482) +module_eval(<<'.,.,', 'parser.y', 483) def _reduce_112(val, _values, result) result = "nonempty_list" result end .,., -module_eval(<<'.,.,', 'parser.y', 483) +module_eval(<<'.,.,', 'parser.y', 484) def _reduce_113(val, _values, result) result = "list" result end .,., -module_eval(<<'.,.,', 'parser.y', 485) +module_eval(<<'.,.,', 'parser.y', 486) def _reduce_114(val, _values, result) result = [val[0]] result end .,., -module_eval(<<'.,.,', 'parser.y', 486) +module_eval(<<'.,.,', 'parser.y', 487) def _reduce_115(val, _values, result) result = val[0].append(val[2]) result end .,., -module_eval(<<'.,.,', 'parser.y', 487) +module_eval(<<'.,.,', 'parser.y', 488) def _reduce_116(val, _values, result) result = [Lrama::Lexer::Token::InstantiateRule.new(s_value: val[1].s_value, location: @lexer.location, args: val[0])] result end .,., -module_eval(<<'.,.,', 'parser.y', 488) +module_eval(<<'.,.,', 'parser.y', 489) def _reduce_117(val, _values, result) result = [Lrama::Lexer::Token::InstantiateRule.new(s_value: val[0].s_value, location: @lexer.location, args: val[2])] result @@ -2137,7 +2136,7 @@ def _reduce_117(val, _values, result) # reduce 118 omitted -module_eval(<<'.,.,', 'parser.y', 491) +module_eval(<<'.,.,', 'parser.y', 492) def _reduce_119(val, _values, result) result = val[1].s_value result @@ -2148,7 +2147,7 @@ def _reduce_119(val, _values, result) # reduce 121 omitted -module_eval(<<'.,.,', 'parser.y', 498) +module_eval(<<'.,.,', 'parser.y', 499) def _reduce_122(val, _values, result) begin_c_declaration('\Z') @grammar.epilogue_first_lineno = @lexer.line + 1 @@ -2157,7 +2156,7 @@ def _reduce_122(val, _values, result) end .,., -module_eval(<<'.,.,', 'parser.y', 503) +module_eval(<<'.,.,', 'parser.y', 504) def _reduce_123(val, _values, result) end_c_declaration @grammar.epilogue = val[2].s_value @@ -2176,14 +2175,14 @@ def _reduce_123(val, _values, result) # reduce 128 omitted -module_eval(<<'.,.,', 'parser.y', 514) +module_eval(<<'.,.,', 'parser.y', 515) def _reduce_129(val, _values, result) result = [val[0]] result end .,., -module_eval(<<'.,.,', 'parser.y', 515) +module_eval(<<'.,.,', 'parser.y', 516) def _reduce_130(val, _values, result) result = val[0].append(val[1]) result @@ -2194,7 +2193,7 @@ def _reduce_130(val, _values, result) # reduce 132 omitted -module_eval(<<'.,.,', 'parser.y', 520) +module_eval(<<'.,.,', 'parser.y', 521) def _reduce_133(val, _values, result) result = Lrama::Lexer::Token::Ident.new(s_value: val[0]) result diff --git a/tool/lrama/lib/lrama/version.rb b/tool/lrama/lib/lrama/version.rb index 0070c177c65bd3..ccd593f344f532 100644 --- a/tool/lrama/lib/lrama/version.rb +++ b/tool/lrama/lib/lrama/version.rb @@ -1,3 +1,3 @@ module Lrama - VERSION = "0.6.4".freeze + VERSION = "0.6.5".freeze end From 95864a6e353b503920e9b0c3d65c0bab2ab4a937 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 22 Mar 2024 02:32:35 +0000 Subject: [PATCH 008/211] Bump dependabot/fetch-metadata from 1.6.0 to 2.0.0 Bumps [dependabot/fetch-metadata](https://github.com/dependabot/fetch-metadata) from 1.6.0 to 2.0.0. - [Release notes](https://github.com/dependabot/fetch-metadata/releases) - [Commits](https://github.com/dependabot/fetch-metadata/compare/c9c4182bf1b97f5224aee3906fd373f6b61b4526...0fb21704c18a42ce5aa8d720ea4b912f5e6babef) --- updated-dependencies: - dependency-name: dependabot/fetch-metadata dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/dependabot_automerge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dependabot_automerge.yml b/.github/workflows/dependabot_automerge.yml index 5858507ae73763..ca9e63f1ec8749 100644 --- a/.github/workflows/dependabot_automerge.yml +++ b/.github/workflows/dependabot_automerge.yml @@ -11,7 +11,7 @@ jobs: steps: - name: Dependabot metadata - uses: dependabot/fetch-metadata@c9c4182bf1b97f5224aee3906fd373f6b61b4526 # v1.6.0 + uses: dependabot/fetch-metadata@0fb21704c18a42ce5aa8d720ea4b912f5e6babef # v2.0.0 id: metadata - name: Wait for status checks From fdd7ffb70ca6e9f7d790aadde86dbc8172e19f4d Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Mon, 25 Mar 2024 09:24:21 +0900 Subject: [PATCH 009/211] [Bug #20389] Chilled string cannot be a shared root --- string.c | 2 +- test/ruby/test_string.rb | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/string.c b/string.c index cbd8f905858410..9d84b16a073ec8 100644 --- a/string.c +++ b/string.c @@ -1340,7 +1340,7 @@ rb_str_new_shared(VALUE str) VALUE rb_str_new_frozen(VALUE orig) { - if (OBJ_FROZEN(orig)) return orig; + if (RB_FL_TEST_RAW(orig, FL_FREEZE | STR_CHILLED) == FL_FREEZE) return orig; return str_new_frozen(rb_obj_class(orig), orig); } diff --git a/test/ruby/test_string.rb b/test/ruby/test_string.rb index 8df6d3277f867f..8ec80d06fc7f2f 100644 --- a/test/ruby/test_string.rb +++ b/test/ruby/test_string.rb @@ -3648,6 +3648,18 @@ def setivar! Warning[:deprecated] = deprecated end + def test_chilled_string_substring + deprecated = Warning[:deprecated] + Warning[:deprecated] = false + chilled_string = eval('"a chilled string."') + substring = chilled_string[0..-1] + assert_equal("a chilled string.", substring) + chilled_string[0..-1] = "This string is defrosted." + assert_equal("a chilled string.", substring) + ensure + Warning[:deprecated] = deprecated + end + private def assert_bytesplice_result(expected, s, *args) From 348d8bdb0edeb4df2ef40379ab579dfc15afc327 Mon Sep 17 00:00:00 2001 From: David Rodriguez Date: Wed, 20 Mar 2024 16:43:26 +0100 Subject: [PATCH 010/211] Use `$ext_build_dir` consistently Instead of hardcoded "ext". --- tool/rbinstall.rb | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tool/rbinstall.rb b/tool/rbinstall.rb index 2faaece8c92d68..3a36777420672f 100755 --- a/tool/rbinstall.rb +++ b/tool/rbinstall.rb @@ -555,11 +555,17 @@ def collect class Ext < self def skip_install?(files) # install ext only when it's configured - !File.exist?("#{$ext_build_dir}/#{relative_base}/Makefile") + !File.exist?("#{makefile_dir}/Makefile") end def ruby_libraries - Dir.glob("lib/**/*.rb", base: "#{srcdir}/ext/#{relative_base}") + Dir.glob("lib/**/*.rb", base: makefile_dir) + end + + private + + def makefile_dir + File.expand_path("#{$ext_build_dir}/#{relative_base}", srcdir) end end From 1e6117d66547b11b26f4e7dc50612ff26e03b634 Mon Sep 17 00:00:00 2001 From: David Rodriguez Date: Wed, 20 Mar 2024 17:10:36 +0100 Subject: [PATCH 011/211] Simplify FileCollector interface --- tool/rbinstall.rb | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/tool/rbinstall.rb b/tool/rbinstall.rb index 3a36777420672f..62a44af6f93af9 100755 --- a/tool/rbinstall.rb +++ b/tool/rbinstall.rb @@ -553,12 +553,10 @@ def collect end class Ext < self - def skip_install?(files) + def ruby_libraries # install ext only when it's configured - !File.exist?("#{makefile_dir}/Makefile") - end + return [] unless File.exist?("#{makefile_dir}/Makefile") - def ruby_libraries Dir.glob("lib/**/*.rb", base: makefile_dir) end @@ -570,10 +568,6 @@ def makefile_dir end class Lib < self - def skip_install?(files) - files.empty? - end - def ruby_libraries gemname = File.basename(gemspec, ".gemspec") base = relative_base || gemname @@ -763,7 +757,7 @@ def install_default_gem(dir, srcdir, bindir) spec = load_gemspec("#{base}/#{src}") file_collector = RbInstall::Specs::FileCollector.for(srcdir, dir, src) files = file_collector.collect - if file_collector.skip_install?(files) + if files.empty? next end spec.files = files From bece07e6c3716f93c0f2b98159430aa5df51c0a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Rodr=C3=ADguez?= Date: Wed, 24 Jan 2024 10:16:48 +0100 Subject: [PATCH 012/211] Fix gemspec file list for extension gems So that it also includes requirable features provided by extensions. --- tool/rbinstall.rb | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/tool/rbinstall.rb b/tool/rbinstall.rb index 62a44af6f93af9..b13e6d5ba9792a 100755 --- a/tool/rbinstall.rb +++ b/tool/rbinstall.rb @@ -549,26 +549,44 @@ def initialize(gemspec, srcdir, relative_base) end def collect - ruby_libraries.sort + libraries.sort end class Ext < self - def ruby_libraries + def libraries # install ext only when it's configured - return [] unless File.exist?("#{makefile_dir}/Makefile") + return [] unless File.exist?(makefile_path) - Dir.glob("lib/**/*.rb", base: makefile_dir) + ruby_libraries + ext_libraries end private + def ruby_libraries + Dir.glob("**/*.rb", base: "#{makefile_dir}/lib") + end + + def ext_libraries + makefile = File.read(makefile_path) + + name = makefile[/^TARGET[ \t]*=[ \t]*((?:.*\\\n)*.*)/, 1] + return [] if name.empty? + + feature = makefile[/^DLLIB[ \t]*=[ \t]*((?:.*\\\n)*.*)/, 1] + Array(feature.sub("$(TARGET)", name)) + end + + def makefile_path + "#{makefile_dir}/Makefile" + end + def makefile_dir File.expand_path("#{$ext_build_dir}/#{relative_base}", srcdir) end end class Lib < self - def ruby_libraries + def libraries gemname = File.basename(gemspec, ".gemspec") base = relative_base || gemname # for lib/net/net-smtp.gemspec From 2b703eed46237d28ca71f3ed24f77bf53cf81720 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Rodr=C3=ADguez?= Date: Wed, 24 Jan 2024 10:22:26 +0100 Subject: [PATCH 013/211] Consistently put requirable features in default gemspecs file list --- tool/rbinstall.rb | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tool/rbinstall.rb b/tool/rbinstall.rb index b13e6d5ba9792a..ee09397cd24b73 100755 --- a/tool/rbinstall.rb +++ b/tool/rbinstall.rb @@ -549,24 +549,24 @@ def initialize(gemspec, srcdir, relative_base) end def collect - libraries.sort + requirable_features.sort end class Ext < self - def libraries + def requirable_features # install ext only when it's configured return [] unless File.exist?(makefile_path) - ruby_libraries + ext_libraries + ruby_features + ext_features end private - def ruby_libraries + def ruby_features Dir.glob("**/*.rb", base: "#{makefile_dir}/lib") end - def ext_libraries + def ext_features makefile = File.read(makefile_path) name = makefile[/^TARGET[ \t]*=[ \t]*((?:.*\\\n)*.*)/, 1] @@ -586,24 +586,24 @@ def makefile_dir end class Lib < self - def libraries + def requirable_features gemname = File.basename(gemspec, ".gemspec") base = relative_base || gemname # for lib/net/net-smtp.gemspec if m = /.*(?=-(.*)\z)/.match(gemname) base = File.join(base, *m.to_a.select {|n| !base.include?(n)}) end - files = Dir.glob("lib/#{base}{.rb,/**/*.rb}", base: srcdir) + files = Dir.glob("#{base}{.rb,/**/*.rb}", base: "#{srcdir}/lib") if !relative_base and files.empty? # no files at the toplevel # pseudo gem like ruby2_keywords - files << "lib/#{gemname}.rb" + files << "#{gemname}.rb" end case gemname when "net-http" - files << "lib/net/https.rb" + files << "net/https.rb" when "optparse" - files << "lib/optionparser.rb" + files << "optionparser.rb" end files From 083e7080d19a5f6489cae5a060c533ea1932de75 Mon Sep 17 00:00:00 2001 From: David Rodriguez Date: Mon, 18 Mar 2024 20:58:49 +0100 Subject: [PATCH 014/211] Consider `target_prefix` in extension Makefiles --- tool/rbinstall.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tool/rbinstall.rb b/tool/rbinstall.rb index ee09397cd24b73..1a5119d87e91c5 100755 --- a/tool/rbinstall.rb +++ b/tool/rbinstall.rb @@ -573,7 +573,12 @@ def ext_features return [] if name.empty? feature = makefile[/^DLLIB[ \t]*=[ \t]*((?:.*\\\n)*.*)/, 1] - Array(feature.sub("$(TARGET)", name)) + feature = feature.sub("$(TARGET)", name) + + target_prefix = makefile[/^target_prefix[ \t]*=[ \t]*((?:.*\\\n)*.*)/, 1] + feature = File.join(target_prefix.delete_prefix("/"), feature) unless target_prefix.empty? + + Array(feature) end def makefile_path From 2dd2204d8cac9ffb25cbd415fabaf315e0c7e839 Mon Sep 17 00:00:00 2001 From: David Rodriguez Date: Wed, 20 Mar 2024 17:17:40 +0100 Subject: [PATCH 015/211] Extract `root` helper It holds the root directory for each type of default gem (ext/ or lib/). --- tool/rbinstall.rb | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tool/rbinstall.rb b/tool/rbinstall.rb index 1a5119d87e91c5..c5ee78cc798dbb 100755 --- a/tool/rbinstall.rb +++ b/tool/rbinstall.rb @@ -586,7 +586,11 @@ def makefile_path end def makefile_dir - File.expand_path("#{$ext_build_dir}/#{relative_base}", srcdir) + "#{root}/#{relative_base}" + end + + def root + File.expand_path($ext_build_dir, srcdir) end end @@ -598,7 +602,7 @@ def requirable_features if m = /.*(?=-(.*)\z)/.match(gemname) base = File.join(base, *m.to_a.select {|n| !base.include?(n)}) end - files = Dir.glob("#{base}{.rb,/**/*.rb}", base: "#{srcdir}/lib") + files = Dir.glob("#{base}{.rb,/**/*.rb}", base: root) if !relative_base and files.empty? # no files at the toplevel # pseudo gem like ruby2_keywords files << "#{gemname}.rb" @@ -613,6 +617,10 @@ def requirable_features files end + + def root + "#{srcdir}/lib" + end end end end From ea31228d0c46763a62abef047d37d66709bdaf87 Mon Sep 17 00:00:00 2001 From: David Rodriguez Date: Wed, 20 Mar 2024 17:18:20 +0100 Subject: [PATCH 016/211] Consider extensions in gems outside of ext/ --- tool/rbinstall.rb | 49 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 37 insertions(+), 12 deletions(-) diff --git a/tool/rbinstall.rb b/tool/rbinstall.rb index c5ee78cc798dbb..a2149317500813 100755 --- a/tool/rbinstall.rb +++ b/tool/rbinstall.rb @@ -552,6 +552,23 @@ def collect requirable_features.sort end + private + + def features_from_makefile(makefile_path) + makefile = File.read(makefile_path) + + name = makefile[/^TARGET[ \t]*=[ \t]*((?:.*\\\n)*.*)/, 1] + return [] if name.empty? + + feature = makefile[/^DLLIB[ \t]*=[ \t]*((?:.*\\\n)*.*)/, 1] + feature = feature.sub("$(TARGET)", name) + + target_prefix = makefile[/^target_prefix[ \t]*=[ \t]*((?:.*\\\n)*.*)/, 1] + feature = File.join(target_prefix.delete_prefix("/"), feature) unless target_prefix.empty? + + Array(feature) + end + class Ext < self def requirable_features # install ext only when it's configured @@ -567,18 +584,7 @@ def ruby_features end def ext_features - makefile = File.read(makefile_path) - - name = makefile[/^TARGET[ \t]*=[ \t]*((?:.*\\\n)*.*)/, 1] - return [] if name.empty? - - feature = makefile[/^DLLIB[ \t]*=[ \t]*((?:.*\\\n)*.*)/, 1] - feature = feature.sub("$(TARGET)", name) - - target_prefix = makefile[/^target_prefix[ \t]*=[ \t]*((?:.*\\\n)*.*)/, 1] - feature = File.join(target_prefix.delete_prefix("/"), feature) unless target_prefix.empty? - - Array(feature) + features_from_makefile(makefile_path) end def makefile_path @@ -596,6 +602,12 @@ def root class Lib < self def requirable_features + ruby_features + ext_features + end + + private + + def ruby_features gemname = File.basename(gemspec, ".gemspec") base = relative_base || gemname # for lib/net/net-smtp.gemspec @@ -618,6 +630,19 @@ def requirable_features files end + def ext_features + loaded_gemspec = Gem::Specification.load("#{root}/#{gemspec}") + extension = loaded_gemspec.extensions.first + return [] unless extension + + extconf = File.expand_path(extension, srcdir) + ext_build_dir = File.dirname(extconf) + makefile_path = "#{ext_build_dir}/Makefile" + return [] unless File.exist?(makefile_path) + + features_from_makefile(makefile_path) + end + def root "#{srcdir}/lib" end From 48d3bdddbaeabed5fb6a97bfbe65e250d1383a9c Mon Sep 17 00:00:00 2001 From: KJ Tsanaktsidis Date: Sat, 24 Feb 2024 19:31:27 +1100 Subject: [PATCH 017/211] Move asan_fake_stack_handle to EC, not thread It's really a property of the EC; each fiber (which has its own EC) also has its own asan_fake_stack_handle. [Bug #20310] --- gc.c | 2 +- thread.c | 3 --- thread_pthread.c | 1 + vm_core.h | 9 ++++----- 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/gc.c b/gc.c index eaa98928ffc6d2..07c0735a44c9bb 100644 --- a/gc.c +++ b/gc.c @@ -6410,7 +6410,7 @@ gc_mark_machine_stack_location_maybe(rb_objspace_t *objspace, VALUE obj) void *fake_frame_start; void *fake_frame_end; bool is_fake_frame = asan_get_fake_stack_extents( - ec->thread_ptr->asan_fake_stack_handle, obj, + ec->machine.asan_fake_stack_handle, obj, ec->machine.stack_start, ec->machine.stack_end, &fake_frame_start, &fake_frame_end ); diff --git a/thread.c b/thread.c index b8dd8626344b87..e672172a4deec3 100644 --- a/thread.c +++ b/thread.c @@ -527,9 +527,6 @@ void ruby_thread_init_stack(rb_thread_t *th, void *local_in_parent_frame) { native_thread_init_stack(th, local_in_parent_frame); -#ifdef RUBY_ASAN_ENABLED - th->asan_fake_stack_handle = asan_get_thread_fake_stack_handle(); -#endif } const VALUE * diff --git a/thread_pthread.c b/thread_pthread.c index fb446eee82c08e..cdaf6f240c436a 100644 --- a/thread_pthread.c +++ b/thread_pthread.c @@ -2081,6 +2081,7 @@ native_thread_init_stack(rb_thread_t *th, void *local_in_parent_frame) rb_nativethread_id_t curr = pthread_self(); #ifdef RUBY_ASAN_ENABLED local_in_parent_frame = asan_get_real_stack_addr(local_in_parent_frame); + th->ec->machine.asan_fake_stack_handle = asan_get_thread_fake_stack_handle(); #endif if (!native_main_thread.id) { diff --git a/vm_core.h b/vm_core.h index da1f568b66194c..805df186162d16 100644 --- a/vm_core.h +++ b/vm_core.h @@ -1044,6 +1044,10 @@ struct rb_execution_context_struct { VALUE *stack_end; size_t stack_maxsize; RUBY_ALIGNAS(SIZEOF_VALUE) jmp_buf regs; + +#ifdef RUBY_ASAN_ENABLED + void *asan_fake_stack_handle; +#endif } machine; }; @@ -1164,11 +1168,6 @@ typedef struct rb_thread_struct { void **specific_storage; struct rb_ext_config ext_config; - -#ifdef RUBY_ASAN_ENABLED - void *asan_fake_stack_handle; -#endif - } rb_thread_t; static inline unsigned int From 2535a09e85dd68249d9f390db32313ddd482480e Mon Sep 17 00:00:00 2001 From: KJ Tsanaktsidis Date: Sat, 24 Feb 2024 19:32:17 +1100 Subject: [PATCH 018/211] Check ASAN fake stacks when marking non-current threads Currently, we check the values on the machine stack & register state to see if they're actually a pointer to an ASAN fake stack, and mark the values on the fake stack too if required. However, we are only doing that for the _current_ thread (the one actually running the GC), not for any other thread in the program. Make rb_gc_mark_machine_context (which is called for marking non-current threads) perform the same ASAN fake stack handling that mark_current_machine_context performs. [Bug #20310] --- gc.c | 20 ++++++++++++++++---- vm.c | 5 +---- vm_core.h | 2 +- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/gc.c b/gc.c index 07c0735a44c9bb..282e9df0cfd653 100644 --- a/gc.c +++ b/gc.c @@ -949,7 +949,7 @@ typedef struct rb_objspace { rb_postponed_job_handle_t finalize_deferred_pjob; #ifdef RUBY_ASAN_ENABLED - rb_execution_context_t *marking_machine_context_ec; + const rb_execution_context_t *marking_machine_context_ec; #endif } rb_objspace_t; @@ -6406,7 +6406,7 @@ gc_mark_machine_stack_location_maybe(rb_objspace_t *objspace, VALUE obj) gc_mark_maybe(objspace, obj); #ifdef RUBY_ASAN_ENABLED - rb_execution_context_t *ec = objspace->marking_machine_context_ec; + const rb_execution_context_t *ec = objspace->marking_machine_context_ec; void *fake_frame_start; void *fake_frame_end; bool is_fake_frame = asan_get_fake_stack_extents( @@ -6495,13 +6495,25 @@ mark_current_machine_context(rb_objspace_t *objspace, rb_execution_context_t *ec #endif void -rb_gc_mark_machine_stack(const rb_execution_context_t *ec) +rb_gc_mark_machine_context(const rb_execution_context_t *ec) { + rb_objspace_t *objspace = &rb_objspace; +#ifdef RUBY_ASAN_ENABLED + objspace->marking_machine_context_ec = ec; +#endif + VALUE *stack_start, *stack_end; + GET_STACK_BOUNDS(stack_start, stack_end, 0); RUBY_DEBUG_LOG("ec->th:%u stack_start:%p stack_end:%p", rb_ec_thread_ptr(ec)->serial, stack_start, stack_end); - rb_gc_mark_locations(stack_start, stack_end); + each_stack_location(objspace, ec, stack_start, stack_end, gc_mark_machine_stack_location_maybe); + int num_regs = sizeof(ec->machine.regs)/(sizeof(VALUE)); + each_location(objspace, (VALUE*)&ec->machine.regs, num_regs, gc_mark_machine_stack_location_maybe); + +#ifdef RUBY_ASAN_ENABLED + objspace->marking_machine_context_ec = NULL; +#endif } static void diff --git a/vm.c b/vm.c index 82bf1ac658d3f3..882514db41e57a 100644 --- a/vm.c +++ b/vm.c @@ -3399,10 +3399,7 @@ rb_execution_context_mark(const rb_execution_context_t *ec) if (ec->machine.stack_start && ec->machine.stack_end && ec != GET_EC() /* marked for current ec at the first stage of marking */ ) { - rb_gc_mark_machine_stack(ec); - rb_gc_mark_locations((VALUE *)&ec->machine.regs, - (VALUE *)(&ec->machine.regs) + - sizeof(ec->machine.regs) / (sizeof(VALUE))); + rb_gc_mark_machine_context(ec); } rb_gc_mark(ec->errinfo); diff --git a/vm_core.h b/vm_core.h index 805df186162d16..2a9d5f906f101f 100644 --- a/vm_core.h +++ b/vm_core.h @@ -1874,7 +1874,7 @@ void rb_vm_register_special_exception_str(enum ruby_special_exceptions sp, VALUE #define rb_vm_register_special_exception(sp, e, m) \ rb_vm_register_special_exception_str(sp, e, rb_usascii_str_new_static((m), (long)rb_strlen_lit(m))) -void rb_gc_mark_machine_stack(const rb_execution_context_t *ec); +void rb_gc_mark_machine_context(const rb_execution_context_t *ec); void rb_vm_rewrite_cref(rb_cref_t *node, VALUE old_klass, VALUE new_klass, rb_cref_t **new_cref_ptr); From 69579ed57a2aa1c3ad739417db70564d570bf2c1 Mon Sep 17 00:00:00 2001 From: KJ Tsanaktsidis Date: Wed, 28 Feb 2024 14:37:25 +1100 Subject: [PATCH 019/211] Mark fiber stacks in rb_execution_context_mark Currently, fiber stacks are marked separately from the rest of the execution context. The fiber code deliberately does _NOT_ set ec->machine.stack_end on the saved EC, so that the code in `rb_execution_context_mark` does not mark it; instead, the stack marking is done in `cont_mark`. Instead, we can set ec->machine.stack_end, and skip out on doing the stack marking separately in `cont_mark`; that way, all machine stack marking shares the same code (which does the nescessary ASAN things). [Bug #20310] --- cont.c | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/cont.c b/cont.c index b07b5e380f4b0a..c6a94e0709f369 100644 --- a/cont.c +++ b/cont.c @@ -796,6 +796,9 @@ static inline void ec_switch(rb_thread_t *th, rb_fiber_t *fiber) { rb_execution_context_t *ec = &fiber->cont.saved_ec; +#ifdef RUBY_ASAN_ENABLED + ec->machine.asan_fake_stack_handle = asan_get_thread_fake_stack_handle(); +#endif rb_ractor_set_current_ec(th->ractor, th->ec = ec); // ruby_current_execution_context_ptr = th->ec = ec; @@ -1023,13 +1026,8 @@ cont_mark(void *ptr) cont->machine.stack + cont->machine.stack_size); } else { - /* fiber */ - const rb_fiber_t *fiber = (rb_fiber_t*)cont; - - if (!FIBER_TERMINATED_P(fiber)) { - rb_gc_mark_locations(cont->machine.stack, - cont->machine.stack + cont->machine.stack_size); - } + /* fiber machine context is marked as part of rb_execution_context_mark, no need to + * do anything here. */ } } @@ -1568,11 +1566,10 @@ fiber_setcontext(rb_fiber_t *new_fiber, rb_fiber_t *old_fiber) } } - /* exchange machine_stack_start between old_fiber and new_fiber */ + /* these values are used in rb_gc_mark_machine_context to mark the fiber's stack. */ old_fiber->cont.saved_ec.machine.stack_start = th->ec->machine.stack_start; + old_fiber->cont.saved_ec.machine.stack_end = FIBER_TERMINATED_P(old_fiber) ? NULL : th->ec->machine.stack_end; - /* old_fiber->machine.stack_end should be NULL */ - old_fiber->cont.saved_ec.machine.stack_end = NULL; // if (DEBUG) fprintf(stderr, "fiber_setcontext: %p[%p] -> %p[%p]\n", (void*)old_fiber, old_fiber->stack.base, (void*)new_fiber, new_fiber->stack.base); From 9db300219f75db5951b7e2f61683b1241e7afcf0 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Mon, 25 Mar 2024 12:45:30 +0900 Subject: [PATCH 020/211] Added test script for bundled_gems.rb with zeitwerk --- tool/test_for_warn_bundled_gems/test.sh | 3 +++ .../test_no_warn_zeitwerk.rb | 12 ++++++++++++ 2 files changed, 15 insertions(+) create mode 100644 tool/test_for_warn_bundled_gems/test_no_warn_zeitwerk.rb diff --git a/tool/test_for_warn_bundled_gems/test.sh b/tool/test_for_warn_bundled_gems/test.sh index ce714c7e13314f..2b1279eeb17112 100755 --- a/tool/test_for_warn_bundled_gems/test.sh +++ b/tool/test_for_warn_bundled_gems/test.sh @@ -24,6 +24,9 @@ ruby test_no_warn_dependency.rb echo "* Don't show warning with bootsnap" ruby test_no_warn_bootsnap.rb +echo "* Don't show warning with zeitwerk" +ruby test_no_warn_zeitwerk.rb + echo "* Don't show warning with net/smtp when net-smtp on Gemfile" ruby test_no_warn_dash_gem.rb diff --git a/tool/test_for_warn_bundled_gems/test_no_warn_zeitwerk.rb b/tool/test_for_warn_bundled_gems/test_no_warn_zeitwerk.rb new file mode 100644 index 00000000000000..d554a0e6756845 --- /dev/null +++ b/tool/test_for_warn_bundled_gems/test_no_warn_zeitwerk.rb @@ -0,0 +1,12 @@ +require "bundler/inline" + +gemfile do + source "https://rubygems.org" + gem "zeitwerk", require: false +end + +require "zeitwerk" +loader = Zeitwerk::Loader.for_gem(warn_on_extra_files: false) +loader.setup + +require 'csv' From 02a4bdd6076b84da35d4a5d1aa749f1e9683ad1b Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Mon, 25 Mar 2024 13:22:45 +0900 Subject: [PATCH 021/211] Add newline each test script --- tool/test_for_warn_bundled_gems/test.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tool/test_for_warn_bundled_gems/test.sh b/tool/test_for_warn_bundled_gems/test.sh index 2b1279eeb17112..adeccae4051721 100755 --- a/tool/test_for_warn_bundled_gems/test.sh +++ b/tool/test_for_warn_bundled_gems/test.sh @@ -2,33 +2,44 @@ echo "* Show warning require and LoadError" ruby test_warn_bundled_gems.rb +echo echo "* Show warning when bundled gems called as dependency" ruby test_warn_dependency.rb +echo echo "* Show warning sub-feature like bigdecimal/util" ruby test_warn_sub_feature.rb +echo echo "* Show warning dash gem like net/smtp" ruby test_warn_dash_gem.rb +echo echo "* Show warning when bundle exec with ruby and script" bundle exec ruby test_warn_bundle_exec.rb +echo echo "* Show warning when bundle exec with shebang's script" bundle exec ./test_warn_bundle_exec_shebang.rb +echo echo "* Don't show warning bundled gems on Gemfile" ruby test_no_warn_dependency.rb +echo echo "* Don't show warning with bootsnap" ruby test_no_warn_bootsnap.rb +echo echo "* Don't show warning with zeitwerk" ruby test_no_warn_zeitwerk.rb +echo echo "* Don't show warning with net/smtp when net-smtp on Gemfile" ruby test_no_warn_dash_gem.rb +echo echo "* Don't show warning bigdecimal/util when bigdecimal on Gemfile" ruby test_no_warn_sub_feature.rb +echo From d6e9367edb940ea26a51e56b2be1032a97eea4c2 Mon Sep 17 00:00:00 2001 From: Xavier Noria Date: Wed, 13 Mar 2024 22:52:41 +0100 Subject: [PATCH 022/211] Skip Bootsnap and Zeitwerk in bundled gems warning --- lib/bundled_gems.rb | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/lib/bundled_gems.rb b/lib/bundled_gems.rb index 55286725c0fb45..1578b48040f371 100644 --- a/lib/bundled_gems.rb +++ b/lib/bundled_gems.rb @@ -109,12 +109,7 @@ def self.warning?(name, specs: nil) else return end - # Warning feature is not working correctly with Bootsnap. - # caller_locations returns: - # lib/ruby/3.3.0+0/bundled_gems.rb:65:in `block (2 levels) in replace_require' - # $GEM_HOME/gems/bootsnap-1.17.0/lib/bootsnap/load_path_cache/core_ext/kernel_require.rb:32:in `require'" - # ... - return if caller_locations(2).find {|c| c&.path.match?(/bootsnap/) } + return if WARNED[name] WARNED[name] = true if gem == true @@ -134,11 +129,29 @@ def self.build_message(gem) if defined?(Bundler) msg += " Add #{gem} to your Gemfile or gemspec." + # We detect the gem name from caller_locations. We need to skip 2 frames like: # lib/ruby/3.3.0+0/bundled_gems.rb:90:in `warning?'", # lib/ruby/3.3.0+0/bundler/rubygems_integration.rb:247:in `block (2 levels) in replace_require'", - location = caller_locations(3,1)[0]&.path - if File.file?(location) && !location.start_with?(Gem::BUNDLED_GEMS::LIBDIR) + # + # Additionally, we need to skip Bootsnap and Zeitwerk if present, these + # gems decorate Kernel#require, so they are not really the ones issuing + # the require call users should be warned about. Those are upwards. + frames_to_skip = 2 + location = nil + Thread.each_caller_location do |cl| + if frames_to_skip >= 1 + frames_to_skip -= 1 + next + end + + unless cl.path.match?(/bootsnap|zeitwerk/) + location = cl.path + break + end + end + + if location && File.file?(location) && !location.start_with?(Gem::BUNDLED_GEMS::LIBDIR) caller_gem = nil Gem.path.each do |path| if location =~ %r{#{path}/gems/([\w\-\.]+)} From 6b1691ebe8d8134c963f99b059beb449a8ec450b Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Mon, 25 Mar 2024 13:29:22 +0900 Subject: [PATCH 023/211] Remove rubylibdir from bootsnap --- lib/bundled_gems.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/bundled_gems.rb b/lib/bundled_gems.rb index 1578b48040f371..e756af61eaaed4 100644 --- a/lib/bundled_gems.rb +++ b/lib/bundled_gems.rb @@ -95,8 +95,10 @@ def self.find_gem(path) end def self.warning?(name, specs: nil) - feature = File.path(name) # name can be a feature name or a file path with String or Pathname - name = feature.tr("/", "-") + # name can be a feature name or a file path with String or Pathname + feature = File.path(name) + # bootsnap expand `require "csv"` to `require "#{LIBDIR}/csv.rb"` + name = feature.delete_prefix(LIBDIR).chomp(".rb").tr("/", "-") name.sub!(LIBEXT, "") return if specs.include?(name) _t, path = $:.resolve_feature_path(feature) From 990e11b60eae06d7e8205405fc99320267b5843b Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Mon, 25 Mar 2024 13:31:19 +0900 Subject: [PATCH 024/211] Bootsnap and Zeitwerk are working correctly now --- tool/test_for_warn_bundled_gems/test.sh | 12 ++++++------ ...est_no_warn_bootsnap.rb => test_warn_bootsnap.rb} | 0 ...est_no_warn_zeitwerk.rb => test_warn_zeitwerk.rb} | 0 3 files changed, 6 insertions(+), 6 deletions(-) rename tool/test_for_warn_bundled_gems/{test_no_warn_bootsnap.rb => test_warn_bootsnap.rb} (100%) rename tool/test_for_warn_bundled_gems/{test_no_warn_zeitwerk.rb => test_warn_zeitwerk.rb} (100%) diff --git a/tool/test_for_warn_bundled_gems/test.sh b/tool/test_for_warn_bundled_gems/test.sh index adeccae4051721..ef5007f320ab77 100755 --- a/tool/test_for_warn_bundled_gems/test.sh +++ b/tool/test_for_warn_bundled_gems/test.sh @@ -24,16 +24,16 @@ echo "* Show warning when bundle exec with shebang's script" bundle exec ./test_warn_bundle_exec_shebang.rb echo -echo "* Don't show warning bundled gems on Gemfile" -ruby test_no_warn_dependency.rb +echo "* Show warning with bootsnap" +ruby test_warn_bootsnap.rb echo -echo "* Don't show warning with bootsnap" -ruby test_no_warn_bootsnap.rb +echo "* Show warning with zeitwerk" +ruby test_warn_zeitwerk.rb echo -echo "* Don't show warning with zeitwerk" -ruby test_no_warn_zeitwerk.rb +echo "* Don't show warning bundled gems on Gemfile" +ruby test_no_warn_dependency.rb echo echo "* Don't show warning with net/smtp when net-smtp on Gemfile" diff --git a/tool/test_for_warn_bundled_gems/test_no_warn_bootsnap.rb b/tool/test_for_warn_bundled_gems/test_warn_bootsnap.rb similarity index 100% rename from tool/test_for_warn_bundled_gems/test_no_warn_bootsnap.rb rename to tool/test_for_warn_bundled_gems/test_warn_bootsnap.rb diff --git a/tool/test_for_warn_bundled_gems/test_no_warn_zeitwerk.rb b/tool/test_for_warn_bundled_gems/test_warn_zeitwerk.rb similarity index 100% rename from tool/test_for_warn_bundled_gems/test_no_warn_zeitwerk.rb rename to tool/test_for_warn_bundled_gems/test_warn_zeitwerk.rb From e720a6b48582900efd1bd326210644b2fc0f2c5d Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Mon, 25 Mar 2024 14:33:20 +0900 Subject: [PATCH 025/211] Show the chilled status of a String [ci skip] --- lib/bundled_gems.rb | 35 ++++++------------- misc/lldb_rb/utils.py | 4 +++ tool/test_for_warn_bundled_gems/test.sh | 20 ++--------- ...n_bootsnap.rb => test_no_warn_bootsnap.rb} | 0 .../test_warn_zeitwerk.rb | 12 ------- 5 files changed, 17 insertions(+), 54 deletions(-) rename tool/test_for_warn_bundled_gems/{test_warn_bootsnap.rb => test_no_warn_bootsnap.rb} (100%) delete mode 100644 tool/test_for_warn_bundled_gems/test_warn_zeitwerk.rb diff --git a/lib/bundled_gems.rb b/lib/bundled_gems.rb index e756af61eaaed4..55286725c0fb45 100644 --- a/lib/bundled_gems.rb +++ b/lib/bundled_gems.rb @@ -95,10 +95,8 @@ def self.find_gem(path) end def self.warning?(name, specs: nil) - # name can be a feature name or a file path with String or Pathname - feature = File.path(name) - # bootsnap expand `require "csv"` to `require "#{LIBDIR}/csv.rb"` - name = feature.delete_prefix(LIBDIR).chomp(".rb").tr("/", "-") + feature = File.path(name) # name can be a feature name or a file path with String or Pathname + name = feature.tr("/", "-") name.sub!(LIBEXT, "") return if specs.include?(name) _t, path = $:.resolve_feature_path(feature) @@ -111,7 +109,12 @@ def self.warning?(name, specs: nil) else return end - + # Warning feature is not working correctly with Bootsnap. + # caller_locations returns: + # lib/ruby/3.3.0+0/bundled_gems.rb:65:in `block (2 levels) in replace_require' + # $GEM_HOME/gems/bootsnap-1.17.0/lib/bootsnap/load_path_cache/core_ext/kernel_require.rb:32:in `require'" + # ... + return if caller_locations(2).find {|c| c&.path.match?(/bootsnap/) } return if WARNED[name] WARNED[name] = true if gem == true @@ -131,29 +134,11 @@ def self.build_message(gem) if defined?(Bundler) msg += " Add #{gem} to your Gemfile or gemspec." - # We detect the gem name from caller_locations. We need to skip 2 frames like: # lib/ruby/3.3.0+0/bundled_gems.rb:90:in `warning?'", # lib/ruby/3.3.0+0/bundler/rubygems_integration.rb:247:in `block (2 levels) in replace_require'", - # - # Additionally, we need to skip Bootsnap and Zeitwerk if present, these - # gems decorate Kernel#require, so they are not really the ones issuing - # the require call users should be warned about. Those are upwards. - frames_to_skip = 2 - location = nil - Thread.each_caller_location do |cl| - if frames_to_skip >= 1 - frames_to_skip -= 1 - next - end - - unless cl.path.match?(/bootsnap|zeitwerk/) - location = cl.path - break - end - end - - if location && File.file?(location) && !location.start_with?(Gem::BUNDLED_GEMS::LIBDIR) + location = caller_locations(3,1)[0]&.path + if File.file?(location) && !location.start_with?(Gem::BUNDLED_GEMS::LIBDIR) caller_gem = nil Gem.path.each do |path| if location =~ %r{#{path}/gems/([\w\-\.]+)} diff --git a/misc/lldb_rb/utils.py b/misc/lldb_rb/utils.py index a321426234befd..054c206cef02f3 100644 --- a/misc/lldb_rb/utils.py +++ b/misc/lldb_rb/utils.py @@ -119,6 +119,10 @@ def inspect(self, val): self.result.write('T_STRING: %s' % flaginfo) tRString = self.target.FindFirstType("struct RString").GetPointerType() + chilled = self.ruby_globals["RUBY_FL_USER3"] + if (rval.flags & chilled) != 0: + self.result.write("[CHILLED] ") + rb_enc_mask = self.ruby_globals["RUBY_ENCODING_MASK"] rb_enc_shift = self.ruby_globals["RUBY_ENCODING_SHIFT"] encidx = ((rval.flags & rb_enc_mask) >> rb_enc_shift) diff --git a/tool/test_for_warn_bundled_gems/test.sh b/tool/test_for_warn_bundled_gems/test.sh index ef5007f320ab77..ce714c7e13314f 100755 --- a/tool/test_for_warn_bundled_gems/test.sh +++ b/tool/test_for_warn_bundled_gems/test.sh @@ -2,44 +2,30 @@ echo "* Show warning require and LoadError" ruby test_warn_bundled_gems.rb -echo echo "* Show warning when bundled gems called as dependency" ruby test_warn_dependency.rb -echo echo "* Show warning sub-feature like bigdecimal/util" ruby test_warn_sub_feature.rb -echo echo "* Show warning dash gem like net/smtp" ruby test_warn_dash_gem.rb -echo echo "* Show warning when bundle exec with ruby and script" bundle exec ruby test_warn_bundle_exec.rb -echo echo "* Show warning when bundle exec with shebang's script" bundle exec ./test_warn_bundle_exec_shebang.rb -echo - -echo "* Show warning with bootsnap" -ruby test_warn_bootsnap.rb -echo - -echo "* Show warning with zeitwerk" -ruby test_warn_zeitwerk.rb -echo echo "* Don't show warning bundled gems on Gemfile" ruby test_no_warn_dependency.rb -echo + +echo "* Don't show warning with bootsnap" +ruby test_no_warn_bootsnap.rb echo "* Don't show warning with net/smtp when net-smtp on Gemfile" ruby test_no_warn_dash_gem.rb -echo echo "* Don't show warning bigdecimal/util when bigdecimal on Gemfile" ruby test_no_warn_sub_feature.rb -echo diff --git a/tool/test_for_warn_bundled_gems/test_warn_bootsnap.rb b/tool/test_for_warn_bundled_gems/test_no_warn_bootsnap.rb similarity index 100% rename from tool/test_for_warn_bundled_gems/test_warn_bootsnap.rb rename to tool/test_for_warn_bundled_gems/test_no_warn_bootsnap.rb diff --git a/tool/test_for_warn_bundled_gems/test_warn_zeitwerk.rb b/tool/test_for_warn_bundled_gems/test_warn_zeitwerk.rb deleted file mode 100644 index d554a0e6756845..00000000000000 --- a/tool/test_for_warn_bundled_gems/test_warn_zeitwerk.rb +++ /dev/null @@ -1,12 +0,0 @@ -require "bundler/inline" - -gemfile do - source "https://rubygems.org" - gem "zeitwerk", require: false -end - -require "zeitwerk" -loader = Zeitwerk::Loader.for_gem(warn_on_extra_files: false) -loader.setup - -require 'csv' From 675dcf9153fdf9041c1220d827fe6e6ebbf14cc8 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Mon, 25 Mar 2024 15:04:26 +0900 Subject: [PATCH 026/211] Partly Revert "Show the chilled status of a String [ci skip]" This reverts commit e720a6b48582900efd1bd326210644b2fc0f2c5d. --- lib/bundled_gems.rb | 35 +++++++++++++------ tool/test_for_warn_bundled_gems/test.sh | 20 +++++++++-- ...warn_bootsnap.rb => test_warn_bootsnap.rb} | 0 .../test_warn_zeitwerk.rb | 12 +++++++ 4 files changed, 54 insertions(+), 13 deletions(-) rename tool/test_for_warn_bundled_gems/{test_no_warn_bootsnap.rb => test_warn_bootsnap.rb} (100%) create mode 100644 tool/test_for_warn_bundled_gems/test_warn_zeitwerk.rb diff --git a/lib/bundled_gems.rb b/lib/bundled_gems.rb index 55286725c0fb45..e756af61eaaed4 100644 --- a/lib/bundled_gems.rb +++ b/lib/bundled_gems.rb @@ -95,8 +95,10 @@ def self.find_gem(path) end def self.warning?(name, specs: nil) - feature = File.path(name) # name can be a feature name or a file path with String or Pathname - name = feature.tr("/", "-") + # name can be a feature name or a file path with String or Pathname + feature = File.path(name) + # bootsnap expand `require "csv"` to `require "#{LIBDIR}/csv.rb"` + name = feature.delete_prefix(LIBDIR).chomp(".rb").tr("/", "-") name.sub!(LIBEXT, "") return if specs.include?(name) _t, path = $:.resolve_feature_path(feature) @@ -109,12 +111,7 @@ def self.warning?(name, specs: nil) else return end - # Warning feature is not working correctly with Bootsnap. - # caller_locations returns: - # lib/ruby/3.3.0+0/bundled_gems.rb:65:in `block (2 levels) in replace_require' - # $GEM_HOME/gems/bootsnap-1.17.0/lib/bootsnap/load_path_cache/core_ext/kernel_require.rb:32:in `require'" - # ... - return if caller_locations(2).find {|c| c&.path.match?(/bootsnap/) } + return if WARNED[name] WARNED[name] = true if gem == true @@ -134,11 +131,29 @@ def self.build_message(gem) if defined?(Bundler) msg += " Add #{gem} to your Gemfile or gemspec." + # We detect the gem name from caller_locations. We need to skip 2 frames like: # lib/ruby/3.3.0+0/bundled_gems.rb:90:in `warning?'", # lib/ruby/3.3.0+0/bundler/rubygems_integration.rb:247:in `block (2 levels) in replace_require'", - location = caller_locations(3,1)[0]&.path - if File.file?(location) && !location.start_with?(Gem::BUNDLED_GEMS::LIBDIR) + # + # Additionally, we need to skip Bootsnap and Zeitwerk if present, these + # gems decorate Kernel#require, so they are not really the ones issuing + # the require call users should be warned about. Those are upwards. + frames_to_skip = 2 + location = nil + Thread.each_caller_location do |cl| + if frames_to_skip >= 1 + frames_to_skip -= 1 + next + end + + unless cl.path.match?(/bootsnap|zeitwerk/) + location = cl.path + break + end + end + + if location && File.file?(location) && !location.start_with?(Gem::BUNDLED_GEMS::LIBDIR) caller_gem = nil Gem.path.each do |path| if location =~ %r{#{path}/gems/([\w\-\.]+)} diff --git a/tool/test_for_warn_bundled_gems/test.sh b/tool/test_for_warn_bundled_gems/test.sh index ce714c7e13314f..ef5007f320ab77 100755 --- a/tool/test_for_warn_bundled_gems/test.sh +++ b/tool/test_for_warn_bundled_gems/test.sh @@ -2,30 +2,44 @@ echo "* Show warning require and LoadError" ruby test_warn_bundled_gems.rb +echo echo "* Show warning when bundled gems called as dependency" ruby test_warn_dependency.rb +echo echo "* Show warning sub-feature like bigdecimal/util" ruby test_warn_sub_feature.rb +echo echo "* Show warning dash gem like net/smtp" ruby test_warn_dash_gem.rb +echo echo "* Show warning when bundle exec with ruby and script" bundle exec ruby test_warn_bundle_exec.rb +echo echo "* Show warning when bundle exec with shebang's script" bundle exec ./test_warn_bundle_exec_shebang.rb +echo + +echo "* Show warning with bootsnap" +ruby test_warn_bootsnap.rb +echo + +echo "* Show warning with zeitwerk" +ruby test_warn_zeitwerk.rb +echo echo "* Don't show warning bundled gems on Gemfile" ruby test_no_warn_dependency.rb - -echo "* Don't show warning with bootsnap" -ruby test_no_warn_bootsnap.rb +echo echo "* Don't show warning with net/smtp when net-smtp on Gemfile" ruby test_no_warn_dash_gem.rb +echo echo "* Don't show warning bigdecimal/util when bigdecimal on Gemfile" ruby test_no_warn_sub_feature.rb +echo diff --git a/tool/test_for_warn_bundled_gems/test_no_warn_bootsnap.rb b/tool/test_for_warn_bundled_gems/test_warn_bootsnap.rb similarity index 100% rename from tool/test_for_warn_bundled_gems/test_no_warn_bootsnap.rb rename to tool/test_for_warn_bundled_gems/test_warn_bootsnap.rb diff --git a/tool/test_for_warn_bundled_gems/test_warn_zeitwerk.rb b/tool/test_for_warn_bundled_gems/test_warn_zeitwerk.rb new file mode 100644 index 00000000000000..d554a0e6756845 --- /dev/null +++ b/tool/test_for_warn_bundled_gems/test_warn_zeitwerk.rb @@ -0,0 +1,12 @@ +require "bundler/inline" + +gemfile do + source "https://rubygems.org" + gem "zeitwerk", require: false +end + +require "zeitwerk" +loader = Zeitwerk::Loader.for_gem(warn_on_extra_files: false) +loader.setup + +require 'csv' From bd85fd6db9c6e065c65c82ba590b1a16a07791a5 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Mon, 25 Mar 2024 15:16:22 +0900 Subject: [PATCH 027/211] Guard makefile target at cross-build http://rubyci.s3.amazonaws.com/crossruby/crossruby-master-aarch64/log/20240325T041917Z.fail.html.gz --- tool/rbinstall.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tool/rbinstall.rb b/tool/rbinstall.rb index a2149317500813..788277bb7009a2 100755 --- a/tool/rbinstall.rb +++ b/tool/rbinstall.rb @@ -558,7 +558,7 @@ def features_from_makefile(makefile_path) makefile = File.read(makefile_path) name = makefile[/^TARGET[ \t]*=[ \t]*((?:.*\\\n)*.*)/, 1] - return [] if name.empty? + return [] if name.nil? || name.empty? feature = makefile[/^DLLIB[ \t]*=[ \t]*((?:.*\\\n)*.*)/, 1] feature = feature.sub("$(TARGET)", name) From 6c65f11c080f73cee91be386f0ca26dd0fbdb717 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Mar 2024 02:40:20 +0000 Subject: [PATCH 028/211] Bump github/codeql-action from 3.24.8 to 3.24.9 Bumps [github/codeql-action](https://github.com/github/codeql-action) from 3.24.8 to 3.24.9. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/05963f47d870e2cb19a537396c1f668a348c7d8f...1b1aada464948af03b950897e5eb522f92603cc2) --- updated-dependencies: - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 8 ++++---- .github/workflows/scorecards.yml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 0ea0e49076081f..430a2c950d21b6 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -80,15 +80,15 @@ jobs: run: sudo rm /usr/lib/ruby/vendor_ruby/rubygems/defaults/operating_system.rb - name: Initialize CodeQL - uses: github/codeql-action/init@05963f47d870e2cb19a537396c1f668a348c7d8f # v3.24.8 + uses: github/codeql-action/init@1b1aada464948af03b950897e5eb522f92603cc2 # v3.24.9 with: languages: ${{ matrix.language }} - name: Autobuild - uses: github/codeql-action/autobuild@05963f47d870e2cb19a537396c1f668a348c7d8f # v3.24.8 + uses: github/codeql-action/autobuild@1b1aada464948af03b950897e5eb522f92603cc2 # v3.24.9 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@05963f47d870e2cb19a537396c1f668a348c7d8f # v3.24.8 + uses: github/codeql-action/analyze@1b1aada464948af03b950897e5eb522f92603cc2 # v3.24.9 with: category: '/language:${{ matrix.language }}' upload: False @@ -118,7 +118,7 @@ jobs: continue-on-error: true - name: Upload SARIF - uses: github/codeql-action/upload-sarif@05963f47d870e2cb19a537396c1f668a348c7d8f # v3.24.8 + uses: github/codeql-action/upload-sarif@1b1aada464948af03b950897e5eb522f92603cc2 # v3.24.9 with: sarif_file: sarif-results/${{ matrix.language }}.sarif continue-on-error: true diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index 51bce58de44e81..2889d4d47c4747 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -67,6 +67,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard. - name: 'Upload to code-scanning' - uses: github/codeql-action/upload-sarif@05963f47d870e2cb19a537396c1f668a348c7d8f # v2.1.27 + uses: github/codeql-action/upload-sarif@1b1aada464948af03b950897e5eb522f92603cc2 # v2.1.27 with: sarif_file: results.sarif From 152192c96f30b66f3257faafeb7b34043506ae8f Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Mon, 25 Mar 2024 17:12:31 +0900 Subject: [PATCH 029/211] Use load_gemspec instead of Gem::Specification.load. We need to purge `git ls-files` from gemspec in default gems. --- tool/rbinstall.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tool/rbinstall.rb b/tool/rbinstall.rb index 788277bb7009a2..cdaeff666832b4 100755 --- a/tool/rbinstall.rb +++ b/tool/rbinstall.rb @@ -631,7 +631,7 @@ def ruby_features end def ext_features - loaded_gemspec = Gem::Specification.load("#{root}/#{gemspec}") + loaded_gemspec = load_gemspec("#{root}/#{gemspec}") extension = loaded_gemspec.extensions.first return [] unless extension From e70ccc91e56b44ec523cb66323d486631ea1409b Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 25 Mar 2024 09:31:59 +0100 Subject: [PATCH 030/211] Update minitest and power_assert bundled gems They were pointing to branches to be chilled string compatible. Both patches have been merged now. --- gems/bundled_gems | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/gems/bundled_gems b/gems/bundled_gems index 8a3555f9f826a6..00a49537de3f86 100644 --- a/gems/bundled_gems +++ b/gems/bundled_gems @@ -6,11 +6,8 @@ # - revision: revision in repository-url to test # if `revision` is not given, "v"+`version` or `version` will be used. -# Waiting for https://github.com/minitest/minitest/pull/991 -minitest 5.22.3 https://github.com/Shopify/minitest b5f5202575894796e00109a8f8a5041b778991ee - -# Waiting for https://github.com/ruby/power_assert/pull/48 -power_assert 2.0.3 https://github.com/ruby/power_assert 78dd2ab3ccd93796d83c0b35b978c39bfabb938c +minitest 5.22.3 https://github.com/minitest/minitest 287b35d60c8e19c11ba090efc6eeb225325a8520 +power_assert 2.0.3 https://github.com/ruby/power_assert 84e85124c5014a139af39161d484156cfe87a9ed rake 13.1.0 https://github.com/ruby/rake test-unit 3.6.2 https://github.com/test-unit/test-unit rexml 3.2.6 https://github.com/ruby/rexml From 219d7cc1aa14e3f0afaf84f9c127bd6125cb8481 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 25 Mar 2024 08:45:02 +0100 Subject: [PATCH 031/211] lib/bundled_gems.rb: dynamically ignore Kernel.require decorators Followup: https://github.com/ruby/ruby/pull/10347 This avoid directly referencing bootsnap and zeitwerk, and also handle other gems that may decorate `require`. --- lib/bundled_gems.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/bundled_gems.rb b/lib/bundled_gems.rb index e756af61eaaed4..a8ea9fe211c2ee 100644 --- a/lib/bundled_gems.rb +++ b/lib/bundled_gems.rb @@ -147,7 +147,7 @@ def self.build_message(gem) next end - unless cl.path.match?(/bootsnap|zeitwerk/) + if cl.base_label != "require" location = cl.path break end From 5f334b67d23f1397b71931646c96fb6b8aa96e08 Mon Sep 17 00:00:00 2001 From: Stan Lo Date: Mon, 25 Mar 2024 11:06:35 +0000 Subject: [PATCH 032/211] [ruby/reline] Remove useless tests (https://github.com/ruby/reline/pull/665) The pasting tests hadn't been working since as early as v0.2.0. Since what it tried to cover is already gone for such a long time, I think it's better to write new ones if needed then to keep them around. And since these tests are gone, the helper methods for just them are also gone. https://github.com/ruby/reline/commit/0eedf0e4a0 --- lib/reline/general_io.rb | 8 -------- test/reline/helper.rb | 8 -------- test/reline/test_key_actor_vi.rb | 16 ---------------- 3 files changed, 32 deletions(-) diff --git a/lib/reline/general_io.rb b/lib/reline/general_io.rb index 8fb0f25a3471ab..0ac1c6c56d40bc 100644 --- a/lib/reline/general_io.rb +++ b/lib/reline/general_io.rb @@ -102,14 +102,6 @@ def self.in_pasting? @@pasting end - def self.start_pasting - @@pasting = true - end - - def self.finish_pasting - @@pasting = false - end - def self.prep end diff --git a/test/reline/helper.rb b/test/reline/helper.rb index b7f849591b7d0d..f2f3421ded0bf4 100644 --- a/test/reline/helper.rb +++ b/test/reline/helper.rb @@ -78,14 +78,6 @@ def test_rubybin end end -def start_pasting - Reline::GeneralIO.start_pasting -end - -def finish_pasting - Reline::GeneralIO.finish_pasting -end - class Reline::TestCase < Test::Unit::TestCase private def convert_str(input, options = {}, normalized = nil) return nil if input.nil? diff --git a/test/reline/test_key_actor_vi.rb b/test/reline/test_key_actor_vi.rb index 0db24c6be7685d..91cbd49d74bfd5 100644 --- a/test/reline/test_key_actor_vi.rb +++ b/test/reline/test_key_actor_vi.rb @@ -764,22 +764,6 @@ def test_vi_next_char_with_operator assert_line_around_cursor('', 'bar') end - def test_pasting - start_pasting - input_keys('ab') - finish_pasting - input_keys('c') - assert_line_around_cursor('abc', '') - end - - def test_pasting_fullwidth - start_pasting - input_keys('あ') - finish_pasting - input_keys('い') - assert_line_around_cursor('あい', '') - end - def test_ed_delete_next_char_at_eol input_keys('"あ"') assert_line_around_cursor('"あ"', '') From f53209f02370b3590662de76fe63ec2818633139 Mon Sep 17 00:00:00 2001 From: tomoya ishida Date: Mon, 25 Mar 2024 20:48:08 +0900 Subject: [PATCH 033/211] [ruby/irb] Cache RDoc::RI::Driver.new (https://github.com/ruby/irb/pull/911) * Cache RDoc::RI::Driver.new to improve performance and to avoid flaky test * Insert sleep to fix flaky rendering test that renders document dialog https://github.com/ruby/irb/commit/da84e6cb56 --- lib/irb/input-method.rb | 28 +++++++++++++++--------- test/irb/test_input_method.rb | 13 ++++++----- test/irb/yamatanooroti/test_rendering.rb | 4 +++- 3 files changed, 28 insertions(+), 17 deletions(-) diff --git a/lib/irb/input-method.rb b/lib/irb/input-method.rb index c1998309283fc3..e5adb350e8ceef 100644 --- a/lib/irb/input-method.rb +++ b/lib/irb/input-method.rb @@ -308,6 +308,20 @@ def retrieve_doc_namespace(matched) @completor.doc_namespace(preposing, matched, postposing, bind: bind) end + def rdoc_ri_driver + return @rdoc_ri_driver if defined?(@rdoc_ri_driver) + + begin + require 'rdoc' + rescue LoadError + @rdoc_ri_driver = nil + else + options = {} + options[:extra_doc_dirs] = IRB.conf[:EXTRA_DOC_DIRS] unless IRB.conf[:EXTRA_DOC_DIRS].empty? + @rdoc_ri_driver = RDoc::RI::Driver.new(options) + end + end + def show_doc_dialog_proc input_method = self # self is changed in the lambda below. ->() { @@ -331,9 +345,7 @@ def show_doc_dialog_proc show_easter_egg = name&.match?(/\ARubyVM/) && !ENV['RUBY_YES_I_AM_NOT_A_NORMAL_USER'] - options = {} - options[:extra_doc_dirs] = IRB.conf[:EXTRA_DOC_DIRS] unless IRB.conf[:EXTRA_DOC_DIRS].empty? - driver = RDoc::RI::Driver.new(options) + driver = input_method.rdoc_ri_driver if key.match?(dialog.name) if show_easter_egg @@ -421,12 +433,9 @@ def show_doc_dialog_proc } end - def display_document(matched, driver: nil) - begin - require 'rdoc' - rescue LoadError - return - end + def display_document(matched) + driver = rdoc_ri_driver + return unless driver if matched =~ /\A(?:::)?RubyVM/ and not ENV['RUBY_YES_I_AM_NOT_A_NORMAL_USER'] IRB.__send__(:easter_egg) @@ -436,7 +445,6 @@ def display_document(matched, driver: nil) namespace = retrieve_doc_namespace(matched) return unless namespace - driver ||= RDoc::RI::Driver.new if namespace.is_a?(Array) out = RDoc::Markup::Document.new namespace.each do |m| diff --git a/test/irb/test_input_method.rb b/test/irb/test_input_method.rb index 7644d3176aabb7..ce317b4b32ed3d 100644 --- a/test/irb/test_input_method.rb +++ b/test/irb/test_input_method.rb @@ -88,17 +88,18 @@ def setup @driver = RDoc::RI::Driver.new(use_stdout: true) end - def display_document(target, bind) + def display_document(target, bind, driver = nil) input_method = IRB::RelineInputMethod.new(IRB::RegexpCompletor.new) + input_method.instance_variable_set(:@rdoc_ri_driver, driver) if driver input_method.instance_variable_set(:@completion_params, ['', target, '', bind]) - input_method.display_document(target, driver: @driver) + input_method.display_document(target) end def test_perfectly_matched_namespace_triggers_document_display omit unless has_rdoc_content? out, err = capture_output do - display_document("String", binding) + display_document("String", binding, @driver) end assert_empty(err) @@ -109,7 +110,7 @@ def test_perfectly_matched_namespace_triggers_document_display def test_perfectly_matched_multiple_namespaces_triggers_document_display result = nil out, err = capture_output do - result = display_document("{}.nil?", binding) + result = display_document("{}.nil?", binding, @driver) end assert_empty(err) @@ -131,7 +132,7 @@ def test_perfectly_matched_multiple_namespaces_triggers_document_display def test_not_matched_namespace_triggers_nothing result = nil out, err = capture_output do - result = display_document("Stri", binding) + result = display_document("Stri", binding, @driver) end assert_empty(err) @@ -156,7 +157,7 @@ def test_perfect_matching_stops_without_rdoc def test_perfect_matching_handles_nil_namespace out, err = capture_output do # symbol literal has `nil` doc namespace so it's a good test subject - assert_nil(display_document(":aiueo", binding)) + assert_nil(display_document(":aiueo", binding, @driver)) end assert_empty(err) diff --git a/test/irb/yamatanooroti/test_rendering.rb b/test/irb/yamatanooroti/test_rendering.rb index df4ec01a5cafa6..44e07a3a1232c1 100644 --- a/test/irb/yamatanooroti/test_rendering.rb +++ b/test/irb/yamatanooroti/test_rendering.rb @@ -256,9 +256,9 @@ def test_autocomplete_with_multiple_doc_namespaces start_terminal(3, 50, %W{ruby -I#{@pwd}/lib #{@pwd}/exe/irb}, startup_message: 'start IRB') write("{}.__id_") write("\C-i") + sleep 0.2 close screen = result.join("\n").sub(/\n*\z/, "\n") - # This assertion passes whether showdoc dialog completed or not. assert_match(/start\ IRB\nirb\(main\):001> {}\.__id__\n }\.__id__(?:Press )?/, screen) end @@ -278,6 +278,7 @@ def test_autocomplete_with_showdoc_in_gaps_on_narrow_screen_right start_terminal(4, 19, %W{ruby -I#{@pwd}/lib #{@pwd}/exe/irb}, startup_message: 'start IRB') write("IR") write("\C-i") + sleep 0.2 close # This is because on macOS we display different shortcut for displaying the full doc @@ -315,6 +316,7 @@ def test_autocomplete_with_showdoc_in_gaps_on_narrow_screen_left start_terminal(4, 12, %W{ruby -I#{@pwd}/lib #{@pwd}/exe/irb}, startup_message: 'start IRB') write("IR") write("\C-i") + sleep 0.2 close assert_screen(<<~EOC) start IRB From daf1d7bfefa89a0473af9b304a3798ba1e67166a Mon Sep 17 00:00:00 2001 From: Mari Imaizumi Date: Mon, 25 Mar 2024 21:13:27 +0900 Subject: [PATCH 034/211] [ruby/reline] Bump version to v0.5.0 (https://github.com/ruby/reline/pull/664) https://github.com/ruby/reline/commit/1d6569600c --- lib/reline/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/reline/version.rb b/lib/reline/version.rb index 8f21e6a9cfb2d4..06bb3acc8454e9 100644 --- a/lib/reline/version.rb +++ b/lib/reline/version.rb @@ -1,3 +1,3 @@ module Reline - VERSION = '0.5.0.pre.1' + VERSION = '0.5.0' end From f5a2f55aca5f229bb2a9dac62d96d85fc14ae718 Mon Sep 17 00:00:00 2001 From: Justin Collins Date: Sat, 23 Mar 2024 15:20:52 -0700 Subject: [PATCH 035/211] [ruby/prism] Use Sexp#line_max not Sexp#max_line for RubyParser translation https://github.com/ruby/prism/commit/a37169621a --- lib/prism/translation/ruby_parser.rb | 10 +++++----- test/prism/ruby_parser_test.rb | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/prism/translation/ruby_parser.rb b/lib/prism/translation/ruby_parser.rb index 108e6c6928c5cd..5c59fe31810562 100644 --- a/lib/prism/translation/ruby_parser.rb +++ b/lib/prism/translation/ruby_parser.rb @@ -192,19 +192,19 @@ def visit_block_parameters_node(node) if node.opening == "(" result.line = node.opening_loc.start_line - result.max_line = node.closing_loc.end_line + result.line_max = node.closing_loc.end_line shadow_loc = false end if node.locals.any? shadow = s(node, :shadow).concat(visit_all(node.locals)) shadow.line = node.locals.first.location.start_line - shadow.max_line = node.locals.last.location.end_line + shadow.line_max = node.locals.last.location.end_line result << shadow if shadow_loc result.line = shadow.line - result.max_line = shadow.max_line + result.line_max = shadow.line_max end end @@ -1412,7 +1412,7 @@ def visit_x_string_node(node) if node.heredoc? result.line = node.content_loc.start_line - result.max_line = node.content_loc.end_line + result.line_max = node.content_loc.end_line end result @@ -1439,7 +1439,7 @@ def s(node, *arguments) result = Sexp.new(*arguments) result.file = file result.line = node.location.start_line - result.max_line = node.location.end_line + result.line_max = node.location.end_line result end diff --git a/test/prism/ruby_parser_test.rb b/test/prism/ruby_parser_test.rb index 1d22f0e7b8729f..952e493af9159c 100644 --- a/test/prism/ruby_parser_test.rb +++ b/test/prism/ruby_parser_test.rb @@ -18,7 +18,7 @@ Sexp.prepend( Module.new do def ==(other) - super && line == other.line && max_line == other.max_line && file == other.file + super && line == other.line && line_max == other.line_max && file == other.file end end ) From 9b921f662285e785ddbd22d0bcd540fa35151b08 Mon Sep 17 00:00:00 2001 From: git Date: Mon, 25 Mar 2024 12:14:37 +0000 Subject: [PATCH 036/211] Update default gems list at daf1d7bfefa89a0473af9b304a3798 [ci skip] --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 83e5a4073fc9a7..adc0ac5daaa0ef 100644 --- a/NEWS.md +++ b/NEWS.md @@ -46,7 +46,7 @@ The following default gems are updated. * net-http 0.4.1 * prism 0.24.0 * rdoc 6.6.3.1 -* reline 0.5.0.pre.1 +* reline 0.5.0 * resolv 0.4.0 * stringio 3.1.1 * strscan 3.1.1 From 56a2fad2a4578987a371f7a5563812b52ed8e9c6 Mon Sep 17 00:00:00 2001 From: Koichi ITO Date: Thu, 21 Mar 2024 01:46:53 +0900 Subject: [PATCH 037/211] [ruby/prism] Fix incorrect paring when using invalid regexp options Fixes https://github.com/ruby/prism/pull/2617. There was an issue with the lexer as follows. The following are valid regexp options: ```console $ bundle exec ruby -Ilib -rprism -ve 'p Prism.lex("/x/io").value.map {|token| token[0].type }' ruby 3.3.0 (2023-12-25 revision https://github.com/ruby/prism/commit/5124f9ac75) [x86_64-darwin22] [:REGEXP_BEGIN, :STRING_CONTENT, :REGEXP_END, :EOF] ``` The following are invalid regexp options. Unnecessary the `IDENTIFIER` token is appearing: ```console $ bundle exec ruby -Ilib -rprism -ve 'p Prism.lex("/x/az").value.map {|token| token[0].type }' ruby 3.3.0 (2023-12-25 revision https://github.com/ruby/prism/commit/5124f9ac75) [x86_64-darwin22] [:REGEXP_BEGIN, :STRING_CONTENT, :REGEXP_END, :IDENTIFIER, :EOF] ``` As a behavior of Ruby, when given `A` to `Z` and `a` to `z`, they act as invalid regexp options. e.g., ```console $ ruby -e '/regexp/az' -e:1: unknown regexp options - az /regexp/az -e: compile error (SyntaxError) ``` Thus, it should probably not be construed as `IDENTIFIER` token. Therefore, `pm_byte_table` has been adapted to accept those invalid regexp option values. Whether it is a valid regexp option or not is checked by `pm_regular_expression_flags_create`. For invalid regexp options, `PM_ERR_REGEXP_UNKNOWN_OPTIONS` is added to diagnostics. https://github.com/ruby/prism/commit/d2a6096fcf --- lib/prism/translation/parser.rb | 2 ++ prism/config.yml | 1 + prism/prism.c | 21 +++++++++++++++------ prism/templates/src/diagnostic.c.erb | 1 + prism/util/pm_char.c | 8 ++++---- test/prism/errors_test.rb | 14 ++++++++++++++ test/prism/location_test.rb | 2 ++ 7 files changed, 39 insertions(+), 10 deletions(-) diff --git a/lib/prism/translation/parser.rb b/lib/prism/translation/parser.rb index 8df71646882d75..0d11b8f5668cca 100644 --- a/lib/prism/translation/parser.rb +++ b/lib/prism/translation/parser.rb @@ -173,6 +173,8 @@ def error_diagnostic(error, offset_cache) Diagnostic.new(:error, :duplicate_argument, {}, diagnostic_location, []) when :parameter_numbered_reserved Diagnostic.new(:error, :reserved_for_numparam, { name: location.slice }, diagnostic_location, []) + when :regexp_unknown_options + Diagnostic.new(:error, :regexp_options, { options: location.slice[1..] }, diagnostic_location, []) when :singleton_for_literals Diagnostic.new(:error, :singleton_literal, {}, diagnostic_location, []) when :string_literal_eof diff --git a/prism/config.yml b/prism/config.yml index 269bfa73ec5efa..d9e39460d12f2c 100644 --- a/prism/config.yml +++ b/prism/config.yml @@ -199,6 +199,7 @@ errors: - REGEXP_INVALID_UNICODE_RANGE - REGEXP_NON_ESCAPED_MBC - REGEXP_TERM + - REGEXP_UNKNOWN_OPTIONS - REGEXP_UTF8_CHAR_NON_UTF8_REGEXP - RESCUE_EXPRESSION - RESCUE_MODIFIER_VALUE diff --git a/prism/prism.c b/prism/prism.c index 8a6ca0eccc4e6a..b410b0a510220a 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -1214,10 +1214,12 @@ pm_node_flag_set_repeated_parameter(pm_node_t *node) { * Parse out the options for a regular expression. */ static inline pm_node_flags_t -pm_regular_expression_flags_create(const pm_token_t *closing) { +pm_regular_expression_flags_create(pm_parser_t *parser, const pm_token_t *closing) { pm_node_flags_t flags = 0; if (closing->type == PM_TOKEN_REGEXP_END) { + pm_buffer_t unknown_flags = { 0 }; + for (const uint8_t *flag = closing->start + 1; flag < closing->end; flag++) { switch (*flag) { case 'i': flags |= PM_REGULAR_EXPRESSION_FLAGS_IGNORE_CASE; break; @@ -1230,9 +1232,16 @@ pm_regular_expression_flags_create(const pm_token_t *closing) { case 's': flags = (pm_node_flags_t) (((pm_node_flags_t) (flags & PM_REGULAR_EXPRESSION_ENCODING_MASK)) | PM_REGULAR_EXPRESSION_FLAGS_WINDOWS_31J); break; case 'u': flags = (pm_node_flags_t) (((pm_node_flags_t) (flags & PM_REGULAR_EXPRESSION_ENCODING_MASK)) | PM_REGULAR_EXPRESSION_FLAGS_UTF_8); break; - default: assert(false && "unreachable"); + default: pm_buffer_append_byte(&unknown_flags, *flag); } } + + size_t unknown_flags_length = pm_buffer_length(&unknown_flags); + if (unknown_flags_length != 0) { + char *word = unknown_flags_length >= 2 ? "options" : "option"; + PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->previous, PM_ERR_REGEXP_UNKNOWN_OPTIONS, word, unknown_flags_length, pm_buffer_value(&unknown_flags)); + } + pm_buffer_free(&unknown_flags); } return flags; @@ -4297,10 +4306,10 @@ pm_interpolated_regular_expression_node_append(pm_interpolated_regular_expressio } static inline void -pm_interpolated_regular_expression_node_closing_set(pm_interpolated_regular_expression_node_t *node, const pm_token_t *closing) { +pm_interpolated_regular_expression_node_closing_set(pm_parser_t *parser, pm_interpolated_regular_expression_node_t *node, const pm_token_t *closing) { node->closing_loc = PM_LOCATION_TOKEN_VALUE(closing); node->base.location.end = closing->end; - pm_node_flag_set((pm_node_t *)node, pm_regular_expression_flags_create(closing)); + pm_node_flag_set((pm_node_t *)node, pm_regular_expression_flags_create(parser, closing)); } /** @@ -5528,7 +5537,7 @@ pm_regular_expression_node_create_unescaped(pm_parser_t *parser, const pm_token_ *node = (pm_regular_expression_node_t) { { .type = PM_REGULAR_EXPRESSION_NODE, - .flags = pm_regular_expression_flags_create(closing) | PM_NODE_FLAG_STATIC_LITERAL, + .flags = pm_regular_expression_flags_create(parser, closing) | PM_NODE_FLAG_STATIC_LITERAL, .location = { .start = MIN(opening->start, closing->start), .end = MAX(opening->end, closing->end) @@ -17490,7 +17499,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b expect1(parser, PM_TOKEN_REGEXP_END, PM_ERR_REGEXP_TERM); } - pm_interpolated_regular_expression_node_closing_set(interpolated, &closing); + pm_interpolated_regular_expression_node_closing_set(parser, interpolated, &closing); return (pm_node_t *) interpolated; } case PM_TOKEN_BACKTICK: diff --git a/prism/templates/src/diagnostic.c.erb b/prism/templates/src/diagnostic.c.erb index 2a3ac1993009e8..818b12d98be8c1 100644 --- a/prism/templates/src/diagnostic.c.erb +++ b/prism/templates/src/diagnostic.c.erb @@ -277,6 +277,7 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_ERR_REGEXP_INCOMPAT_CHAR_ENCODING] = { "incompatible character encoding: /%.*s/", PM_ERROR_LEVEL_FATAL }, [PM_ERR_REGEXP_NON_ESCAPED_MBC] = { "/.../n has a non escaped non ASCII character in non ASCII-8BIT script: /%.*s/", PM_ERROR_LEVEL_FATAL }, [PM_ERR_REGEXP_INVALID_UNICODE_RANGE] = { "invalid Unicode range: /%.*s/", PM_ERROR_LEVEL_FATAL }, + [PM_ERR_REGEXP_UNKNOWN_OPTIONS] = { "unknown regexp %s: %.*s", PM_ERROR_LEVEL_FATAL }, [PM_ERR_REGEXP_TERM] = { "expected a closing delimiter for the regular expression", PM_ERROR_LEVEL_FATAL }, [PM_ERR_REGEXP_UTF8_CHAR_NON_UTF8_REGEXP] = { "UTF-8 character in non UTF-8 regexp: /%s/", PM_ERROR_LEVEL_FATAL }, [PM_ERR_RESCUE_EXPRESSION] = { "expected a rescued expression", PM_ERROR_LEVEL_FATAL }, diff --git a/prism/util/pm_char.c b/prism/util/pm_char.c index 13eddbba481c9d..dce19abd1b102f 100644 --- a/prism/util/pm_char.c +++ b/prism/util/pm_char.c @@ -19,10 +19,10 @@ static const uint8_t pm_byte_table[256] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1x 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 2x 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 3x - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 4x - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 5x - 0, 0, 0, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 4, 4, 4, // 6x - 0, 0, 0, 4, 0, 4, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, // 7x + 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, // 4x + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, // 5x + 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, // 6x + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, // 7x 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 8x 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 9x 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // Ax diff --git a/test/prism/errors_test.rb b/test/prism/errors_test.rb index ccf7485c7bdb26..9221d52ef35a4c 100644 --- a/test/prism/errors_test.rb +++ b/test/prism/errors_test.rb @@ -2067,6 +2067,20 @@ def test_it_with_ordinary_parameter assert_errors expression(source), source, errors, compare_ripper: false end + def test_regular_expression_with_unknown_regexp_options + source = "/foo/AZaz" + errors = [["unknown regexp options: AZaz", 4..9]] + + assert_errors expression(source), source, errors + end + + def test_interpolated_regular_expression_with_unknown_regexp_options + source = "/\#{foo}/AZaz" + errors = [["unknown regexp options: AZaz", 7..12]] + + assert_errors expression(source), source, errors + end + def test_singleton_method_for_literals source = <<~'RUBY' def (1).g; end diff --git a/test/prism/location_test.rb b/test/prism/location_test.rb index c7ce248b569f00..b7b9a754cae781 100644 --- a/test/prism/location_test.rb +++ b/test/prism/location_test.rb @@ -527,6 +527,7 @@ def test_InterpolatedMatchLastLineNode def test_InterpolatedRegularExpressionNode assert_location(InterpolatedRegularExpressionNode, "/\#{foo}/") + assert_location(InterpolatedRegularExpressionNode, "/\#{foo}/io") end def test_InterpolatedStringNode @@ -730,6 +731,7 @@ def test_RedoNode def test_RegularExpressionNode assert_location(RegularExpressionNode, "/foo/") + assert_location(RegularExpressionNode, "/foo/io") end def test_RequiredKeywordParameterNode From d05135727f51236fd6ec121a8265167a857681e8 Mon Sep 17 00:00:00 2001 From: Franck Trouillez Date: Mon, 25 Mar 2024 10:13:25 +0100 Subject: [PATCH 038/211] [ruby/prism] Fix comment typos in prism.c This fixes some comment typos in English in the prism.c file. It fixes some typos and follows the current conventions: - Sentences in comments end with `.` - Use infinitive instead of 3rd person present simple to describe functions https://github.com/ruby/prism/commit/01324e89db --- prism/prism.c | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/prism/prism.c b/prism/prism.c index b410b0a510220a..5b0434530597e7 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -973,9 +973,9 @@ pm_conditional_predicate(pm_parser_t *parser, pm_node_t *node, pm_conditional_pr /** * In a lot of places in the tree you can have tokens that are not provided but - * that do not cause an error. For example, in a method call without - * parentheses. In these cases we set the token to the "not provided" type. For - * example: + * that do not cause an error. For example, this happens in a method call + * without parentheses. In these cases we set the token to the "not provided" type. + * For example: * * pm_token_t token = not_provided(parser); */ @@ -5482,7 +5482,7 @@ pm_range_node_create(pm_parser_t *parser, pm_node_t *left, const pm_token_t *ope pm_range_node_t *node = PM_ALLOC_NODE(parser, pm_range_node_t); pm_node_flags_t flags = 0; - // Indicate that this node an exclusive range if the operator is `...`. + // Indicate that this node is an exclusive range if the operator is `...`. if (operator->type == PM_TOKEN_DOT_DOT_DOT || operator->type == PM_TOKEN_UDOT_DOT_DOT) { flags |= PM_RANGE_FLAGS_EXCLUDE_END; } @@ -7113,7 +7113,7 @@ peek(pm_parser_t *parser) { /** * If the character to be read matches the given value, then returns true and - * advanced the current pointer. + * advances the current pointer. */ static inline bool match(pm_parser_t *parser, uint8_t value) { @@ -8161,7 +8161,7 @@ lex_interpolation(pm_parser_t *parser, const uint8_t *pound) { return PM_TOKEN_STRING_CONTENT; } - // Now we'll check against the character the follows the #. If it constitutes + // Now we'll check against the character that follows the #. If it constitutes // valid interplation, we'll handle that, otherwise we'll return // PM_TOKEN_NOT_PROVIDED. switch (pound[1]) { @@ -8193,7 +8193,7 @@ lex_interpolation(pm_parser_t *parser, const uint8_t *pound) { return PM_TOKEN_EMBVAR; } - // If we didn't get an valid interpolation, then this is just regular + // If we didn't get a valid interpolation, then this is just regular // string content. This is like if we get "#@-". In this case the caller // should keep lexing. parser->current.end = pound + 1; @@ -9566,7 +9566,7 @@ parser_lex(pm_parser_t *parser) { // we need to return the call operator. if (next_content[0] == '.') { // To match ripper, we need to emit an ignored newline even though - // its a real newline in the case that we have a beginless range + // it's a real newline in the case that we have a beginless range // on a subsequent line. if (peek_at(parser, next_content + 1) == '.') { if (!lexed_comment) parser_lex_ignored_newline(parser); @@ -10512,7 +10512,7 @@ parser_lex(pm_parser_t *parser) { ) { // Since we know we're about to add an __END__ comment, we know we - // need at add all of the newlines to get the correct column + // need to add all of the newlines to get the correct column // information for it. const uint8_t *cursor = parser->current.end; while ((cursor = next_newline(cursor, parser->end - cursor)) != NULL) { @@ -11363,7 +11363,7 @@ parser_lex(pm_parser_t *parser) { case '\\': { // If we hit an escape, then we need to skip past // however many characters the escape takes up. However - // it's important that if \n or \r\n are escaped that we + // it's important that if \n or \r\n are escaped, we // stop looping before the newline and not after the // newline so that we can still potentially find the // terminator of the heredoc. @@ -11844,7 +11844,6 @@ parse_value_expression(pm_parser_t *parser, pm_binding_power_t binding_power, bo * CRuby parsers that are generated would resolve this by using a lookahead and * potentially backtracking. We attempt to do this by just looking at the next * token and making a decision based on that. I am not sure if this is going to - * * work in all cases, it may need to be refactored later. But it appears to work * for now. */ @@ -11877,7 +11876,7 @@ token_begins_expression_p(pm_token_type_t type) { case PM_TOKEN_SEMICOLON: // The reason we need this short-circuit is because we're using the // binding powers table to tell us if the subsequent token could - // potentially be the start of an expression . If there _is_ a binding + // potentially be the start of an expression. If there _is_ a binding // power for one of these tokens, then we should remove it from this list // and let it be handled by the default case below. assert(pm_binding_powers[type].left == PM_BINDING_POWER_UNSET); @@ -12442,7 +12441,7 @@ pm_when_clause_static_literals_add(pm_parser_t *parser, pm_static_literals_t *li } /** - * Parse all of the elements of a hash. returns true if a double splat was found. + * Parse all of the elements of a hash. Return true if a double splat was found. */ static bool parse_assocs(pm_parser_t *parser, pm_static_literals_t *literals, pm_node_t *node) { @@ -12880,7 +12879,7 @@ update_parameter_state(pm_parser_t *parser, pm_token_t *token, pm_parameters_ord if (state == PM_PARAMETERS_NO_CHANGE) return; // If we see another ordered argument after a optional argument - // we only continue parsing ordered arguments until we stop seeing ordered arguments + // we only continue parsing ordered arguments until we stop seeing ordered arguments. if (*current == PM_PARAMETERS_ORDER_OPTIONAL && state == PM_PARAMETERS_ORDER_NAMED) { *current = PM_PARAMETERS_ORDER_AFTER_OPTIONAL; return; @@ -13331,7 +13330,7 @@ parse_rescues(pm_parser_t *parser, pm_begin_node_t *parent_node, bool def_p) { // The end node locations on rescue nodes will not be set correctly // since we won't know the end until we've found all consequent - // clauses. This sets the end location on all rescues once we know it + // clauses. This sets the end location on all rescues once we know it. if (current) { const uint8_t *end_to_set = current->base.location.end; current = parent_node->rescue_clause; @@ -15194,7 +15193,7 @@ parse_negative_numeric(pm_node_t *node) { } /** - * Returns a string content token at a particular location that is empty. + * Return a string content token at a particular location that is empty. */ static pm_token_t parse_strings_empty_content(const uint8_t *location) { @@ -17858,7 +17857,7 @@ parse_assignment_values(pm_parser_t *parser, pm_binding_power_t previous_binding } /** - * Ensures a call node that is about to become a call operator node does not + * Ensure a call node that is about to become a call operator node does not * have arguments or a block attached. If it does, then we'll need to add an * error message and destroy the arguments/block. Ideally we would keep the node * around so that consumers would still have access to it, but we don't have a @@ -18799,8 +18798,8 @@ parse_expression(pm_parser_t *parser, pm_binding_power_t binding_power, bool acc case PM_RANGE_NODE: // Range operators are non-associative, so that it does not // associate with other range operators (i.e. `..1..` should be - // rejected.) For this reason, we check such a case for unary ranges - // here, and if so, it returns the node immediately, + // rejected). For this reason, we check such a case for unary ranges + // here, and if so, it returns the node immediately. if ((((pm_range_node_t *) node)->left == NULL) && pm_binding_powers[parser->current.type].left >= PM_BINDING_POWER_RANGE) { return node; } From 54428c2bf10c15663de084bbac20783395f2d8fd Mon Sep 17 00:00:00 2001 From: Franck Trouillez Date: Mon, 25 Mar 2024 10:13:49 +0100 Subject: [PATCH 039/211] [ruby/prism] Add missing symbol in comment for binding powers for `||=` This adds in the descriptive comment the `||=` operator corresponding to `PM_TOKEN_PIPE_PIPE_EQUAL` for pm_binding_powers[PM_TOKEN_MAXIMUM] in prism.c https://github.com/ruby/prism/commit/315ca16e23 --- prism/prism.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prism/prism.c b/prism/prism.c index 5b0434530597e7..64f43dcdee4cc1 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -11575,7 +11575,7 @@ pm_binding_powers_t pm_binding_powers[PM_TOKEN_MAXIMUM] = { [PM_TOKEN_EQUAL_GREATER] = NON_ASSOCIATIVE(PM_BINDING_POWER_MATCH), [PM_TOKEN_KEYWORD_IN] = NON_ASSOCIATIVE(PM_BINDING_POWER_MATCH), - // &&= &= ^= = >>= <<= -= %= |= += /= *= **= + // &&= &= ^= = >>= <<= -= %= |= ||= += /= *= **= [PM_TOKEN_AMPERSAND_AMPERSAND_EQUAL] = BINDING_POWER_ASSIGNMENT, [PM_TOKEN_AMPERSAND_EQUAL] = BINDING_POWER_ASSIGNMENT, [PM_TOKEN_CARET_EQUAL] = BINDING_POWER_ASSIGNMENT, From 4235bc295b1a0952fd610afb4d333dd03b162bea Mon Sep 17 00:00:00 2001 From: Xavier Noria Date: Mon, 25 Mar 2024 10:09:56 +0100 Subject: [PATCH 040/211] Add an example to base_label API --- vm_backtrace.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/vm_backtrace.c b/vm_backtrace.c index 120f9f9c26ee95..c60618e5c3ac7c 100644 --- a/vm_backtrace.c +++ b/vm_backtrace.c @@ -315,9 +315,10 @@ location_base_label(rb_backtrace_location_t *loc) } /* - * Returns the base label of this frame. + * Returns the label of this frame without decoration. * - * Usually same as #label, without decoration. + * For example, if the label is `foo`, this method returns `foo`, same, but if + * the label is +rescue in foo+, this method returns just +foo+. */ static VALUE location_base_label_m(VALUE self) From dadaa1142fc89d9752e9454f99b0b4d32068ef41 Mon Sep 17 00:00:00 2001 From: Xavier Noria Date: Mon, 25 Mar 2024 10:21:07 +0100 Subject: [PATCH 041/211] Update vm_backtrace.c Co-authored-by: Jean Boussier --- vm_backtrace.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm_backtrace.c b/vm_backtrace.c index c60618e5c3ac7c..9f004df7920a83 100644 --- a/vm_backtrace.c +++ b/vm_backtrace.c @@ -317,7 +317,7 @@ location_base_label(rb_backtrace_location_t *loc) /* * Returns the label of this frame without decoration. * - * For example, if the label is `foo`, this method returns `foo`, same, but if + * For example, if the label is `foo`, this method returns `foo` as well, but if * the label is +rescue in foo+, this method returns just +foo+. */ static VALUE From 3b4dacf2ede0dafbcf942ac696439237f8b31dc6 Mon Sep 17 00:00:00 2001 From: Xavier Noria Date: Mon, 25 Mar 2024 11:09:14 +0100 Subject: [PATCH 042/211] Let the docs of base_label and label be similar --- vm_backtrace.c | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/vm_backtrace.c b/vm_backtrace.c index 9f004df7920a83..11b26adbf49781 100644 --- a/vm_backtrace.c +++ b/vm_backtrace.c @@ -287,15 +287,14 @@ location_label(rb_backtrace_location_t *loc) * 1.times do * puts caller_locations(0).first.label * end - * * end * end * * The result of calling +foo+ is this: * - * label: foo - * label: block in foo - * label: block (2 levels) in foo + * foo + * block in foo + * block (2 levels) in foo * */ static VALUE @@ -315,10 +314,28 @@ location_base_label(rb_backtrace_location_t *loc) } /* - * Returns the label of this frame without decoration. + * Returns the base label of this frame, which is usually equal to the label, + * without decoration. + * + * Consider the following example: + * + * def foo + * puts caller_locations(0).first.base_label + * + * 1.times do + * puts caller_locations(0).first.base_label + * + * 1.times do + * puts caller_locations(0).first.base_label + * end + * end + * end + * + * The result of calling +foo+ is this: * - * For example, if the label is `foo`, this method returns `foo` as well, but if - * the label is +rescue in foo+, this method returns just +foo+. + * foo + * foo + * foo */ static VALUE location_base_label_m(VALUE self) From 65264b0dfb73a4162b638fa2c9cc0e99c66360e2 Mon Sep 17 00:00:00 2001 From: Koichi ITO Date: Mon, 25 Mar 2024 22:23:38 +0900 Subject: [PATCH 043/211] [ruby/prism] Fix build error for C99 and C23 CI matrix This PR fixes the following build error for C99 and C23 Ruby's CI matrix: ```console ../src/prism/prism.c:1241:19: error: initializing 'char *' with an expression of type 'const char *' discards qualifiers [-Werror,-Wincompatible-pointer-types-discards-qualifiers] 1241 | char *word = unknown_flags_length >= 2 ? "options" : "option"; | ^ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ assembling ../src/coroutine/amd64/Context.S ``` - c99 ... https://github.com/ruby/ruby/actions/runs/8419905079/job/23053543994#step:10:249 - c23 ... https://github.com/ruby/ruby/actions/runs/8419905079/job/23053544274#step:10:257 This is an incorrect code introduced in https://github.com/ruby/prism/pull/2618. https://github.com/ruby/prism/commit/4d9d73fcb9 --- prism/prism.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prism/prism.c b/prism/prism.c index 64f43dcdee4cc1..77cbcea2fe7a2d 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -1238,7 +1238,7 @@ pm_regular_expression_flags_create(pm_parser_t *parser, const pm_token_t *closin size_t unknown_flags_length = pm_buffer_length(&unknown_flags); if (unknown_flags_length != 0) { - char *word = unknown_flags_length >= 2 ? "options" : "option"; + const char *word = unknown_flags_length >= 2 ? "options" : "option"; PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->previous, PM_ERR_REGEXP_UNKNOWN_OPTIONS, word, unknown_flags_length, pm_buffer_value(&unknown_flags)); } pm_buffer_free(&unknown_flags); From 552647175e8319aa7cc117d418f35ed761fdb822 Mon Sep 17 00:00:00 2001 From: Cody Cutrer Date: Mon, 9 Oct 2023 11:24:32 -0600 Subject: [PATCH 044/211] [rubygems/rubygems] Improve validation of `bundle plugin install` options Ensure only one source type is specified, and ensure options that are only relevant to git sources are only specified with git. https://github.com/rubygems/rubygems/commit/58b043215e --- lib/bundler/plugin/installer.rb | 13 +++++++++++++ spec/bundler/plugins/install_spec.rb | 14 ++++++++------ 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/lib/bundler/plugin/installer.rb b/lib/bundler/plugin/installer.rb index 7ae56440fbbf2f..7267f58f5d5ea9 100644 --- a/lib/bundler/plugin/installer.rb +++ b/lib/bundler/plugin/installer.rb @@ -43,11 +43,24 @@ def check_sources_consistency!(options) if options.key?(:git) && options.key?(:local_git) raise InvalidOption, "Remote and local plugin git sources can't be both specified" end + # back-compat; local_git is an alias for git if options.key?(:local_git) Bundler::SharedHelpers.major_deprecation(2, "--local_git is deprecated, use --git") options[:git] = options.delete(:local_git) end + + if (options.keys & [:source, :git]).length > 1 + raise InvalidOption, "Only one of --source, or --git may be specified" + end + + if (options.key?(:branch) || options.key?(:ref)) && !options.key?(:git) + raise InvalidOption, "--#{options.key?(:branch) ? "branch" : "ref"} can only be used with git sources" + end + + if options.key?(:branch) && options.key?(:ref) + raise InvalidOption, "--branch and --ref can't be both specified" + end end def install_git(names, version, options) diff --git a/spec/bundler/plugins/install_spec.rb b/spec/bundler/plugins/install_spec.rb index d4776bbed7d15a..86eb4e584c6a8b 100644 --- a/spec/bundler/plugins/install_spec.rb +++ b/spec/bundler/plugins/install_spec.rb @@ -92,16 +92,18 @@ expect(out).to include("Using foo 1.1") end - it "installs when --branch specified" do - bundle "plugin install foo --branch main --source #{file_uri_for(gem_repo2)}" + it "raises an error when when --branch specified" do + bundle "plugin install foo --branch main --source #{file_uri_for(gem_repo2)}", raise_on_error: false - expect(out).to include("Installed plugin foo") + expect(out).not_to include("Installed plugin foo") + + expect(err).to include("--branch can only be used with git sources") end - it "installs when --ref specified" do - bundle "plugin install foo --ref v1.2.3 --source #{file_uri_for(gem_repo2)}" + it "raises an error when --ref specified" do + bundle "plugin install foo --ref v1.2.3 --source #{file_uri_for(gem_repo2)}", raise_on_error: false - expect(out).to include("Installed plugin foo") + expect(err).to include("--ref can only be used with git sources") end it "raises error when both --branch and --ref options are specified" do From a31ca3500d995b6706f94ff72166d699c5faeb27 Mon Sep 17 00:00:00 2001 From: Gannon McGibbon Date: Fri, 22 Mar 2024 15:38:28 -0500 Subject: [PATCH 045/211] Mark iseq structs with rb_gc_mark_movable Using rb_gc_mark_movable and a reference update function, we can make instruction sequences movable in memory, and avoid pinning compiled iseqs. ``` require "objspace" iseqs = [] GC.disable 50_000.times do iseqs << RubyVM::InstructionSequence.compile("") end GC.enable GC.compact p ObjectSpace.dump_all(output: :string).lines.grep(/"pinned":true/).count ``` Co-authored-by: Peter Zhu --- iseq.c | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/iseq.c b/iseq.c index 057e047ef9fded..d00cc9e9cc495b 100644 --- a/iseq.c +++ b/iseq.c @@ -1386,18 +1386,30 @@ rb_iseq_remove_coverage_all(void) static void iseqw_mark(void *ptr) { - rb_gc_mark((VALUE)ptr); + rb_gc_mark_movable(*(VALUE *)ptr); } static size_t iseqw_memsize(const void *ptr) { - return rb_iseq_memsize((const rb_iseq_t *)ptr); + return rb_iseq_memsize(*(const rb_iseq_t **)ptr); +} + +static void +iseqw_ref_update(void *ptr) +{ + VALUE *vptr = ptr; + *vptr = rb_gc_location(*vptr); } static const rb_data_type_t iseqw_data_type = { "T_IMEMO/iseq", - {iseqw_mark, NULL, iseqw_memsize,}, + { + iseqw_mark, + RUBY_TYPED_DEFAULT_FREE, + iseqw_memsize, + iseqw_ref_update, + }, 0, 0, RUBY_TYPED_FREE_IMMEDIATELY|RUBY_TYPED_WB_PROTECTED }; @@ -1408,11 +1420,9 @@ iseqw_new(const rb_iseq_t *iseq) return iseq->wrapper; } else { - union { const rb_iseq_t *in; void *out; } deconst; - VALUE obj; - deconst.in = iseq; - obj = TypedData_Wrap_Struct(rb_cISeq, &iseqw_data_type, deconst.out); - RB_OBJ_WRITTEN(obj, Qundef, iseq); + rb_iseq_t **ptr; + VALUE obj = TypedData_Make_Struct(rb_cISeq, rb_iseq_t *, &iseqw_data_type, ptr); + RB_OBJ_WRITE(obj, ptr, iseq); /* cache a wrapper object */ RB_OBJ_WRITE((VALUE)iseq, &iseq->wrapper, obj); @@ -1736,7 +1746,9 @@ iseqw_s_compile_option_get(VALUE self) static const rb_iseq_t * iseqw_check(VALUE iseqw) { - rb_iseq_t *iseq = DATA_PTR(iseqw); + rb_iseq_t **iseq_ptr; + TypedData_Get_Struct(iseqw, rb_iseq_t *, &iseqw_data_type, iseq_ptr); + rb_iseq_t *iseq = *iseq_ptr; if (!ISEQ_BODY(iseq)) { rb_ibf_load_iseq_complete(iseq); From 14ab698967cdaedc0a922a2bdf30dfc69bdba7eb Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 25 Mar 2024 08:32:58 -0400 Subject: [PATCH 046/211] [ruby/prism] Handle CLRF inside heredoc contents https://github.com/ruby/prism/commit/1fbac72485 --- prism/prism.c | 19 +++++++++++++++-- test/prism/ruby_parser_test.rb | 21 ++++--------------- test/prism/snapshots/dos_endings.txt | 4 ++-- test/prism/snapshots/heredoc_with_comment.txt | 2 +- .../heredoc__backslash_dos_format.txt | 2 +- ...c_with_carriage_return_escapes_windows.txt | 2 +- ...redoc_with_extra_carriage_horrible_mix.txt | 2 +- .../heredoc_with_extra_carriage_returns.txt | 2 +- ...oc_with_extra_carriage_returns_windows.txt | 2 +- ...on_and_carriage_return_escapes_windows.txt | 2 +- .../heredoc_with_only_carriage_returns.txt | 2 +- ...doc_with_only_carriage_returns_windows.txt | 2 +- 12 files changed, 32 insertions(+), 30 deletions(-) diff --git a/prism/prism.c b/prism/prism.c index 77cbcea2fe7a2d..a140dc734f0b5a 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -11267,11 +11267,11 @@ parser_lex(pm_parser_t *parser) { // Otherwise we'll be parsing string content. These are the places // where we need to split up the content of the heredoc. We'll use // strpbrk to find the first of these characters. - uint8_t breakpoints[] = "\n\\#"; + uint8_t breakpoints[] = "\r\n\\#"; pm_heredoc_quote_t quote = lex_mode->as.heredoc.quote; if (quote == PM_HEREDOC_QUOTE_SINGLE) { - breakpoints[2] = '\0'; + breakpoints[3] = '\0'; } const uint8_t *breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, true); @@ -11285,6 +11285,21 @@ parser_lex(pm_parser_t *parser) { parser->current.end = breakpoint + 1; breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, true); break; + case '\r': + parser->current.end = breakpoint + 1; + + if (peek_at(parser, breakpoint + 1) != '\n') { + breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, true); + break; + } + + // If we hit a \r\n sequence, then we want to replace it + // with a single \n character in the final string. + pm_token_buffer_escape(parser, &token_buffer); + breakpoint++; + token_buffer.cursor = breakpoint; + + /* fallthrough */ case '\n': { if (parser->heredoc_end != NULL && (parser->heredoc_end > breakpoint)) { parser_flush_heredoc_end(parser); diff --git a/test/prism/ruby_parser_test.rb b/test/prism/ruby_parser_test.rb index 952e493af9159c..e06b7ae4384bf4 100644 --- a/test/prism/ruby_parser_test.rb +++ b/test/prism/ruby_parser_test.rb @@ -52,25 +52,10 @@ class RubyParserTest < TestCase whitequark/string_concat.txt ] - # These files contain CRLF line endings, which ruby_parser translates into - # LF before it gets back to the node. This means the node actually has the - # wrong contents. - crlf = %w[ - dos_endings.txt - heredoc_with_comment.txt - seattlerb/heredoc__backslash_dos_format.txt - seattlerb/heredoc_with_carriage_return_escapes_windows.txt - seattlerb/heredoc_with_extra_carriage_horrible_mix.txt - seattlerb/heredoc_with_extra_carriage_returns_windows.txt - seattlerb/heredoc_with_extra_carriage_returns.txt - seattlerb/heredoc_with_interpolation_and_carriage_return_escapes_windows.txt - seattlerb/heredoc_with_only_carriage_returns_windows.txt - seattlerb/heredoc_with_only_carriage_returns.txt - ] - # https://github.com/seattlerb/ruby_parser/issues/344 - failures = crlf | %w[ + failures = %w[ alias.txt + dos_endings.txt heredocs_with_ignored_newlines.txt method_calls.txt methods.txt @@ -79,8 +64,10 @@ class RubyParserTest < TestCase patterns.txt regex.txt seattlerb/and_multi.txt + seattlerb/heredoc__backslash_dos_format.txt seattlerb/heredoc_bad_hex_escape.txt seattlerb/heredoc_bad_oct_escape.txt + seattlerb/heredoc_with_extra_carriage_horrible_mix.txt spanning_heredoc_newlines.txt spanning_heredoc.txt tilde_heredocs.txt diff --git a/test/prism/snapshots/dos_endings.txt b/test/prism/snapshots/dos_endings.txt index ed75b8a52fdaa7..c5b962f2181cf3 100644 --- a/test/prism/snapshots/dos_endings.txt +++ b/test/prism/snapshots/dos_endings.txt @@ -48,7 +48,7 @@ │ ├── opening_loc: (7,0)-(7,4) = "<<-E" │ ├── content_loc: (8,0)-(11,0) = " 1 \\\r\n 2\r\n 3\r\n" │ ├── closing_loc: (11,0)-(12,0) = "E\r\n" - │ └── unescaped: " 1 2\r\n 3\r\n" + │ └── unescaped: " 1 2\n 3\n" ├── @ LocalVariableWriteNode (location: (13,0)-(15,0)) │ ├── name: :x │ ├── depth: 0 @@ -94,7 +94,7 @@ │ │ │ │ ├── opening_loc: ∅ │ │ │ │ ├── content_loc: (19,0)-(20,0) = " baz\r\n" │ │ │ │ ├── closing_loc: ∅ - │ │ │ │ └── unescaped: "baz\r\n" + │ │ │ │ └── unescaped: "baz\n" │ │ │ └── closing_loc: (20,0)-(21,0) = " EOF\r\n" │ │ ├── call_operator_loc: (17,14)-(17,15) = "." │ │ ├── name: :chop diff --git a/test/prism/snapshots/heredoc_with_comment.txt b/test/prism/snapshots/heredoc_with_comment.txt index 117fdc117a3782..f2225ca981373d 100644 --- a/test/prism/snapshots/heredoc_with_comment.txt +++ b/test/prism/snapshots/heredoc_with_comment.txt @@ -11,7 +11,7 @@ │ ├── opening_loc: (1,0)-(1,9) = "<<-TARGET" │ ├── content_loc: (2,0)-(3,0) = " content makes for an obvious error\r\n" │ ├── closing_loc: (3,0)-(3,6) = "TARGET" - │ └── unescaped: " content makes for an obvious error\r\n" + │ └── unescaped: " content makes for an obvious error\n" ├── call_operator_loc: (1,9)-(1,10) = "." ├── name: :chomp ├── message_loc: (1,10)-(1,15) = "chomp" diff --git a/test/prism/snapshots/seattlerb/heredoc__backslash_dos_format.txt b/test/prism/snapshots/seattlerb/heredoc__backslash_dos_format.txt index 6ba437e36a907b..353e4c6964ceec 100644 --- a/test/prism/snapshots/seattlerb/heredoc__backslash_dos_format.txt +++ b/test/prism/snapshots/seattlerb/heredoc__backslash_dos_format.txt @@ -13,5 +13,5 @@ │ ├── opening_loc: (1,6)-(1,12) = "<<-XXX" │ ├── content_loc: (2,0)-(4,0) = "before\\\r\nafter\r\n" │ ├── closing_loc: (4,0)-(5,0) = "XXX\r\n" - │ └── unescaped: "beforeafter\r\n" + │ └── unescaped: "beforeafter\n" └── operator_loc: (1,4)-(1,5) = "=" diff --git a/test/prism/snapshots/seattlerb/heredoc_with_carriage_return_escapes_windows.txt b/test/prism/snapshots/seattlerb/heredoc_with_carriage_return_escapes_windows.txt index 21802c5707e6eb..2ef6763389d3c6 100644 --- a/test/prism/snapshots/seattlerb/heredoc_with_carriage_return_escapes_windows.txt +++ b/test/prism/snapshots/seattlerb/heredoc_with_carriage_return_escapes_windows.txt @@ -8,4 +8,4 @@ ├── opening_loc: (1,0)-(1,5) = "< Date: Mon, 25 Mar 2024 08:41:57 -0400 Subject: [PATCH 047/211] [ruby/prism] Handle CLRF inside string contents https://github.com/ruby/prism/commit/aac606301e --- prism/prism.c | 52 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/prism/prism.c b/prism/prism.c index a140dc734f0b5a..9f0ecdb93845b5 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -340,10 +340,10 @@ lex_mode_push_string(pm_parser_t *parser, bool interpolation, bool label_allowed // These are the places where we need to split up the content of the // string. We'll use strpbrk to find the first of these characters. uint8_t *breakpoints = lex_mode.as.string.breakpoints; - memcpy(breakpoints, "\n\\\0\0\0", sizeof(lex_mode.as.string.breakpoints)); + memcpy(breakpoints, "\r\n\\\0\0\0", sizeof(lex_mode.as.string.breakpoints)); // Now add in the terminator. - size_t index = 2; + size_t index = 3; breakpoints[index++] = terminator; // If interpolation is allowed, then we're going to check for the # @@ -11042,29 +11042,43 @@ parser_lex(pm_parser_t *parser) { LEX(PM_TOKEN_STRING_END); } - // When we hit a newline, we need to flush any potential heredocs. Note - // that this has to happen after we check for the terminator in case the - // terminator is a newline character. - if (*breakpoint == '\n') { - if (parser->heredoc_end == NULL) { - pm_newline_list_append(&parser->newline_list, breakpoint); - parser->current.end = breakpoint + 1; - breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, true); - continue; - } else { - parser->current.end = breakpoint + 1; - parser_flush_heredoc_end(parser); - pm_token_buffer_flush(parser, &token_buffer); - LEX(PM_TOKEN_STRING_CONTENT); - } - } - switch (*breakpoint) { case '\0': // Skip directly past the null character. parser->current.end = breakpoint + 1; breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, true); break; + case '\r': + if (peek_at(parser, breakpoint + 1) != '\n') { + parser->current.end = breakpoint + 1; + breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, true); + break; + } + + // If we hit a \r\n sequence, then we need to treat it + // as a newline. + parser->current.end = breakpoint + 1; + pm_token_buffer_escape(parser, &token_buffer); + breakpoint++; + token_buffer.cursor = breakpoint; + + /* fallthrough */ + case '\n': + // When we hit a newline, we need to flush any potential + // heredocs. Note that this has to happen after we check + // for the terminator in case the terminator is a + // newline character. + if (parser->heredoc_end == NULL) { + pm_newline_list_append(&parser->newline_list, breakpoint); + parser->current.end = breakpoint + 1; + breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, true); + break; + } + + parser->current.end = breakpoint + 1; + parser_flush_heredoc_end(parser); + pm_token_buffer_flush(parser, &token_buffer); + LEX(PM_TOKEN_STRING_CONTENT); case '\\': { // Here we hit escapes. parser->current.end = breakpoint + 1; From 86077fbcde05f4abd6b306ad0fcc88ee891f8e8b Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 25 Mar 2024 09:24:35 -0400 Subject: [PATCH 048/211] [ruby/prism] Refactor regexp lexing to make it easier to support CLRF https://github.com/ruby/prism/commit/60805d85ca --- prism/prism.c | 239 ++++++++++++++++++------------------ test/prism/encoding_test.rb | 2 +- 2 files changed, 120 insertions(+), 121 deletions(-) diff --git a/prism/prism.c b/prism/prism.c index 9f0ecdb93845b5..6aa611624a2336 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -10783,36 +10783,6 @@ parser_lex(pm_parser_t *parser) { pm_regexp_token_buffer_t token_buffer = { 0 }; while (breakpoint != NULL) { - // If we hit a null byte, skip directly past it. - if (*breakpoint == '\0') { - parser->current.end = breakpoint + 1; - breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, false); - continue; - } - - // If we've hit a newline, then we need to track that in the - // list of newlines. - if (*breakpoint == '\n') { - // For the special case of a newline-terminated regular expression, we will pass - // through this branch twice -- once with PM_TOKEN_REGEXP_BEGIN and then again - // with PM_TOKEN_STRING_CONTENT. Let's avoid tracking the newline twice, by - // tracking it only in the REGEXP_BEGIN case. - if ( - !(lex_mode->as.regexp.terminator == '\n' && parser->current.type != PM_TOKEN_REGEXP_BEGIN) - && parser->heredoc_end == NULL - ) { - pm_newline_list_append(&parser->newline_list, breakpoint); - } - - if (lex_mode->as.regexp.terminator != '\n') { - // If the terminator is not a newline, then we can set - // the next breakpoint and continue. - parser->current.end = breakpoint + 1; - breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, false); - continue; - } - } - // If we hit the terminator, we need to determine what kind of // token to return. if (*breakpoint == lex_mode->as.regexp.terminator) { @@ -10832,9 +10802,17 @@ parser_lex(pm_parser_t *parser) { LEX(PM_TOKEN_STRING_CONTENT); } + // Check here if we need to track the newline. + size_t eol_length = match_eol_at(parser, breakpoint); + if (eol_length) { + parser->current.end = breakpoint + eol_length; + pm_newline_list_append(&parser->newline_list, parser->current.end - 1); + } else { + parser->current.end = breakpoint + 1; + } + // Since we've hit the terminator of the regular expression, // we now need to parse the options. - parser->current.end = breakpoint + 1; parser->current.end += pm_strspn_regexp_option(parser->current.end, parser->end - parser->current.end); lex_mode_pop(parser); @@ -10842,114 +10820,135 @@ parser_lex(pm_parser_t *parser) { LEX(PM_TOKEN_REGEXP_END); } - // If we hit escapes, then we need to treat the next token - // literally. In this case we'll skip past the next character + // If we've hit the incrementor, then we need to skip past it // and find the next breakpoint. - if (*breakpoint == '\\') { + if (*breakpoint && *breakpoint == lex_mode->as.regexp.incrementor) { parser->current.end = breakpoint + 1; + breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, false); + lex_mode->as.regexp.nesting++; + continue; + } - // If we've hit the end of the file, then break out of the - // loop by setting the breakpoint to NULL. - if (parser->current.end == parser->end) { - breakpoint = NULL; - continue; - } - - pm_regexp_token_buffer_escape(parser, &token_buffer); - uint8_t peeked = peek(parser); + switch (*breakpoint) { + case '\0': + // If we hit a null byte, skip directly past it. + parser->current.end = breakpoint + 1; + breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, false); + break; + case '\n': + // If we've hit a newline, then we need to track that in + // the list of newlines. + if (parser->heredoc_end == NULL) { + pm_newline_list_append(&parser->newline_list, breakpoint); + } - switch (peeked) { - case '\r': - parser->current.end++; - if (peek(parser) != '\n') { - if (lex_mode->as.regexp.terminator != '\r') { - pm_token_buffer_push_byte(&token_buffer.base, '\\'); - } - pm_regexp_token_buffer_push_byte(&token_buffer, '\r'); - pm_token_buffer_push_byte(&token_buffer.base, '\r'); - break; - } - /* fallthrough */ - case '\n': - if (parser->heredoc_end) { - // ... if we are on the same line as a heredoc, - // flush the heredoc and continue parsing after - // heredoc_end. - parser_flush_heredoc_end(parser); - pm_regexp_token_buffer_copy(parser, &token_buffer); - LEX(PM_TOKEN_STRING_CONTENT); - } else { - // ... else track the newline. - pm_newline_list_append(&parser->newline_list, parser->current.end); - } + parser->current.end = breakpoint + 1; + breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, false); + break; + case '\\': { + // If we hit escapes, then we need to treat the next + // token literally. In this case we'll skip past the + // next character and find the next breakpoint. + parser->current.end = breakpoint + 1; - parser->current.end++; - break; - case 'c': - case 'C': - case 'M': - case 'u': - case 'x': - escape_read(parser, &token_buffer.regexp_buffer, &token_buffer.base.buffer, PM_ESCAPE_FLAG_REGEXP); + // If we've hit the end of the file, then break out of + // the loop by setting the breakpoint to NULL. + if (parser->current.end == parser->end) { + breakpoint = NULL; break; - default: - if (lex_mode->as.regexp.terminator == peeked) { - // Some characters when they are used as the - // terminator also receive an escape. They are - // enumerated here. - switch (peeked) { - case '$': case ')': case '*': case '+': - case '.': case '>': case '?': case ']': - case '^': case '|': case '}': + } + + pm_regexp_token_buffer_escape(parser, &token_buffer); + uint8_t peeked = peek(parser); + + switch (peeked) { + case '\r': + parser->current.end++; + if (peek(parser) != '\n') { + if (lex_mode->as.regexp.terminator != '\r') { pm_token_buffer_push_byte(&token_buffer.base, '\\'); - break; - default: - break; + } + pm_regexp_token_buffer_push_byte(&token_buffer, '\r'); + pm_token_buffer_push_byte(&token_buffer.base, '\r'); + break; + } + /* fallthrough */ + case '\n': + if (parser->heredoc_end) { + // ... if we are on the same line as a heredoc, + // flush the heredoc and continue parsing after + // heredoc_end. + parser_flush_heredoc_end(parser); + pm_regexp_token_buffer_copy(parser, &token_buffer); + LEX(PM_TOKEN_STRING_CONTENT); + } else { + // ... else track the newline. + pm_newline_list_append(&parser->newline_list, parser->current.end); } - pm_regexp_token_buffer_push_byte(&token_buffer, peeked); - pm_token_buffer_push_byte(&token_buffer.base, peeked); parser->current.end++; break; - } - - if (peeked < 0x80) pm_token_buffer_push_byte(&token_buffer.base, '\\'); - pm_regexp_token_buffer_push_escaped(&token_buffer, parser); - break; - } + case 'c': + case 'C': + case 'M': + case 'u': + case 'x': + escape_read(parser, &token_buffer.regexp_buffer, &token_buffer.base.buffer, PM_ESCAPE_FLAG_REGEXP); + break; + default: + if (lex_mode->as.regexp.terminator == peeked) { + // Some characters when they are used as the + // terminator also receive an escape. They are + // enumerated here. + switch (peeked) { + case '$': case ')': case '*': case '+': + case '.': case '>': case '?': case ']': + case '^': case '|': case '}': + pm_token_buffer_push_byte(&token_buffer.base, '\\'); + break; + default: + break; + } - token_buffer.base.cursor = parser->current.end; - breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, false); - continue; - } + pm_regexp_token_buffer_push_byte(&token_buffer, peeked); + pm_token_buffer_push_byte(&token_buffer.base, peeked); + parser->current.end++; + break; + } - // If we hit a #, then we will attempt to lex interpolation. - if (*breakpoint == '#') { - pm_token_type_t type = lex_interpolation(parser, breakpoint); + if (peeked < 0x80) pm_token_buffer_push_byte(&token_buffer.base, '\\'); + pm_regexp_token_buffer_push_escaped(&token_buffer, parser); + break; + } - if (type == PM_TOKEN_NOT_PROVIDED) { - // If we haven't returned at this point then we had - // something that looked like an interpolated class or - // instance variable like "#@" but wasn't actually. In - // this case we'll just skip to the next breakpoint. + token_buffer.base.cursor = parser->current.end; breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, false); - continue; + break; } + case '#': { + // If we hit a #, then we will attempt to lex + // interpolation. + pm_token_type_t type = lex_interpolation(parser, breakpoint); - if (type == PM_TOKEN_STRING_CONTENT) { - pm_regexp_token_buffer_flush(parser, &token_buffer); - } + if (type == PM_TOKEN_NOT_PROVIDED) { + // If we haven't returned at this point then we had + // something that looked like an interpolated class or + // instance variable like "#@" but wasn't actually. In + // this case we'll just skip to the next breakpoint. + breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, false); + break; + } - LEX(type); - } + if (type == PM_TOKEN_STRING_CONTENT) { + pm_regexp_token_buffer_flush(parser, &token_buffer); + } - // If we've hit the incrementor, then we need to skip past it - // and find the next breakpoint. - assert(*breakpoint == lex_mode->as.regexp.incrementor); - parser->current.end = breakpoint + 1; - breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, false); - lex_mode->as.regexp.nesting++; - continue; + LEX(type); + } + default: + assert(false && "unreachable"); + break; + } } if (parser->current.end > parser->current.start) { diff --git a/test/prism/encoding_test.rb b/test/prism/encoding_test.rb index 649d05b874164b..2aee473ddf9901 100644 --- a/test/prism/encoding_test.rb +++ b/test/prism/encoding_test.rb @@ -344,7 +344,7 @@ def assert_encoding(encoding, name, range) next if ["/", "{"].include?(character) source = "# encoding: #{name}\n/(?##{character})/\n" - assert Prism.parse(source).success? + assert Prism.parse(source).success?, "Expected #{source.inspect} to parse successfully." end rescue RangeError source = "# encoding: #{name}\n\\x#{codepoint.to_s(16)}" From 53cc2723877f7794807684e31a530daca1a72ed6 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 25 Mar 2024 09:27:27 -0400 Subject: [PATCH 049/211] [ruby/prism] Handle CLRF in regexp https://github.com/ruby/prism/commit/b96bada9ae --- prism/parser.h | 4 ++-- prism/prism.c | 19 ++++++++++++++++--- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/prism/parser.h b/prism/parser.h index b685fa377d7eac..7e4bb99197259e 100644 --- a/prism/parser.h +++ b/prism/parser.h @@ -173,7 +173,7 @@ typedef struct pm_lex_mode { * This is the character set that should be used to delimit the * tokens within the regular expression. */ - uint8_t breakpoints[6]; + uint8_t breakpoints[7]; } regexp; struct { @@ -206,7 +206,7 @@ typedef struct pm_lex_mode { * This is the character set that should be used to delimit the * tokens within the string. */ - uint8_t breakpoints[6]; + uint8_t breakpoints[7]; } string; struct { diff --git a/prism/prism.c b/prism/prism.c index 6aa611624a2336..58c70dba69cdc0 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -308,14 +308,14 @@ lex_mode_push_regexp(pm_parser_t *parser, uint8_t incrementor, uint8_t terminato // regular expression. We'll use strpbrk to find the first of these // characters. uint8_t *breakpoints = lex_mode.as.regexp.breakpoints; - memcpy(breakpoints, "\n\\#\0\0", sizeof(lex_mode.as.regexp.breakpoints)); + memcpy(breakpoints, "\r\n\\#\0\0", sizeof(lex_mode.as.regexp.breakpoints)); // First we'll add the terminator. - breakpoints[3] = terminator; + breakpoints[4] = terminator; // Next, if there is an incrementor, then we'll check for that as well. if (incrementor != '\0') { - breakpoints[4] = incrementor; + breakpoints[5] = incrementor; } return lex_mode_push(parser, lex_mode); @@ -10835,6 +10835,19 @@ parser_lex(pm_parser_t *parser) { parser->current.end = breakpoint + 1; breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, false); break; + case '\r': + if (peek_at(parser, breakpoint + 1) != '\n') { + parser->current.end = breakpoint + 1; + breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, false); + break; + } + + parser->current.end = breakpoint + 1; + pm_regexp_token_buffer_escape(parser, &token_buffer); + breakpoint++; + token_buffer.base.cursor = breakpoint; + + /* fallthrough */ case '\n': // If we've hit a newline, then we need to track that in // the list of newlines. From a08954569f197312db4d6b217f1b8ba3441fc078 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 25 Mar 2024 10:08:21 -0400 Subject: [PATCH 050/211] [ruby/prism] Fix up minimal build setting https://github.com/ruby/prism/commit/98c85c4acb --- prism/api_pack.c | 9 +++++++++ prism/defines.h | 21 +++++++++++++++++++++ prism/extension.c | 20 +++++++++++++++++--- prism/pack.h | 4 ++-- prism/parser.h | 2 +- prism/prettyprint.h | 4 ++-- prism/prism.c | 10 +++++----- test/prism/ruby_parser_test.rb | 3 +++ 8 files changed, 60 insertions(+), 13 deletions(-) diff --git a/prism/api_pack.c b/prism/api_pack.c index c9f0b18a397bc7..98509ae65ccae0 100644 --- a/prism/api_pack.c +++ b/prism/api_pack.c @@ -1,5 +1,12 @@ #include "prism/extension.h" +#ifdef PRISM_EXCLUDE_PACK + +void +Init_prism_pack(void) {} + +#else + static VALUE rb_cPrism; static VALUE rb_cPrismPack; static VALUE rb_cPrismPackDirective; @@ -265,3 +272,5 @@ Init_prism_pack(void) { pack_symbol = ID2SYM(rb_intern("pack")); unpack_symbol = ID2SYM(rb_intern("unpack")); } + +#endif diff --git a/prism/defines.h b/prism/defines.h index aca3c6dc088fcf..2fe73fe3d8cbd9 100644 --- a/prism/defines.h +++ b/prism/defines.h @@ -182,4 +182,25 @@ #endif #endif +/** + * If PRISM_BUILD_MINIMAL is defined, then we're going to define every possible + * switch that will turn off certain features of prism. + */ +#ifdef PRISM_BUILD_MINIMAL + /** Exclude the serialization API. */ + #define PRISM_EXCLUDE_SERIALIZATION + + /** Exclude the JSON serialization API. */ + #define PRISM_EXCLUDE_JSON + + /** Exclude the Array#pack parser API. */ + #define PRISM_EXCLUDE_PACK + + /** Exclude the prettyprint API. */ + #define PRISM_EXCLUDE_PRETTYPRINT + + /** Exclude the full set of encodings, using the minimal only. */ + #define PRISM_ENCODING_EXCLUDE_FULL +#endif + #endif diff --git a/prism/extension.c b/prism/extension.c index 7c8636e3dfc15c..88ba006ac2ec30 100644 --- a/prism/extension.c +++ b/prism/extension.c @@ -270,6 +270,8 @@ file_options(int argc, VALUE *argv, pm_string_t *input, pm_options_t *options) { } } +#ifndef PRISM_EXCLUDE_SERIALIZATION + /******************************************************************************/ /* Serializing the AST */ /******************************************************************************/ @@ -351,6 +353,8 @@ dump_file(int argc, VALUE *argv, VALUE self) { return value; } +#endif + /******************************************************************************/ /* Extracting values for the parse result */ /******************************************************************************/ @@ -1129,6 +1133,8 @@ profile_file(VALUE self, VALUE filepath) { return Qnil; } +#ifndef PRISM_EXCLUDE_PRETTYPRINT + /** * call-seq: * Debug::inspect_node(source) -> inspected @@ -1159,6 +1165,8 @@ inspect_node(VALUE self, VALUE source) { return string; } +#endif + /** * call-seq: * Debug::format_errors(source, colorize) -> String @@ -1349,8 +1357,6 @@ Init_prism(void) { rb_define_const(rb_cPrism, "VERSION", rb_str_new2(EXPECTED_PRISM_VERSION)); // First, the functions that have to do with lexing and parsing. - rb_define_singleton_method(rb_cPrism, "dump", dump, -1); - rb_define_singleton_method(rb_cPrism, "dump_file", dump_file, -1); rb_define_singleton_method(rb_cPrism, "lex", lex, -1); rb_define_singleton_method(rb_cPrism, "lex_file", lex_file, -1); rb_define_singleton_method(rb_cPrism, "parse", parse, -1); @@ -1363,6 +1369,11 @@ Init_prism(void) { rb_define_singleton_method(rb_cPrism, "parse_success?", parse_success_p, -1); rb_define_singleton_method(rb_cPrism, "parse_file_success?", parse_file_success_p, -1); +#ifndef PRISM_EXCLUDE_SERIALIZATION + rb_define_singleton_method(rb_cPrism, "dump", dump, -1); + rb_define_singleton_method(rb_cPrism, "dump_file", dump_file, -1); +#endif + // Next, the functions that will be called by the parser to perform various // internal tasks. We expose these to make them easier to test. VALUE rb_cPrismDebug = rb_define_module_under(rb_cPrism, "Debug"); @@ -1370,10 +1381,13 @@ Init_prism(void) { rb_define_singleton_method(rb_cPrismDebug, "integer_parse", integer_parse, 1); rb_define_singleton_method(rb_cPrismDebug, "memsize", memsize, 1); rb_define_singleton_method(rb_cPrismDebug, "profile_file", profile_file, 1); - rb_define_singleton_method(rb_cPrismDebug, "inspect_node", inspect_node, 1); rb_define_singleton_method(rb_cPrismDebug, "format_errors", format_errors, 2); rb_define_singleton_method(rb_cPrismDebug, "static_inspect", static_inspect, -1); +#ifndef PRISM_EXCLUDE_PRETTYPRINT + rb_define_singleton_method(rb_cPrismDebug, "inspect_node", inspect_node, 1); +#endif + // Next, define the functions that are exposed through the private // Debug::Encoding class. rb_cPrismDebugEncoding = rb_define_class_under(rb_cPrismDebug, "Encoding", rb_cObject); diff --git a/prism/pack.h b/prism/pack.h index cfdc251fe6d589..0b0b4b19cc85d4 100644 --- a/prism/pack.h +++ b/prism/pack.h @@ -6,6 +6,8 @@ #ifndef PRISM_PACK_H #define PRISM_PACK_H +#include "prism/defines.h" + // We optionally support parsing String#pack templates. For systems that don't // want or need this functionality, it can be turned off with the // PRISM_EXCLUDE_PACK define. @@ -15,8 +17,6 @@ void pm_pack_parse(void); #else -#include "prism/defines.h" - #include #include diff --git a/prism/parser.h b/prism/parser.h index 7e4bb99197259e..f706a67de7c75d 100644 --- a/prism/parser.h +++ b/prism/parser.h @@ -6,8 +6,8 @@ #ifndef PRISM_PARSER_H #define PRISM_PARSER_H -#include "prism/ast.h" #include "prism/defines.h" +#include "prism/ast.h" #include "prism/encoding.h" #include "prism/options.h" #include "prism/util/pm_constant_pool.h" diff --git a/prism/prettyprint.h b/prism/prettyprint.h index ea11b4a24687b3..5a52b2b6b8eb43 100644 --- a/prism/prettyprint.h +++ b/prism/prettyprint.h @@ -6,14 +6,14 @@ #ifndef PRISM_PRETTYPRINT_H #define PRISM_PRETTYPRINT_H +#include "prism/defines.h" + #ifdef PRISM_EXCLUDE_PRETTYPRINT void pm_prettyprint(void); #else -#include "prism/defines.h" - #include #include "prism/ast.h" diff --git a/prism/prism.c b/prism/prism.c index 58c70dba69cdc0..6f2ab81e9ae3f3 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -10842,9 +10842,9 @@ parser_lex(pm_parser_t *parser) { break; } - parser->current.end = breakpoint + 1; - pm_regexp_token_buffer_escape(parser, &token_buffer); breakpoint++; + parser->current.end = breakpoint; + pm_regexp_token_buffer_escape(parser, &token_buffer); token_buffer.base.cursor = breakpoint; /* fallthrough */ @@ -11069,9 +11069,9 @@ parser_lex(pm_parser_t *parser) { // If we hit a \r\n sequence, then we need to treat it // as a newline. - parser->current.end = breakpoint + 1; - pm_token_buffer_escape(parser, &token_buffer); breakpoint++; + parser->current.end = breakpoint; + pm_token_buffer_escape(parser, &token_buffer); token_buffer.cursor = breakpoint; /* fallthrough */ @@ -11321,8 +11321,8 @@ parser_lex(pm_parser_t *parser) { // If we hit a \r\n sequence, then we want to replace it // with a single \n character in the final string. - pm_token_buffer_escape(parser, &token_buffer); breakpoint++; + pm_token_buffer_escape(parser, &token_buffer); token_buffer.cursor = breakpoint; /* fallthrough */ diff --git a/test/prism/ruby_parser_test.rb b/test/prism/ruby_parser_test.rb index e06b7ae4384bf4..c9883574a68b31 100644 --- a/test/prism/ruby_parser_test.rb +++ b/test/prism/ruby_parser_test.rb @@ -68,6 +68,9 @@ class RubyParserTest < TestCase seattlerb/heredoc_bad_hex_escape.txt seattlerb/heredoc_bad_oct_escape.txt seattlerb/heredoc_with_extra_carriage_horrible_mix.txt + seattlerb/heredoc_with_extra_carriage_returns_windows.txt + seattlerb/heredoc_with_only_carriage_returns_windows.txt + seattlerb/heredoc_with_only_carriage_returns.txt spanning_heredoc_newlines.txt spanning_heredoc.txt tilde_heredocs.txt From 46bf6ae886dc14d5e3a76d53eb4f97375f7c03c5 Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Mon, 25 Mar 2024 09:06:47 -0700 Subject: [PATCH 051/211] YJIT: Propagate Array, Hash, and String classes (#10323) --- bootstraptest/test_yjit.rb | 28 ++++++++++++++++++ class.c | 2 ++ common.mk | 1 + yjit.h | 2 ++ yjit/src/codegen.rs | 57 ++++++++++++++++++++++++++----------- yjit/src/core.rs | 53 +++++++++++++++++++++------------- yjit/src/invariants.rs | 58 ++++++++++++++++++++++++++++++++++++++ yjit/src/stats.rs | 4 ++- 8 files changed, 168 insertions(+), 37 deletions(-) diff --git a/bootstraptest/test_yjit.rb b/bootstraptest/test_yjit.rb index 3eef86e0a9f4b7..24253a10c9aa2c 100644 --- a/bootstraptest/test_yjit.rb +++ b/bootstraptest/test_yjit.rb @@ -4725,3 +4725,31 @@ def test_cases(file, chain) test_cases(File, Enumerator::Chain) } + +# singleton class should invalidate Type::CString assumption +assert_equal 'foo', %q{ + def define_singleton(str, define) + if define + # Wrap a C method frame to avoid exiting JIT code on defineclass + [nil].reverse_each do + class << str + def +(_) + "foo" + end + end + end + end + "bar" + end + + def entry(define) + str = "" + # When `define` is false, #+ compiles to rb_str_plus() without a class guard. + # When the code is reused with `define` is true, the class of `str` is changed + # to a singleton class, so the block should be invalidated. + str + define_singleton(str, define) + end + + entry(false) + entry(true) +} diff --git a/class.c b/class.c index 46b8c1f74a5a6b..879c0ead2af2ed 100644 --- a/class.c +++ b/class.c @@ -29,6 +29,7 @@ #include "internal/variable.h" #include "ruby/st.h" #include "vm_core.h" +#include "yjit.h" /* Flags of T_CLASS * @@ -805,6 +806,7 @@ make_singleton_class(VALUE obj) FL_SET(klass, FL_SINGLETON); RBASIC_SET_CLASS(obj, klass); rb_singleton_class_attached(klass, obj); + rb_yjit_invalidate_no_singleton_class(orig_class); SET_METACLASS_OF(klass, METACLASS_OF(rb_class_real(orig_class))); return klass; diff --git a/common.mk b/common.mk index 21ceaa3f01279b..9dbca5b731da3f 100644 --- a/common.mk +++ b/common.mk @@ -3106,6 +3106,7 @@ class.$(OBJEXT): {$(VPATH)}vm_core.h class.$(OBJEXT): {$(VPATH)}vm_debug.h class.$(OBJEXT): {$(VPATH)}vm_opts.h class.$(OBJEXT): {$(VPATH)}vm_sync.h +class.$(OBJEXT): {$(VPATH)}yjit.h compar.$(OBJEXT): $(hdrdir)/ruby/ruby.h compar.$(OBJEXT): $(hdrdir)/ruby/version.h compar.$(OBJEXT): $(top_srcdir)/internal/basic_operators.h diff --git a/yjit.h b/yjit.h index 46218a47d72d44..2f5317ad977b6e 100644 --- a/yjit.h +++ b/yjit.h @@ -46,6 +46,7 @@ void rb_yjit_constant_ic_update(const rb_iseq_t *const iseq, IC ic, unsigned ins void rb_yjit_tracing_invalidate_all(void); void rb_yjit_show_usage(int help, int highlight, unsigned int width, int columns); void rb_yjit_lazy_push_frame(const VALUE *pc); +void rb_yjit_invalidate_no_singleton_class(VALUE klass); #else // !USE_YJIT @@ -68,6 +69,7 @@ static inline void rb_yjit_before_ractor_spawn(void) {} static inline void rb_yjit_constant_ic_update(const rb_iseq_t *const iseq, IC ic, unsigned insn_idx) {} static inline void rb_yjit_tracing_invalidate_all(void) {} static inline void rb_yjit_lazy_push_frame(const VALUE *pc) {} +static inline void rb_yjit_invalidate_no_singleton_class(VALUE klass) {} #endif // #if USE_YJIT diff --git a/yjit/src/codegen.rs b/yjit/src/codegen.rs index 1d9955260b2322..7d9085d4b30d4e 100644 --- a/yjit/src/codegen.rs +++ b/yjit/src/codegen.rs @@ -97,6 +97,9 @@ pub struct JITState { /// not been written to for the block to be valid. pub stable_constant_names_assumption: Option<*const ID>, + /// A list of classes that are not supposed to have a singleton class. + pub no_singleton_class_assumptions: Vec, + /// When true, the block is valid only when there is a total of one ractor running pub block_assumes_single_ractor: bool, @@ -125,6 +128,7 @@ impl JITState { method_lookup_assumptions: vec![], bop_assumptions: vec![], stable_constant_names_assumption: None, + no_singleton_class_assumptions: vec![], block_assumes_single_ractor: false, perf_map: Rc::default(), perf_stack: vec![], @@ -231,6 +235,20 @@ impl JITState { Some(()) } + /// Assume that objects of a given class will have no singleton class. + /// Return true if there has been no such singleton class since boot + /// and we can safely invalidate it. + pub fn assume_no_singleton_class(&mut self, asm: &mut Assembler, ocb: &mut OutlinedCb, klass: VALUE) -> bool { + if jit_ensure_block_entry_exit(self, asm, ocb).is_none() { + return false; // out of space, give up + } + if has_singleton_class_of(klass) { + return false; // we've seen a singleton class. disable the optimization to avoid an invalidation loop. + } + self.no_singleton_class_assumptions.push(klass); + true + } + fn get_cfp(&self) -> *mut rb_control_frame_struct { unsafe { get_ec_cfp(self.ec) } } @@ -1504,7 +1522,7 @@ fn gen_newarray( ); asm.stack_pop(n.as_usize()); - let stack_ret = asm.stack_push(Type::TArray); + let stack_ret = asm.stack_push(Type::CArray); asm.mov(stack_ret, new_ary); Some(KeepCompiling) @@ -1527,7 +1545,7 @@ fn gen_duparray( vec![ary.into()], ); - let stack_ret = asm.stack_push(Type::TArray); + let stack_ret = asm.stack_push(Type::CArray); asm.mov(stack_ret, new_ary); Some(KeepCompiling) @@ -1547,7 +1565,7 @@ fn gen_duphash( // call rb_hash_resurrect(VALUE hash); let hash = asm.ccall(rb_hash_resurrect as *const u8, vec![hash.into()]); - let stack_ret = asm.stack_push(Type::THash); + let stack_ret = asm.stack_push(Type::CHash); asm.mov(stack_ret, hash); Some(KeepCompiling) @@ -2303,12 +2321,12 @@ fn gen_newhash( asm.cpop_into(new_hash); // x86 alignment asm.stack_pop(num.try_into().unwrap()); - let stack_ret = asm.stack_push(Type::THash); + let stack_ret = asm.stack_push(Type::CHash); asm.mov(stack_ret, new_hash); } else { // val = rb_hash_new(); let new_hash = asm.ccall(rb_hash_new as *const u8, vec![]); - let stack_ret = asm.stack_push(Type::THash); + let stack_ret = asm.stack_push(Type::CHash); asm.mov(stack_ret, new_hash); } @@ -2330,7 +2348,7 @@ fn gen_putstring( vec![EC, put_val.into(), 0.into()] ); - let stack_top = asm.stack_push(Type::TString); + let stack_top = asm.stack_push(Type::CString); asm.mov(stack_top, str_opnd); Some(KeepCompiling) @@ -2351,7 +2369,7 @@ fn gen_putchilledstring( vec![EC, put_val.into(), 1.into()] ); - let stack_top = asm.stack_push(Type::TString); + let stack_top = asm.stack_push(Type::CString); asm.mov(stack_top, str_opnd); Some(KeepCompiling) @@ -4493,8 +4511,18 @@ fn jit_guard_known_klass( let val_type = asm.ctx.get_opnd_type(insn_opnd); if val_type.known_class() == Some(known_klass) { - // We already know from type information that this is a match - return; + // Unless frozen, Array, Hash, and String objects may change their RBASIC_CLASS + // when they get a singleton class. Those types need invalidations. + if unsafe { [rb_cArray, rb_cHash, rb_cString].contains(&known_klass) } { + if jit.assume_no_singleton_class(asm, ocb, known_klass) { + // Speculate that this object will not have a singleton class, + // and invalidate the block in case it does. + return; + } + } else { + // We already know from type information that this is a match + return; + } } if unsafe { known_klass == rb_cNilClass } { @@ -4613,14 +4641,11 @@ fn jit_guard_known_klass( jit_chain_guard(JCC_JNE, jit, asm, ocb, max_chain_depth, counter); if known_klass == unsafe { rb_cString } { - // Upgrading to Type::CString here is incorrect. - // The guard we put only checks RBASIC_CLASS(obj), - // which adding a singleton class can change. We - // additionally need to know the string is frozen - // to claim Type::CString. - asm.ctx.upgrade_opnd_type(insn_opnd, Type::TString); + asm.ctx.upgrade_opnd_type(insn_opnd, Type::CString); } else if known_klass == unsafe { rb_cArray } { - asm.ctx.upgrade_opnd_type(insn_opnd, Type::TArray); + asm.ctx.upgrade_opnd_type(insn_opnd, Type::CArray); + } else if known_klass == unsafe { rb_cHash } { + asm.ctx.upgrade_opnd_type(insn_opnd, Type::CHash); } } } diff --git a/yjit/src/core.rs b/yjit/src/core.rs index 000e0c0be03314..7f6e7d47f9defd 100644 --- a/yjit/src/core.rs +++ b/yjit/src/core.rs @@ -52,20 +52,18 @@ pub enum Type { Flonum, ImmSymbol, - #[allow(unused)] - HeapSymbol, - TString, // An object with the T_STRING flag set, possibly an rb_cString CString, // An un-subclassed string of type rb_cString (can have instance vars in some cases) TArray, // An object with the T_ARRAY flag set, possibly an rb_cArray + CArray, // An un-subclassed array of type rb_cArray (can have instance vars in some cases) THash, // An object with the T_HASH flag set, possibly an rb_cHash + CHash, // An un-subclassed hash of type rb_cHash (can have instance vars in some cases) BlockParamProxy, // A special sentinel value indicating the block parameter should be read from // the current surrounding cfp // The context currently relies on types taking at most 4 bits (max value 15) - // to encode, so if we add two more, we will need to refactor the context, - // or we could remove HeapSymbol, which is currently unused. + // to encode, so if we add any more, we will need to refactor the context. } // Default initialization @@ -98,8 +96,11 @@ impl Type { // Core.rs can't reference rb_cString because it's linked by Rust-only tests. // But CString vs TString is only an optimisation and shouldn't affect correctness. #[cfg(not(test))] - if val.class_of() == unsafe { rb_cString } && val.is_frozen() { - return Type::CString; + match val.class_of() { + class if class == unsafe { rb_cArray } => return Type::CArray, + class if class == unsafe { rb_cHash } => return Type::CHash, + class if class == unsafe { rb_cString } => return Type::CString, + _ => {} } // We likewise can't reference rb_block_param_proxy, but it's again an optimisation; // we can just treat it as a normal Object. @@ -150,8 +151,9 @@ impl Type { match self { Type::UnknownHeap => true, Type::TArray => true, + Type::CArray => true, Type::THash => true, - Type::HeapSymbol => true, + Type::CHash => true, Type::TString => true, Type::CString => true, Type::BlockParamProxy => true, @@ -161,21 +163,17 @@ impl Type { /// Check if it's a T_ARRAY object (both TArray and CArray are T_ARRAY) pub fn is_array(&self) -> bool { - matches!(self, Type::TArray) + matches!(self, Type::TArray | Type::CArray) } - /// Check if it's a T_HASH object + /// Check if it's a T_HASH object (both THash and CHash are T_HASH) pub fn is_hash(&self) -> bool { - matches!(self, Type::THash) + matches!(self, Type::THash | Type::CHash) } /// Check if it's a T_STRING object (both TString and CString are T_STRING) pub fn is_string(&self) -> bool { - match self { - Type::TString => true, - Type::CString => true, - _ => false, - } + matches!(self, Type::TString | Type::CString) } /// Returns an Option with the T_ value type if it is known, otherwise None @@ -186,9 +184,9 @@ impl Type { Type::False => Some(RUBY_T_FALSE), Type::Fixnum => Some(RUBY_T_FIXNUM), Type::Flonum => Some(RUBY_T_FLOAT), - Type::TArray => Some(RUBY_T_ARRAY), - Type::THash => Some(RUBY_T_HASH), - Type::ImmSymbol | Type::HeapSymbol => Some(RUBY_T_SYMBOL), + Type::TArray | Type::CArray => Some(RUBY_T_ARRAY), + Type::THash | Type::CHash => Some(RUBY_T_HASH), + Type::ImmSymbol => Some(RUBY_T_SYMBOL), Type::TString | Type::CString => Some(RUBY_T_STRING), Type::Unknown | Type::UnknownImm | Type::UnknownHeap => None, Type::BlockParamProxy => None, @@ -204,7 +202,9 @@ impl Type { Type::False => Some(rb_cFalseClass), Type::Fixnum => Some(rb_cInteger), Type::Flonum => Some(rb_cFloat), - Type::ImmSymbol | Type::HeapSymbol => Some(rb_cSymbol), + Type::ImmSymbol => Some(rb_cSymbol), + Type::CArray => Some(rb_cArray), + Type::CHash => Some(rb_cHash), Type::CString => Some(rb_cString), _ => None, } @@ -255,6 +255,16 @@ impl Type { return TypeDiff::Compatible(1); } + // A CArray is also a TArray. + if self == Type::CArray && dst == Type::TArray { + return TypeDiff::Compatible(1); + } + + // A CHash is also a THash. + if self == Type::CHash && dst == Type::THash { + return TypeDiff::Compatible(1); + } + // A CString is also a TString. if self == Type::CString && dst == Type::TString { return TypeDiff::Compatible(1); @@ -1644,6 +1654,9 @@ impl JITState { if let Some(idlist) = self.stable_constant_names_assumption { track_stable_constant_names_assumption(blockref, idlist); } + for klass in self.no_singleton_class_assumptions { + track_no_singleton_class_assumption(blockref, klass); + } blockref } diff --git a/yjit/src/invariants.rs b/yjit/src/invariants.rs index 59f7b70e201d8f..c4facb31dd00d9 100644 --- a/yjit/src/invariants.rs +++ b/yjit/src/invariants.rs @@ -53,6 +53,12 @@ pub struct Invariants { /// A map from a block to a set of IDs that it is assuming have not been /// redefined. block_constant_states: HashMap>, + + /// A map from a class to a set of blocks that assume objects of the class + /// will have no singleton class. When the set is empty, it means that + /// there has been a singleton class for the class after boot, so you cannot + /// assume no singleton class going forward. + no_singleton_classes: HashMap>, } /// Private singleton instance of the invariants global struct. @@ -69,6 +75,7 @@ impl Invariants { single_ractor: HashSet::new(), constant_state_blocks: HashMap::new(), block_constant_states: HashMap::new(), + no_singleton_classes: HashMap::new(), }); } } @@ -130,6 +137,23 @@ pub fn track_method_lookup_stability_assumption( .insert(uninit_block); } +/// Track that a block will assume that `klass` objects will have no singleton class. +pub fn track_no_singleton_class_assumption(uninit_block: BlockRef, klass: VALUE) { + Invariants::get_instance() + .no_singleton_classes + .entry(klass) + .or_default() + .insert(uninit_block); +} + +/// Returns true if we've seen a singleton class of a given class since boot. +pub fn has_singleton_class_of(klass: VALUE) -> bool { + Invariants::get_instance() + .no_singleton_classes + .get(&klass) + .map_or(false, |blocks| blocks.is_empty()) +} + // Checks rb_method_basic_definition_p and registers the current block for invalidation if method // lookup changes. // A "basic method" is one defined during VM boot, so we can use this to check assumptions based on @@ -391,6 +415,11 @@ pub fn block_assumptions_free(blockref: BlockRef) { if invariants.constant_state_blocks.is_empty() { invariants.constant_state_blocks.shrink_to_fit(); } + + // Remove tracking for blocks assumping no singleton class + for (_, blocks) in invariants.no_singleton_classes.iter_mut() { + blocks.remove(&blockref); + } } /// Callback from the opt_setinlinecache instruction in the interpreter. @@ -457,6 +486,35 @@ pub extern "C" fn rb_yjit_constant_ic_update(iseq: *const rb_iseq_t, ic: IC, ins }); } +/// Invalidate blocks that assume objects of a given class will have no singleton class. +#[no_mangle] +pub extern "C" fn rb_yjit_invalidate_no_singleton_class(klass: VALUE) { + // Skip tracking singleton classes during boot. Such objects already have a singleton class + // before entering JIT code, so they get rejected when they're checked for the first time. + if unsafe { INVARIANTS.is_none() } { + return; + } + + // We apply this optimization only to Array, Hash, and String for now. + if unsafe { [rb_cArray, rb_cHash, rb_cString].contains(&klass) } { + let no_singleton_classes = &mut Invariants::get_instance().no_singleton_classes; + match no_singleton_classes.get_mut(&klass) { + Some(blocks) => { + // Invalidate existing blocks and let has_singleton_class_of() + // return true when they are compiled again + for block in mem::take(blocks) { + invalidate_block_version(&block); + incr_counter!(invalidate_no_singleton_class); + } + } + None => { + // Let has_singleton_class_of() return true for this class + no_singleton_classes.insert(klass, HashSet::new()); + } + } + } +} + // Invalidate all generated code and patch C method return code to contain // logic for firing the c_return TracePoint event. Once rb_vm_barrier() // returns, all other ractors are pausing inside RB_VM_LOCK_ENTER(), which diff --git a/yjit/src/stats.rs b/yjit/src/stats.rs index c1315aad90c4c9..621854d48793b2 100644 --- a/yjit/src/stats.rs +++ b/yjit/src/stats.rs @@ -261,7 +261,7 @@ macro_rules! make_counters { /// The list of counters that are available without --yjit-stats. /// They are incremented only by `incr_counter!` and don't use `gen_counter_incr`. -pub const DEFAULT_COUNTERS: [Counter; 15] = [ +pub const DEFAULT_COUNTERS: [Counter; 16] = [ Counter::code_gc_count, Counter::compiled_iseq_entry, Counter::cold_iseq_entry, @@ -278,6 +278,7 @@ pub const DEFAULT_COUNTERS: [Counter; 15] = [ Counter::invalidate_ractor_spawn, Counter::invalidate_constant_state_bump, Counter::invalidate_constant_ic_fill, + Counter::invalidate_no_singleton_class, ]; /// Macro to increase a counter by name and count @@ -559,6 +560,7 @@ make_counters! { invalidate_ractor_spawn, invalidate_constant_state_bump, invalidate_constant_ic_fill, + invalidate_no_singleton_class, // Currently, it's out of the ordinary (might be impossible) for YJIT to leave gaps in // executable memory, so this should be 0. From 9cf754b648bc04f0c1e8f9274e6047ff25c1b3e3 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Fri, 22 Mar 2024 10:48:01 -0400 Subject: [PATCH 052/211] Fix --debug=gc_stress flag ruby_env_debug_option gets called after Init_gc_stress, so the --debug=gc_stress flag never works. --- debug.c | 9 ++++++--- gc.c | 27 ++++++--------------------- inits.c | 1 - internal/gc.h | 3 ++- 4 files changed, 14 insertions(+), 26 deletions(-) diff --git a/debug.c b/debug.c index 755f27531668ab..e194472c1aaf9d 100644 --- a/debug.c +++ b/debug.c @@ -185,9 +185,9 @@ ruby_env_debug_option(const char *str, int len, void *arg) int ov; size_t retlen; unsigned long n; +#define NAME_MATCH(name) (len == sizeof(name) - 1 && strncmp(str, (name), len) == 0) #define SET_WHEN(name, var, val) do { \ - if (len == sizeof(name) - 1 && \ - strncmp(str, (name), len) == 0) { \ + if (NAME_MATCH(name)) { \ (var) = (val); \ return 1; \ } \ @@ -221,7 +221,10 @@ ruby_env_debug_option(const char *str, int len, void *arg) #define SET_WHEN_UINT(name, vals, num, req) \ if (NAME_MATCH_VALUE(name)) SET_UINT_LIST(name, vals, num); - SET_WHEN("gc_stress", *ruby_initial_gc_stress_ptr, Qtrue); + if (NAME_MATCH("gc_stress")) { + rb_gc_stress_set(Qtrue); + return 1; + } SET_WHEN("core", ruby_enable_coredump, 1); SET_WHEN("ci", ruby_on_ci, 1); if (NAME_MATCH_VALUE("rgengc")) { diff --git a/gc.c b/gc.c index 282e9df0cfd653..4b30bb171d6fd3 100644 --- a/gc.c +++ b/gc.c @@ -439,8 +439,6 @@ typedef struct { size_t oldmalloc_limit_min; size_t oldmalloc_limit_max; double oldmalloc_limit_growth_factor; - - VALUE gc_stress; } ruby_gc_params_t; static ruby_gc_params_t gc_params = { @@ -462,8 +460,6 @@ static ruby_gc_params_t gc_params = { GC_OLDMALLOC_LIMIT_MIN, GC_OLDMALLOC_LIMIT_MAX, GC_OLDMALLOC_LIMIT_GROWTH_FACTOR, - - FALSE, }; /* GC_DEBUG: @@ -1135,10 +1131,6 @@ RVALUE_AGE_SET(VALUE obj, int age) if (unless_objspace_vm) objspace = unless_objspace_vm->objspace; \ else /* return; or objspace will be warned uninitialized */ -#define ruby_initial_gc_stress gc_params.gc_stress - -VALUE *ruby_initial_gc_stress_ptr = &ruby_initial_gc_stress; - #define malloc_limit objspace->malloc_params.limit #define malloc_increase objspace->malloc_params.increase #define malloc_allocated_size objspace->malloc_params.allocated_size @@ -1389,7 +1381,6 @@ NO_SANITIZE("memory", static inline int is_pointer_to_heap(rb_objspace_t *objspa static size_t obj_memsize_of(VALUE obj, int use_all_types); static void gc_verify_internal_consistency(rb_objspace_t *objspace); -static void gc_stress_set(rb_objspace_t *objspace, VALUE flag); static VALUE gc_disable_no_rest(rb_objspace_t *); static double getrusage_time(void); @@ -3552,14 +3543,6 @@ Init_heap(void) finalizer_table = st_init_numtable(); } -void -Init_gc_stress(void) -{ - rb_objspace_t *objspace = &rb_objspace; - - gc_stress_set(objspace, ruby_initial_gc_stress); -} - typedef int each_obj_callback(void *, void *, size_t, void *); typedef int each_page_callback(struct heap_page *, void *); @@ -11163,9 +11146,11 @@ gc_stress_get(rb_execution_context_t *ec, VALUE self) return ruby_gc_stress_mode; } -static void -gc_stress_set(rb_objspace_t *objspace, VALUE flag) +void +rb_gc_stress_set(VALUE flag) { + rb_objspace_t *objspace = &rb_objspace; + objspace->flags.gc_stressful = RTEST(flag); objspace->gc_stress_mode = flag; } @@ -11173,8 +11158,8 @@ gc_stress_set(rb_objspace_t *objspace, VALUE flag) static VALUE gc_stress_set_m(rb_execution_context_t *ec, VALUE self, VALUE flag) { - rb_objspace_t *objspace = &rb_objspace; - gc_stress_set(objspace, flag); + + rb_gc_stress_set(flag); return flag; } diff --git a/inits.c b/inits.c index 9ed104f369e02f..677a384f9a37c5 100644 --- a/inits.c +++ b/inits.c @@ -73,7 +73,6 @@ rb_call_inits(void) CALL(vm_trace); CALL(vm_stack_canary); CALL(ast); - CALL(gc_stress); CALL(shape); CALL(Prism); diff --git a/internal/gc.h b/internal/gc.h index f86af2659456b2..595dbb9ef6064f 100644 --- a/internal/gc.h +++ b/internal/gc.h @@ -189,7 +189,6 @@ typedef struct ractor_newobj_cache { } rb_ractor_newobj_cache_t; /* gc.c */ -extern VALUE *ruby_initial_gc_stress_ptr; extern int ruby_disable_gc; RUBY_ATTR_MALLOC void *ruby_mimmalloc(size_t size); void ruby_mimfree(void *ptr); @@ -224,6 +223,8 @@ void rb_gc_remove_weak(VALUE parent_obj, VALUE *ptr); void rb_gc_ref_update_table_values_only(st_table *tbl); +void rb_gc_stress_set(VALUE flag); + #define rb_gc_mark_and_move_ptr(ptr) do { \ VALUE _obj = (VALUE)*(ptr); \ rb_gc_mark_and_move(&_obj); \ From eef272f1548b92ca53867c94f7cc38b1f35ef656 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 25 Mar 2024 13:18:39 -0400 Subject: [PATCH 053/211] [ruby/prism] Mark inner parts of interpolated* nodes as frozen https://github.com/ruby/prism/commit/58a127cd5d --- prism/prism.c | 17 +++++++++++++++++ test/prism/snapshots/dos_endings.txt | 4 ++-- test/prism/snapshots/dstring.txt | 4 ++-- test/prism/snapshots/regex.txt | 8 ++++---- .../seattlerb/parse_line_evstr_after_break.txt | 2 +- .../snapshots/seattlerb/qsymbols_interp.txt | 2 +- .../seattlerb/str_lit_concat_bad_encodings.txt | 4 ++-- test/prism/snapshots/seattlerb/words_interp.txt | 2 +- test/prism/snapshots/spanning_heredoc.txt | 16 ++++++++-------- test/prism/snapshots/strings.txt | 8 ++++---- test/prism/snapshots/symbols.txt | 8 ++++---- .../unparser/corpus/literal/literal.txt | 14 +++++++------- .../snapshots/unparser/corpus/semantic/dstr.txt | 16 ++++++++-------- .../unparser/corpus/semantic/literal.txt | 2 +- .../whitequark/array_symbols_interp.txt | 2 +- .../snapshots/whitequark/array_words_interp.txt | 2 +- .../whitequark/non_lvar_injecting_match.txt | 2 +- .../prism/snapshots/whitequark/regex_interp.txt | 4 ++-- .../snapshots/whitequark/ruby_bug_11990.txt | 4 ++-- .../snapshots/whitequark/string_concat.txt | 2 +- .../snapshots/whitequark/xstring_interp.txt | 4 ++-- test/prism/snapshots/xstring.txt | 4 ++-- 22 files changed, 74 insertions(+), 57 deletions(-) diff --git a/prism/prism.c b/prism/prism.c index 6f2ab81e9ae3f3..f1f7fc052d059b 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -4302,6 +4302,11 @@ pm_interpolated_regular_expression_node_append(pm_interpolated_regular_expressio if (node->base.location.end < part->location.end) { node->base.location.end = part->location.end; } + + if (PM_NODE_TYPE_P(part, PM_STRING_NODE)) { + pm_node_flag_set(part, PM_NODE_FLAG_STATIC_LITERAL | PM_STRING_FLAGS_FROZEN); + } + pm_node_list_append(&node->parts, part); } @@ -4348,6 +4353,10 @@ pm_interpolated_string_node_append(pm_interpolated_string_node_t *node, pm_node_ node->base.location.start = part->location.start; } + if (PM_NODE_TYPE_P(part, PM_STRING_NODE)) { + pm_node_flag_set(part, PM_NODE_FLAG_STATIC_LITERAL | PM_STRING_FLAGS_FROZEN); + } + pm_node_list_append(&node->parts, part); node->base.location.end = part->location.end; } @@ -4394,6 +4403,10 @@ pm_interpolated_symbol_node_append(pm_interpolated_symbol_node_t *node, pm_node_ node->base.location.start = part->location.start; } + if (PM_NODE_TYPE_P(part, PM_STRING_NODE)) { + pm_node_flag_set(part, PM_NODE_FLAG_STATIC_LITERAL | PM_STRING_FLAGS_FROZEN); + } + pm_node_list_append(&node->parts, part); node->base.location.end = part->location.end; } @@ -4423,6 +4436,10 @@ pm_interpolated_xstring_node_create(pm_parser_t *parser, const pm_token_t *openi static inline void pm_interpolated_xstring_node_append(pm_interpolated_x_string_node_t *node, pm_node_t *part) { + if (PM_NODE_TYPE_P(part, PM_STRING_NODE)) { + pm_node_flag_set(part, PM_NODE_FLAG_STATIC_LITERAL | PM_STRING_FLAGS_FROZEN); + } + pm_node_list_append(&node->parts, part); node->base.location.end = part->location.end; } diff --git a/test/prism/snapshots/dos_endings.txt b/test/prism/snapshots/dos_endings.txt index c5b962f2181cf3..5ac946b207e135 100644 --- a/test/prism/snapshots/dos_endings.txt +++ b/test/prism/snapshots/dos_endings.txt @@ -18,13 +18,13 @@ │ │ ├── opening_loc: ∅ │ │ ├── parts: (length: 2) │ │ │ ├── @ StringNode (location: (1,5)-(1,9)) - │ │ │ │ ├── flags: ∅ + │ │ │ │ ├── flags: frozen │ │ │ │ ├── opening_loc: (1,5)-(1,6) = "\"" │ │ │ │ ├── content_loc: (1,6)-(1,8) = "hi" │ │ │ │ ├── closing_loc: (1,8)-(1,9) = "\"" │ │ │ │ └── unescaped: "hi" │ │ │ └── @ StringNode (location: (2,5)-(2,12)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: (2,5)-(2,6) = "\"" │ │ │ ├── content_loc: (2,6)-(2,11) = "there" │ │ │ ├── closing_loc: (2,11)-(2,12) = "\"" diff --git a/test/prism/snapshots/dstring.txt b/test/prism/snapshots/dstring.txt index ad395f8a8eb94b..64ab6e8a823bbe 100644 --- a/test/prism/snapshots/dstring.txt +++ b/test/prism/snapshots/dstring.txt @@ -39,13 +39,13 @@ │ ├── opening_loc: ∅ │ ├── parts: (length: 2) │ │ ├── @ StringNode (location: (7,0)-(8,2)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: (7,0)-(7,1) = "\"" │ │ │ ├── content_loc: (7,1)-(8,1) = "fo\no" │ │ │ ├── closing_loc: (8,1)-(8,2) = "\"" │ │ │ └── unescaped: "fo\no" │ │ └── @ StringNode (location: (8,3)-(9,2)) - │ │ ├── flags: ∅ + │ │ ├── flags: frozen │ │ ├── opening_loc: (8,3)-(8,4) = "\"" │ │ ├── content_loc: (8,4)-(9,1) = "ba\nr" │ │ ├── closing_loc: (9,1)-(9,2) = "\"" diff --git a/test/prism/snapshots/regex.txt b/test/prism/snapshots/regex.txt index 9e19bbb18d273c..d07ab8c5e71900 100644 --- a/test/prism/snapshots/regex.txt +++ b/test/prism/snapshots/regex.txt @@ -39,7 +39,7 @@ │ ├── opening_loc: (7,0)-(7,1) = "/" │ ├── parts: (length: 2) │ │ ├── @ StringNode (location: (7,1)-(7,5)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (7,1)-(7,5) = "aaa " │ │ │ ├── closing_loc: ∅ @@ -55,7 +55,7 @@ │ ├── opening_loc: (9,0)-(9,1) = "/" │ ├── parts: (length: 3) │ │ ├── @ StringNode (location: (9,1)-(9,5)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (9,1)-(9,5) = "aaa " │ │ │ ├── closing_loc: ∅ @@ -77,7 +77,7 @@ │ │ │ │ └── block: ∅ │ │ │ └── closing_loc: (9,10)-(9,11) = "}" │ │ └── @ StringNode (location: (9,11)-(9,15)) - │ │ ├── flags: ∅ + │ │ ├── flags: frozen │ │ ├── opening_loc: ∅ │ │ ├── content_loc: (9,11)-(9,15) = " ccc" │ │ ├── closing_loc: ∅ @@ -192,7 +192,7 @@ │ ├── opening_loc: (30,0)-(30,1) = "/" │ ├── parts: (length: 2) │ │ ├── @ StringNode (location: (30,1)-(30,5)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (30,1)-(30,5) = "aaa " │ │ │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/seattlerb/parse_line_evstr_after_break.txt b/test/prism/snapshots/seattlerb/parse_line_evstr_after_break.txt index 9cb200e94bce5b..d8acbc05c49f07 100644 --- a/test/prism/snapshots/seattlerb/parse_line_evstr_after_break.txt +++ b/test/prism/snapshots/seattlerb/parse_line_evstr_after_break.txt @@ -7,7 +7,7 @@ ├── opening_loc: ∅ ├── parts: (length: 2) │ ├── @ StringNode (location: (1,0)-(1,3)) - │ │ ├── flags: ∅ + │ │ ├── flags: frozen │ │ ├── opening_loc: (1,0)-(1,1) = "\"" │ │ ├── content_loc: (1,1)-(1,2) = "a" │ │ ├── closing_loc: (1,2)-(1,3) = "\"" diff --git a/test/prism/snapshots/seattlerb/qsymbols_interp.txt b/test/prism/snapshots/seattlerb/qsymbols_interp.txt index 23c39302475630..97bc6754ff5088 100644 --- a/test/prism/snapshots/seattlerb/qsymbols_interp.txt +++ b/test/prism/snapshots/seattlerb/qsymbols_interp.txt @@ -16,7 +16,7 @@ │ │ ├── opening_loc: ∅ │ │ ├── parts: (length: 2) │ │ │ ├── @ StringNode (location: (1,5)-(1,6)) - │ │ │ │ ├── flags: ∅ + │ │ │ │ ├── flags: frozen │ │ │ │ ├── opening_loc: ∅ │ │ │ │ ├── content_loc: (1,5)-(1,6) = "b" │ │ │ │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/seattlerb/str_lit_concat_bad_encodings.txt b/test/prism/snapshots/seattlerb/str_lit_concat_bad_encodings.txt index b841407cd8df2a..b31245a7f4f664 100644 --- a/test/prism/snapshots/seattlerb/str_lit_concat_bad_encodings.txt +++ b/test/prism/snapshots/seattlerb/str_lit_concat_bad_encodings.txt @@ -7,13 +7,13 @@ ├── opening_loc: ∅ ├── parts: (length: 2) │ ├── @ StringNode (location: (1,0)-(1,62)) - │ │ ├── flags: forced_utf8_encoding + │ │ ├── flags: forced_utf8_encoding, frozen │ │ ├── opening_loc: (1,0)-(1,1) = "\"" │ │ ├── content_loc: (1,1)-(1,61) = "\\xE3\\xD3\\x8B\\xE3\\x83\\xBC\\x83\\xE3\\x83\\xE3\\x82\\xB3\\xA3\\x82\\x99" │ │ ├── closing_loc: (1,61)-(1,62) = "\"" │ │ └── unescaped: "\xE3Ӌー\x83\xE3\x83コ\xA3\x82\x99" │ └── @ StringNode (location: (2,8)-(2,66)) - │ ├── flags: forced_utf8_encoding + │ ├── flags: forced_utf8_encoding, frozen │ ├── opening_loc: (2,8)-(2,9) = "\"" │ ├── content_loc: (2,9)-(2,65) = "\\xE3\\x83\\xB3\\xE3\\x83\\x8F\\xE3\\x82\\x9A\\xC3\\xBD;foo@bar.com" │ ├── closing_loc: (2,65)-(2,66) = "\"" diff --git a/test/prism/snapshots/seattlerb/words_interp.txt b/test/prism/snapshots/seattlerb/words_interp.txt index dfead7d353404b..e0b08bc075639a 100644 --- a/test/prism/snapshots/seattlerb/words_interp.txt +++ b/test/prism/snapshots/seattlerb/words_interp.txt @@ -19,7 +19,7 @@ │ │ │ │ └── value: 1 │ │ │ └── closing_loc: (1,6)-(1,7) = "}" │ │ └── @ StringNode (location: (1,7)-(1,8)) - │ │ ├── flags: ∅ + │ │ ├── flags: frozen │ │ ├── opening_loc: ∅ │ │ ├── content_loc: (1,7)-(1,8) = "b" │ │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/spanning_heredoc.txt b/test/prism/snapshots/spanning_heredoc.txt index 90297d2282c753..7c3db913333712 100644 --- a/test/prism/snapshots/spanning_heredoc.txt +++ b/test/prism/snapshots/spanning_heredoc.txt @@ -36,13 +36,13 @@ │ │ │ │ ├── opening_loc: (4,13)-(4,14) = "/" │ │ │ │ ├── parts: (length: 2) │ │ │ │ │ ├── @ StringNode (location: (4,14)-(4,16)) - │ │ │ │ │ │ ├── flags: ∅ + │ │ │ │ │ │ ├── flags: frozen │ │ │ │ │ │ ├── opening_loc: ∅ │ │ │ │ │ │ ├── content_loc: (4,14)-(4,16) = "b\\" │ │ │ │ │ │ ├── closing_loc: ∅ │ │ │ │ │ │ └── unescaped: "b" │ │ │ │ │ └── @ StringNode (location: (7,0)-(7,1)) - │ │ │ │ │ ├── flags: ∅ + │ │ │ │ │ ├── flags: frozen │ │ │ │ │ ├── opening_loc: ∅ │ │ │ │ │ ├── content_loc: (7,0)-(7,1) = "b" │ │ │ │ │ ├── closing_loc: ∅ @@ -223,13 +223,13 @@ │ │ │ ├── opening_loc: ∅ │ │ │ ├── parts: (length: 2) │ │ │ │ ├── @ StringNode (location: (35,12)-(35,14)) - │ │ │ │ │ ├── flags: ∅ + │ │ │ │ │ ├── flags: frozen │ │ │ │ │ ├── opening_loc: ∅ │ │ │ │ │ ├── content_loc: (35,12)-(35,14) = "l\\" │ │ │ │ │ ├── closing_loc: ∅ │ │ │ │ │ └── unescaped: "l\n" │ │ │ │ └── @ StringNode (location: (38,0)-(38,1)) - │ │ │ │ ├── flags: ∅ + │ │ │ │ ├── flags: frozen │ │ │ │ ├── opening_loc: ∅ │ │ │ │ ├── content_loc: (38,0)-(38,1) = "l" │ │ │ │ ├── closing_loc: ∅ @@ -299,13 +299,13 @@ │ │ │ ├── opening_loc: ∅ │ │ │ ├── parts: (length: 2) │ │ │ │ ├── @ StringNode (location: (48,12)-(48,14)) - │ │ │ │ │ ├── flags: ∅ + │ │ │ │ │ ├── flags: frozen │ │ │ │ │ ├── opening_loc: ∅ │ │ │ │ │ ├── content_loc: (48,12)-(48,14) = "p\\" │ │ │ │ │ ├── closing_loc: ∅ │ │ │ │ │ └── unescaped: "p\n" │ │ │ │ └── @ StringNode (location: (48,12)-(48,14)) - │ │ │ │ ├── flags: ∅ + │ │ │ │ ├── flags: frozen │ │ │ │ ├── opening_loc: ∅ │ │ │ │ ├── content_loc: (48,12)-(48,14) = "p\\" │ │ │ │ ├── closing_loc: ∅ @@ -331,13 +331,13 @@ │ │ │ ├── opening_loc: (53,5)-(53,6) = "/" │ │ │ ├── parts: (length: 2) │ │ │ │ ├── @ StringNode (location: (53,6)-(53,7)) - │ │ │ │ │ ├── flags: ∅ + │ │ │ │ │ ├── flags: frozen │ │ │ │ │ ├── opening_loc: ∅ │ │ │ │ │ ├── content_loc: (53,6)-(53,7) = "\\" │ │ │ │ │ ├── closing_loc: ∅ │ │ │ │ │ └── unescaped: "" │ │ │ │ └── @ StringNode (location: (55,0)-(55,6)) - │ │ │ │ ├── flags: ∅ + │ │ │ │ ├── flags: frozen │ │ │ │ ├── opening_loc: ∅ │ │ │ │ ├── content_loc: (55,0)-(55,6) = "(?)" │ │ │ │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/strings.txt b/test/prism/snapshots/strings.txt index 40a05ce09d5bbf..471b371332c957 100644 --- a/test/prism/snapshots/strings.txt +++ b/test/prism/snapshots/strings.txt @@ -337,7 +337,7 @@ │ │ │ ├── opening_loc: ∅ │ │ │ ├── parts: (length: 3) │ │ │ │ ├── @ StringNode (location: (67,5)-(67,6)) - │ │ │ │ │ ├── flags: ∅ + │ │ │ │ │ ├── flags: frozen │ │ │ │ │ ├── opening_loc: ∅ │ │ │ │ │ ├── content_loc: (67,5)-(67,6) = "b" │ │ │ │ │ ├── closing_loc: ∅ @@ -359,7 +359,7 @@ │ │ │ │ │ │ └── block: ∅ │ │ │ │ │ └── closing_loc: (67,9)-(67,10) = "}" │ │ │ │ └── @ StringNode (location: (67,10)-(67,11)) - │ │ │ │ ├── flags: ∅ + │ │ │ │ ├── flags: frozen │ │ │ │ ├── opening_loc: ∅ │ │ │ │ ├── content_loc: (67,10)-(67,11) = "d" │ │ │ │ ├── closing_loc: ∅ @@ -495,13 +495,13 @@ │ ├── opening_loc: ∅ │ ├── parts: (length: 2) │ │ ├── @ StringNode (location: (99,0)-(99,2)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: (99,0)-(99,1) = "?" │ │ │ ├── content_loc: (99,1)-(99,2) = "a" │ │ │ ├── closing_loc: ∅ │ │ │ └── unescaped: "a" │ │ └── @ StringNode (location: (99,3)-(99,6)) - │ │ ├── flags: ∅ + │ │ ├── flags: frozen │ │ ├── opening_loc: (99,3)-(99,4) = "\"" │ │ ├── content_loc: (99,4)-(99,5) = "a" │ │ ├── closing_loc: (99,5)-(99,6) = "\"" diff --git a/test/prism/snapshots/symbols.txt b/test/prism/snapshots/symbols.txt index c34be74b37d506..b6a854db0b2069 100644 --- a/test/prism/snapshots/symbols.txt +++ b/test/prism/snapshots/symbols.txt @@ -234,7 +234,7 @@ │ │ │ ├── opening_loc: ∅ │ │ │ ├── parts: (length: 2) │ │ │ │ ├── @ StringNode (location: (39,5)-(39,6)) - │ │ │ │ │ ├── flags: ∅ + │ │ │ │ │ ├── flags: frozen │ │ │ │ │ ├── opening_loc: ∅ │ │ │ │ │ ├── content_loc: (39,5)-(39,6) = "b" │ │ │ │ │ ├── closing_loc: ∅ @@ -262,7 +262,7 @@ │ │ │ │ │ │ └── value: 2 │ │ │ │ │ └── closing_loc: (39,14)-(39,15) = "}" │ │ │ │ └── @ StringNode (location: (39,15)-(39,16)) - │ │ │ │ ├── flags: ∅ + │ │ │ │ ├── flags: frozen │ │ │ │ ├── opening_loc: ∅ │ │ │ │ ├── content_loc: (39,15)-(39,16) = "c" │ │ │ │ ├── closing_loc: ∅ @@ -272,7 +272,7 @@ │ │ ├── opening_loc: ∅ │ │ ├── parts: (length: 3) │ │ │ ├── @ StringNode (location: (39,17)-(39,18)) - │ │ │ │ ├── flags: ∅ + │ │ │ │ ├── flags: frozen │ │ │ │ ├── opening_loc: ∅ │ │ │ │ ├── content_loc: (39,17)-(39,18) = "d" │ │ │ │ ├── closing_loc: ∅ @@ -287,7 +287,7 @@ │ │ │ │ │ └── value: 3 │ │ │ │ └── closing_loc: (39,21)-(39,22) = "}" │ │ │ └── @ StringNode (location: (39,22)-(39,23)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (39,22)-(39,23) = "f" │ │ │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/unparser/corpus/literal/literal.txt b/test/prism/snapshots/unparser/corpus/literal/literal.txt index ba7dd70b5b717c..8ecf613e7ce9ad 100644 --- a/test/prism/snapshots/unparser/corpus/literal/literal.txt +++ b/test/prism/snapshots/unparser/corpus/literal/literal.txt @@ -358,13 +358,13 @@ │ ├── opening_loc: ∅ │ ├── parts: (length: 2) │ │ ├── @ StringNode (location: (28,0)-(28,5)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: (28,0)-(28,1) = "\"" │ │ │ ├── content_loc: (28,1)-(28,4) = "foo" │ │ │ ├── closing_loc: (28,4)-(28,5) = "\"" │ │ │ └── unescaped: "foo" │ │ └── @ StringNode (location: (28,6)-(28,11)) - │ │ ├── flags: ∅ + │ │ ├── flags: frozen │ │ ├── opening_loc: (28,6)-(28,7) = "\"" │ │ ├── content_loc: (28,7)-(28,10) = "bar" │ │ ├── closing_loc: (28,10)-(28,11) = "\"" @@ -497,7 +497,7 @@ │ ├── opening_loc: (39,0)-(39,1) = "`" │ ├── parts: (length: 2) │ │ ├── @ StringNode (location: (39,1)-(39,4)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (39,1)-(39,4) = "foo" │ │ │ ├── closing_loc: ∅ @@ -582,7 +582,7 @@ │ ├── opening_loc: (51,0)-(51,1) = "/" │ ├── parts: (length: 2) │ │ ├── @ StringNode (location: (51,1)-(51,4)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (51,1)-(51,4) = "foo" │ │ │ ├── closing_loc: ∅ @@ -601,7 +601,7 @@ │ ├── opening_loc: (52,0)-(52,1) = "/" │ ├── parts: (length: 2) │ │ ├── @ StringNode (location: (52,1)-(52,4)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (52,1)-(52,4) = "foo" │ │ │ ├── closing_loc: ∅ @@ -1156,7 +1156,7 @@ ├── opening_loc: (89,0)-(89,1) = "`" ├── parts: (length: 3) │ ├── @ StringNode (location: (89,1)-(90,0)) - │ │ ├── flags: ∅ + │ │ ├── flags: frozen │ │ ├── opening_loc: ∅ │ │ ├── content_loc: (89,1)-(90,0) = " x\n" │ │ ├── closing_loc: ∅ @@ -1178,7 +1178,7 @@ │ │ │ └── block: ∅ │ │ └── closing_loc: (90,5)-(90,6) = "}" │ └── @ StringNode (location: (90,6)-(91,1)) - │ ├── flags: ∅ + │ ├── flags: frozen │ ├── opening_loc: ∅ │ ├── content_loc: (90,6)-(91,1) = "\n#" │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/unparser/corpus/semantic/dstr.txt b/test/prism/snapshots/unparser/corpus/semantic/dstr.txt index 5ab954b6d4119d..c5337729692b09 100644 --- a/test/prism/snapshots/unparser/corpus/semantic/dstr.txt +++ b/test/prism/snapshots/unparser/corpus/semantic/dstr.txt @@ -429,7 +429,7 @@ │ ├── opening_loc: ∅ │ ├── parts: (length: 2) │ │ ├── @ StringNode (location: (119,0)-(119,3)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: (119,0)-(119,1) = "'" │ │ │ ├── content_loc: (119,1)-(119,2) = "a" │ │ │ ├── closing_loc: (119,2)-(119,3) = "'" @@ -447,19 +447,19 @@ │ ├── opening_loc: ∅ │ ├── parts: (length: 3) │ │ ├── @ StringNode (location: (122,0)-(122,2)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: (122,0)-(122,1) = "\"" │ │ │ ├── content_loc: (122,1)-(122,1) = "" │ │ │ ├── closing_loc: (122,1)-(122,2) = "\"" │ │ │ └── unescaped: "" │ │ ├── @ StringNode (location: (122,3)-(122,5)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: (122,3)-(122,4) = "\"" │ │ │ ├── content_loc: (122,4)-(122,4) = "" │ │ │ ├── closing_loc: (122,4)-(122,5) = "\"" │ │ │ └── unescaped: "" │ │ └── @ StringNode (location: (122,6)-(122,8)) - │ │ ├── flags: ∅ + │ │ ├── flags: frozen │ │ ├── opening_loc: (122,6)-(122,7) = "\"" │ │ ├── content_loc: (122,7)-(122,7) = "" │ │ ├── closing_loc: (122,7)-(122,8) = "\"" @@ -487,7 +487,7 @@ │ │ │ │ └── closing_loc: (124,6)-(124,7) = "}" │ │ │ └── closing_loc: (124,7)-(124,8) = "\"" │ │ └── @ StringNode (location: (124,9)-(124,12)) - │ │ ├── flags: ∅ + │ │ ├── flags: frozen │ │ ├── opening_loc: (124,9)-(124,10) = "\"" │ │ ├── content_loc: (124,10)-(124,11) = "b" │ │ ├── closing_loc: (124,11)-(124,12) = "\"" @@ -512,7 +512,7 @@ │ │ │ │ └── name: :@a │ │ │ └── closing_loc: (125,5)-(125,6) = "\"" │ │ └── @ StringNode (location: (125,7)-(125,10)) - │ │ ├── flags: ∅ + │ │ ├── flags: frozen │ │ ├── opening_loc: (125,7)-(125,8) = "\"" │ │ ├── content_loc: (125,8)-(125,9) = "b" │ │ ├── closing_loc: (125,9)-(125,10) = "\"" @@ -537,7 +537,7 @@ │ │ │ │ └── name: :$a │ │ │ └── closing_loc: (126,5)-(126,6) = "\"" │ │ └── @ StringNode (location: (126,7)-(126,10)) - │ │ ├── flags: ∅ + │ │ ├── flags: frozen │ │ ├── opening_loc: (126,7)-(126,8) = "\"" │ │ ├── content_loc: (126,8)-(126,9) = "b" │ │ ├── closing_loc: (126,9)-(126,10) = "\"" @@ -562,7 +562,7 @@ │ │ │ └── name: :@@a │ │ └── closing_loc: (127,6)-(127,7) = "\"" │ └── @ StringNode (location: (127,8)-(127,11)) - │ ├── flags: ∅ + │ ├── flags: frozen │ ├── opening_loc: (127,8)-(127,9) = "\"" │ ├── content_loc: (127,9)-(127,10) = "b" │ ├── closing_loc: (127,10)-(127,11) = "\"" diff --git a/test/prism/snapshots/unparser/corpus/semantic/literal.txt b/test/prism/snapshots/unparser/corpus/semantic/literal.txt index 59e02be64fcfd2..ef666890be7d34 100644 --- a/test/prism/snapshots/unparser/corpus/semantic/literal.txt +++ b/test/prism/snapshots/unparser/corpus/semantic/literal.txt @@ -55,7 +55,7 @@ │ │ │ │ └── name: :@bar │ │ │ └── closing_loc: (11,9)-(11,10) = "}" │ │ └── @ StringNode (location: (11,10)-(11,13)) - │ │ ├── flags: ∅ + │ │ ├── flags: frozen │ │ ├── opening_loc: ∅ │ │ ├── content_loc: (11,10)-(11,13) = "baz" │ │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/whitequark/array_symbols_interp.txt b/test/prism/snapshots/whitequark/array_symbols_interp.txt index 7583ad8833e17d..2437486b43e250 100644 --- a/test/prism/snapshots/whitequark/array_symbols_interp.txt +++ b/test/prism/snapshots/whitequark/array_symbols_interp.txt @@ -41,7 +41,7 @@ │ ├── opening_loc: ∅ │ ├── parts: (length: 2) │ │ ├── @ StringNode (location: (3,3)-(3,6)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (3,3)-(3,6) = "foo" │ │ │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/whitequark/array_words_interp.txt b/test/prism/snapshots/whitequark/array_words_interp.txt index c2f23d2bf19d78..00fd05e7f7befc 100644 --- a/test/prism/snapshots/whitequark/array_words_interp.txt +++ b/test/prism/snapshots/whitequark/array_words_interp.txt @@ -63,7 +63,7 @@ │ │ │ │ └── block: ∅ │ │ │ └── closing_loc: (3,12)-(3,13) = "}" │ │ ├── @ StringNode (location: (3,13)-(3,16)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (3,13)-(3,16) = "foo" │ │ │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/whitequark/non_lvar_injecting_match.txt b/test/prism/snapshots/whitequark/non_lvar_injecting_match.txt index 2df571e8dc1890..584e997df2c6da 100644 --- a/test/prism/snapshots/whitequark/non_lvar_injecting_match.txt +++ b/test/prism/snapshots/whitequark/non_lvar_injecting_match.txt @@ -20,7 +20,7 @@ │ │ │ │ └── value: 1 │ │ │ └── closing_loc: (1,4)-(1,5) = "}" │ │ └── @ StringNode (location: (1,5)-(1,18)) - │ │ ├── flags: ∅ + │ │ ├── flags: frozen │ │ ├── opening_loc: ∅ │ │ ├── content_loc: (1,5)-(1,18) = "(?bar)" │ │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/whitequark/regex_interp.txt b/test/prism/snapshots/whitequark/regex_interp.txt index ee8caf22b9ee55..0a6db4cfdf4b7b 100644 --- a/test/prism/snapshots/whitequark/regex_interp.txt +++ b/test/prism/snapshots/whitequark/regex_interp.txt @@ -8,7 +8,7 @@ ├── opening_loc: (1,0)-(1,1) = "/" ├── parts: (length: 3) │ ├── @ StringNode (location: (1,1)-(1,4)) - │ │ ├── flags: ∅ + │ │ ├── flags: frozen │ │ ├── opening_loc: ∅ │ │ ├── content_loc: (1,1)-(1,4) = "foo" │ │ ├── closing_loc: ∅ @@ -30,7 +30,7 @@ │ │ │ └── block: ∅ │ │ └── closing_loc: (1,9)-(1,10) = "}" │ └── @ StringNode (location: (1,10)-(1,13)) - │ ├── flags: ∅ + │ ├── flags: frozen │ ├── opening_loc: ∅ │ ├── content_loc: (1,10)-(1,13) = "baz" │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/whitequark/ruby_bug_11990.txt b/test/prism/snapshots/whitequark/ruby_bug_11990.txt index 43c04e55a23a15..abdb8d9ddd6ca6 100644 --- a/test/prism/snapshots/whitequark/ruby_bug_11990.txt +++ b/test/prism/snapshots/whitequark/ruby_bug_11990.txt @@ -18,13 +18,13 @@ │ ├── opening_loc: ∅ │ ├── parts: (length: 2) │ │ ├── @ StringNode (location: (1,2)-(1,6)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: (1,2)-(1,6) = "<<~E" │ │ │ ├── content_loc: (2,0)-(3,0) = " x\n" │ │ │ ├── closing_loc: (3,0)-(4,0) = "E\n" │ │ │ └── unescaped: "x\n" │ │ └── @ StringNode (location: (1,7)-(1,12)) - │ │ ├── flags: ∅ + │ │ ├── flags: frozen │ │ ├── opening_loc: (1,7)-(1,8) = "\"" │ │ ├── content_loc: (1,8)-(1,11) = " y" │ │ ├── closing_loc: (1,11)-(1,12) = "\"" diff --git a/test/prism/snapshots/whitequark/string_concat.txt b/test/prism/snapshots/whitequark/string_concat.txt index 38ea0c745dd664..1739b2f65bfed4 100644 --- a/test/prism/snapshots/whitequark/string_concat.txt +++ b/test/prism/snapshots/whitequark/string_concat.txt @@ -22,7 +22,7 @@ │ │ │ └── name: :@a │ │ └── closing_loc: (1,7)-(1,8) = "\"" │ └── @ StringNode (location: (1,9)-(1,14)) - │ ├── flags: ∅ + │ ├── flags: frozen │ ├── opening_loc: (1,9)-(1,10) = "\"" │ ├── content_loc: (1,10)-(1,13) = "bar" │ ├── closing_loc: (1,13)-(1,14) = "\"" diff --git a/test/prism/snapshots/whitequark/xstring_interp.txt b/test/prism/snapshots/whitequark/xstring_interp.txt index e9543098c2b232..0676ea96837f72 100644 --- a/test/prism/snapshots/whitequark/xstring_interp.txt +++ b/test/prism/snapshots/whitequark/xstring_interp.txt @@ -7,7 +7,7 @@ ├── opening_loc: (1,0)-(1,1) = "`" ├── parts: (length: 3) │ ├── @ StringNode (location: (1,1)-(1,4)) - │ │ ├── flags: ∅ + │ │ ├── flags: frozen │ │ ├── opening_loc: ∅ │ │ ├── content_loc: (1,1)-(1,4) = "foo" │ │ ├── closing_loc: ∅ @@ -29,7 +29,7 @@ │ │ │ └── block: ∅ │ │ └── closing_loc: (1,9)-(1,10) = "}" │ └── @ StringNode (location: (1,10)-(1,13)) - │ ├── flags: ∅ + │ ├── flags: frozen │ ├── opening_loc: ∅ │ ├── content_loc: (1,10)-(1,13) = "baz" │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/xstring.txt b/test/prism/snapshots/xstring.txt index 04b4cbf6ea7151..1a177026db6bc4 100644 --- a/test/prism/snapshots/xstring.txt +++ b/test/prism/snapshots/xstring.txt @@ -13,7 +13,7 @@ │ ├── opening_loc: (3,0)-(3,1) = "`" │ ├── parts: (length: 3) │ │ ├── @ StringNode (location: (3,1)-(3,5)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (3,1)-(3,5) = "foo " │ │ │ ├── closing_loc: ∅ @@ -35,7 +35,7 @@ │ │ │ │ └── block: ∅ │ │ │ └── closing_loc: (3,10)-(3,11) = "}" │ │ └── @ StringNode (location: (3,11)-(3,15)) - │ │ ├── flags: ∅ + │ │ ├── flags: frozen │ │ ├── opening_loc: ∅ │ │ ├── content_loc: (3,11)-(3,15) = " baz" │ │ ├── closing_loc: ∅ From ff8f98f5c26febea9ca61278f167ad6582381855 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 25 Mar 2024 13:21:13 -0400 Subject: [PATCH 054/211] [ruby/prism] Mark interpolated nodes as static literal https://github.com/ruby/prism/commit/d00977a9bd --- prism/prism.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/prism/prism.c b/prism/prism.c index f1f7fc052d059b..223a0a92dede62 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -4281,6 +4281,7 @@ pm_interpolated_regular_expression_node_create(pm_parser_t *parser, const pm_tok *node = (pm_interpolated_regular_expression_node_t) { { .type = PM_INTERPOLATED_REGULAR_EXPRESSION_NODE, + .flags = PM_NODE_FLAG_STATIC_LITERAL, .location = { .start = opening->start, .end = NULL, @@ -4307,6 +4308,10 @@ pm_interpolated_regular_expression_node_append(pm_interpolated_regular_expressio pm_node_flag_set(part, PM_NODE_FLAG_STATIC_LITERAL | PM_STRING_FLAGS_FROZEN); } + if (!PM_NODE_FLAG_P(part, PM_NODE_FLAG_STATIC_LITERAL)) { + pm_node_flag_unset((pm_node_t *) node, PM_NODE_FLAG_STATIC_LITERAL); + } + pm_node_list_append(&node->parts, part); } @@ -4327,6 +4332,7 @@ pm_interpolated_string_node_create(pm_parser_t *parser, const pm_token_t *openin *node = (pm_interpolated_string_node_t) { { .type = PM_INTERPOLATED_STRING_NODE, + .flags = PM_NODE_FLAG_STATIC_LITERAL, .location = { .start = opening->start, .end = closing->end, @@ -4357,6 +4363,10 @@ pm_interpolated_string_node_append(pm_interpolated_string_node_t *node, pm_node_ pm_node_flag_set(part, PM_NODE_FLAG_STATIC_LITERAL | PM_STRING_FLAGS_FROZEN); } + if (!PM_NODE_FLAG_P(part, PM_NODE_FLAG_STATIC_LITERAL)) { + pm_node_flag_unset((pm_node_t *) node, PM_NODE_FLAG_STATIC_LITERAL); + } + pm_node_list_append(&node->parts, part); node->base.location.end = part->location.end; } @@ -4380,6 +4390,7 @@ pm_interpolated_symbol_node_create(pm_parser_t *parser, const pm_token_t *openin *node = (pm_interpolated_symbol_node_t) { { .type = PM_INTERPOLATED_SYMBOL_NODE, + .flags = PM_NODE_FLAG_STATIC_LITERAL, .location = { .start = opening->start, .end = closing->end, @@ -4407,6 +4418,10 @@ pm_interpolated_symbol_node_append(pm_interpolated_symbol_node_t *node, pm_node_ pm_node_flag_set(part, PM_NODE_FLAG_STATIC_LITERAL | PM_STRING_FLAGS_FROZEN); } + if (!PM_NODE_FLAG_P(part, PM_NODE_FLAG_STATIC_LITERAL)) { + pm_node_flag_unset((pm_node_t *) node, PM_NODE_FLAG_STATIC_LITERAL); + } + pm_node_list_append(&node->parts, part); node->base.location.end = part->location.end; } @@ -4421,6 +4436,7 @@ pm_interpolated_xstring_node_create(pm_parser_t *parser, const pm_token_t *openi *node = (pm_interpolated_x_string_node_t) { { .type = PM_INTERPOLATED_X_STRING_NODE, + .flags = PM_NODE_FLAG_STATIC_LITERAL, .location = { .start = opening->start, .end = closing->end @@ -4440,6 +4456,10 @@ pm_interpolated_xstring_node_append(pm_interpolated_x_string_node_t *node, pm_no pm_node_flag_set(part, PM_NODE_FLAG_STATIC_LITERAL | PM_STRING_FLAGS_FROZEN); } + if (!PM_NODE_FLAG_P(part, PM_NODE_FLAG_STATIC_LITERAL)) { + pm_node_flag_unset((pm_node_t *) node, PM_NODE_FLAG_STATIC_LITERAL); + } + pm_node_list_append(&node->parts, part); node->base.location.end = part->location.end; } From 453de8c2bc1392be86058510630fab1a55c2a265 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 25 Mar 2024 15:08:21 -0400 Subject: [PATCH 055/211] [ruby/prism] Revert "Frozen parts" https://github.com/ruby/prism/commit/48f2e8c169 --- prism/prism.c | 37 ------------------- test/prism/snapshots/dos_endings.txt | 4 +- test/prism/snapshots/dstring.txt | 4 +- test/prism/snapshots/regex.txt | 8 ++-- .../parse_line_evstr_after_break.txt | 2 +- .../snapshots/seattlerb/qsymbols_interp.txt | 2 +- .../str_lit_concat_bad_encodings.txt | 4 +- .../snapshots/seattlerb/words_interp.txt | 2 +- test/prism/snapshots/spanning_heredoc.txt | 16 ++++---- test/prism/snapshots/strings.txt | 8 ++-- test/prism/snapshots/symbols.txt | 8 ++-- .../unparser/corpus/literal/literal.txt | 14 +++---- .../unparser/corpus/semantic/dstr.txt | 16 ++++---- .../unparser/corpus/semantic/literal.txt | 2 +- .../whitequark/array_symbols_interp.txt | 2 +- .../whitequark/array_words_interp.txt | 2 +- .../whitequark/non_lvar_injecting_match.txt | 2 +- .../snapshots/whitequark/regex_interp.txt | 4 +- .../snapshots/whitequark/ruby_bug_11990.txt | 4 +- .../snapshots/whitequark/string_concat.txt | 2 +- .../snapshots/whitequark/xstring_interp.txt | 4 +- test/prism/snapshots/xstring.txt | 4 +- 22 files changed, 57 insertions(+), 94 deletions(-) diff --git a/prism/prism.c b/prism/prism.c index 223a0a92dede62..6f2ab81e9ae3f3 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -4281,7 +4281,6 @@ pm_interpolated_regular_expression_node_create(pm_parser_t *parser, const pm_tok *node = (pm_interpolated_regular_expression_node_t) { { .type = PM_INTERPOLATED_REGULAR_EXPRESSION_NODE, - .flags = PM_NODE_FLAG_STATIC_LITERAL, .location = { .start = opening->start, .end = NULL, @@ -4303,15 +4302,6 @@ pm_interpolated_regular_expression_node_append(pm_interpolated_regular_expressio if (node->base.location.end < part->location.end) { node->base.location.end = part->location.end; } - - if (PM_NODE_TYPE_P(part, PM_STRING_NODE)) { - pm_node_flag_set(part, PM_NODE_FLAG_STATIC_LITERAL | PM_STRING_FLAGS_FROZEN); - } - - if (!PM_NODE_FLAG_P(part, PM_NODE_FLAG_STATIC_LITERAL)) { - pm_node_flag_unset((pm_node_t *) node, PM_NODE_FLAG_STATIC_LITERAL); - } - pm_node_list_append(&node->parts, part); } @@ -4332,7 +4322,6 @@ pm_interpolated_string_node_create(pm_parser_t *parser, const pm_token_t *openin *node = (pm_interpolated_string_node_t) { { .type = PM_INTERPOLATED_STRING_NODE, - .flags = PM_NODE_FLAG_STATIC_LITERAL, .location = { .start = opening->start, .end = closing->end, @@ -4359,14 +4348,6 @@ pm_interpolated_string_node_append(pm_interpolated_string_node_t *node, pm_node_ node->base.location.start = part->location.start; } - if (PM_NODE_TYPE_P(part, PM_STRING_NODE)) { - pm_node_flag_set(part, PM_NODE_FLAG_STATIC_LITERAL | PM_STRING_FLAGS_FROZEN); - } - - if (!PM_NODE_FLAG_P(part, PM_NODE_FLAG_STATIC_LITERAL)) { - pm_node_flag_unset((pm_node_t *) node, PM_NODE_FLAG_STATIC_LITERAL); - } - pm_node_list_append(&node->parts, part); node->base.location.end = part->location.end; } @@ -4390,7 +4371,6 @@ pm_interpolated_symbol_node_create(pm_parser_t *parser, const pm_token_t *openin *node = (pm_interpolated_symbol_node_t) { { .type = PM_INTERPOLATED_SYMBOL_NODE, - .flags = PM_NODE_FLAG_STATIC_LITERAL, .location = { .start = opening->start, .end = closing->end, @@ -4414,14 +4394,6 @@ pm_interpolated_symbol_node_append(pm_interpolated_symbol_node_t *node, pm_node_ node->base.location.start = part->location.start; } - if (PM_NODE_TYPE_P(part, PM_STRING_NODE)) { - pm_node_flag_set(part, PM_NODE_FLAG_STATIC_LITERAL | PM_STRING_FLAGS_FROZEN); - } - - if (!PM_NODE_FLAG_P(part, PM_NODE_FLAG_STATIC_LITERAL)) { - pm_node_flag_unset((pm_node_t *) node, PM_NODE_FLAG_STATIC_LITERAL); - } - pm_node_list_append(&node->parts, part); node->base.location.end = part->location.end; } @@ -4436,7 +4408,6 @@ pm_interpolated_xstring_node_create(pm_parser_t *parser, const pm_token_t *openi *node = (pm_interpolated_x_string_node_t) { { .type = PM_INTERPOLATED_X_STRING_NODE, - .flags = PM_NODE_FLAG_STATIC_LITERAL, .location = { .start = opening->start, .end = closing->end @@ -4452,14 +4423,6 @@ pm_interpolated_xstring_node_create(pm_parser_t *parser, const pm_token_t *openi static inline void pm_interpolated_xstring_node_append(pm_interpolated_x_string_node_t *node, pm_node_t *part) { - if (PM_NODE_TYPE_P(part, PM_STRING_NODE)) { - pm_node_flag_set(part, PM_NODE_FLAG_STATIC_LITERAL | PM_STRING_FLAGS_FROZEN); - } - - if (!PM_NODE_FLAG_P(part, PM_NODE_FLAG_STATIC_LITERAL)) { - pm_node_flag_unset((pm_node_t *) node, PM_NODE_FLAG_STATIC_LITERAL); - } - pm_node_list_append(&node->parts, part); node->base.location.end = part->location.end; } diff --git a/test/prism/snapshots/dos_endings.txt b/test/prism/snapshots/dos_endings.txt index 5ac946b207e135..c5b962f2181cf3 100644 --- a/test/prism/snapshots/dos_endings.txt +++ b/test/prism/snapshots/dos_endings.txt @@ -18,13 +18,13 @@ │ │ ├── opening_loc: ∅ │ │ ├── parts: (length: 2) │ │ │ ├── @ StringNode (location: (1,5)-(1,9)) - │ │ │ │ ├── flags: frozen + │ │ │ │ ├── flags: ∅ │ │ │ │ ├── opening_loc: (1,5)-(1,6) = "\"" │ │ │ │ ├── content_loc: (1,6)-(1,8) = "hi" │ │ │ │ ├── closing_loc: (1,8)-(1,9) = "\"" │ │ │ │ └── unescaped: "hi" │ │ │ └── @ StringNode (location: (2,5)-(2,12)) - │ │ │ ├── flags: frozen + │ │ │ ├── flags: ∅ │ │ │ ├── opening_loc: (2,5)-(2,6) = "\"" │ │ │ ├── content_loc: (2,6)-(2,11) = "there" │ │ │ ├── closing_loc: (2,11)-(2,12) = "\"" diff --git a/test/prism/snapshots/dstring.txt b/test/prism/snapshots/dstring.txt index 64ab6e8a823bbe..ad395f8a8eb94b 100644 --- a/test/prism/snapshots/dstring.txt +++ b/test/prism/snapshots/dstring.txt @@ -39,13 +39,13 @@ │ ├── opening_loc: ∅ │ ├── parts: (length: 2) │ │ ├── @ StringNode (location: (7,0)-(8,2)) - │ │ │ ├── flags: frozen + │ │ │ ├── flags: ∅ │ │ │ ├── opening_loc: (7,0)-(7,1) = "\"" │ │ │ ├── content_loc: (7,1)-(8,1) = "fo\no" │ │ │ ├── closing_loc: (8,1)-(8,2) = "\"" │ │ │ └── unescaped: "fo\no" │ │ └── @ StringNode (location: (8,3)-(9,2)) - │ │ ├── flags: frozen + │ │ ├── flags: ∅ │ │ ├── opening_loc: (8,3)-(8,4) = "\"" │ │ ├── content_loc: (8,4)-(9,1) = "ba\nr" │ │ ├── closing_loc: (9,1)-(9,2) = "\"" diff --git a/test/prism/snapshots/regex.txt b/test/prism/snapshots/regex.txt index d07ab8c5e71900..9e19bbb18d273c 100644 --- a/test/prism/snapshots/regex.txt +++ b/test/prism/snapshots/regex.txt @@ -39,7 +39,7 @@ │ ├── opening_loc: (7,0)-(7,1) = "/" │ ├── parts: (length: 2) │ │ ├── @ StringNode (location: (7,1)-(7,5)) - │ │ │ ├── flags: frozen + │ │ │ ├── flags: ∅ │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (7,1)-(7,5) = "aaa " │ │ │ ├── closing_loc: ∅ @@ -55,7 +55,7 @@ │ ├── opening_loc: (9,0)-(9,1) = "/" │ ├── parts: (length: 3) │ │ ├── @ StringNode (location: (9,1)-(9,5)) - │ │ │ ├── flags: frozen + │ │ │ ├── flags: ∅ │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (9,1)-(9,5) = "aaa " │ │ │ ├── closing_loc: ∅ @@ -77,7 +77,7 @@ │ │ │ │ └── block: ∅ │ │ │ └── closing_loc: (9,10)-(9,11) = "}" │ │ └── @ StringNode (location: (9,11)-(9,15)) - │ │ ├── flags: frozen + │ │ ├── flags: ∅ │ │ ├── opening_loc: ∅ │ │ ├── content_loc: (9,11)-(9,15) = " ccc" │ │ ├── closing_loc: ∅ @@ -192,7 +192,7 @@ │ ├── opening_loc: (30,0)-(30,1) = "/" │ ├── parts: (length: 2) │ │ ├── @ StringNode (location: (30,1)-(30,5)) - │ │ │ ├── flags: frozen + │ │ │ ├── flags: ∅ │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (30,1)-(30,5) = "aaa " │ │ │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/seattlerb/parse_line_evstr_after_break.txt b/test/prism/snapshots/seattlerb/parse_line_evstr_after_break.txt index d8acbc05c49f07..9cb200e94bce5b 100644 --- a/test/prism/snapshots/seattlerb/parse_line_evstr_after_break.txt +++ b/test/prism/snapshots/seattlerb/parse_line_evstr_after_break.txt @@ -7,7 +7,7 @@ ├── opening_loc: ∅ ├── parts: (length: 2) │ ├── @ StringNode (location: (1,0)-(1,3)) - │ │ ├── flags: frozen + │ │ ├── flags: ∅ │ │ ├── opening_loc: (1,0)-(1,1) = "\"" │ │ ├── content_loc: (1,1)-(1,2) = "a" │ │ ├── closing_loc: (1,2)-(1,3) = "\"" diff --git a/test/prism/snapshots/seattlerb/qsymbols_interp.txt b/test/prism/snapshots/seattlerb/qsymbols_interp.txt index 97bc6754ff5088..23c39302475630 100644 --- a/test/prism/snapshots/seattlerb/qsymbols_interp.txt +++ b/test/prism/snapshots/seattlerb/qsymbols_interp.txt @@ -16,7 +16,7 @@ │ │ ├── opening_loc: ∅ │ │ ├── parts: (length: 2) │ │ │ ├── @ StringNode (location: (1,5)-(1,6)) - │ │ │ │ ├── flags: frozen + │ │ │ │ ├── flags: ∅ │ │ │ │ ├── opening_loc: ∅ │ │ │ │ ├── content_loc: (1,5)-(1,6) = "b" │ │ │ │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/seattlerb/str_lit_concat_bad_encodings.txt b/test/prism/snapshots/seattlerb/str_lit_concat_bad_encodings.txt index b31245a7f4f664..b841407cd8df2a 100644 --- a/test/prism/snapshots/seattlerb/str_lit_concat_bad_encodings.txt +++ b/test/prism/snapshots/seattlerb/str_lit_concat_bad_encodings.txt @@ -7,13 +7,13 @@ ├── opening_loc: ∅ ├── parts: (length: 2) │ ├── @ StringNode (location: (1,0)-(1,62)) - │ │ ├── flags: forced_utf8_encoding, frozen + │ │ ├── flags: forced_utf8_encoding │ │ ├── opening_loc: (1,0)-(1,1) = "\"" │ │ ├── content_loc: (1,1)-(1,61) = "\\xE3\\xD3\\x8B\\xE3\\x83\\xBC\\x83\\xE3\\x83\\xE3\\x82\\xB3\\xA3\\x82\\x99" │ │ ├── closing_loc: (1,61)-(1,62) = "\"" │ │ └── unescaped: "\xE3Ӌー\x83\xE3\x83コ\xA3\x82\x99" │ └── @ StringNode (location: (2,8)-(2,66)) - │ ├── flags: forced_utf8_encoding, frozen + │ ├── flags: forced_utf8_encoding │ ├── opening_loc: (2,8)-(2,9) = "\"" │ ├── content_loc: (2,9)-(2,65) = "\\xE3\\x83\\xB3\\xE3\\x83\\x8F\\xE3\\x82\\x9A\\xC3\\xBD;foo@bar.com" │ ├── closing_loc: (2,65)-(2,66) = "\"" diff --git a/test/prism/snapshots/seattlerb/words_interp.txt b/test/prism/snapshots/seattlerb/words_interp.txt index e0b08bc075639a..dfead7d353404b 100644 --- a/test/prism/snapshots/seattlerb/words_interp.txt +++ b/test/prism/snapshots/seattlerb/words_interp.txt @@ -19,7 +19,7 @@ │ │ │ │ └── value: 1 │ │ │ └── closing_loc: (1,6)-(1,7) = "}" │ │ └── @ StringNode (location: (1,7)-(1,8)) - │ │ ├── flags: frozen + │ │ ├── flags: ∅ │ │ ├── opening_loc: ∅ │ │ ├── content_loc: (1,7)-(1,8) = "b" │ │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/spanning_heredoc.txt b/test/prism/snapshots/spanning_heredoc.txt index 7c3db913333712..90297d2282c753 100644 --- a/test/prism/snapshots/spanning_heredoc.txt +++ b/test/prism/snapshots/spanning_heredoc.txt @@ -36,13 +36,13 @@ │ │ │ │ ├── opening_loc: (4,13)-(4,14) = "/" │ │ │ │ ├── parts: (length: 2) │ │ │ │ │ ├── @ StringNode (location: (4,14)-(4,16)) - │ │ │ │ │ │ ├── flags: frozen + │ │ │ │ │ │ ├── flags: ∅ │ │ │ │ │ │ ├── opening_loc: ∅ │ │ │ │ │ │ ├── content_loc: (4,14)-(4,16) = "b\\" │ │ │ │ │ │ ├── closing_loc: ∅ │ │ │ │ │ │ └── unescaped: "b" │ │ │ │ │ └── @ StringNode (location: (7,0)-(7,1)) - │ │ │ │ │ ├── flags: frozen + │ │ │ │ │ ├── flags: ∅ │ │ │ │ │ ├── opening_loc: ∅ │ │ │ │ │ ├── content_loc: (7,0)-(7,1) = "b" │ │ │ │ │ ├── closing_loc: ∅ @@ -223,13 +223,13 @@ │ │ │ ├── opening_loc: ∅ │ │ │ ├── parts: (length: 2) │ │ │ │ ├── @ StringNode (location: (35,12)-(35,14)) - │ │ │ │ │ ├── flags: frozen + │ │ │ │ │ ├── flags: ∅ │ │ │ │ │ ├── opening_loc: ∅ │ │ │ │ │ ├── content_loc: (35,12)-(35,14) = "l\\" │ │ │ │ │ ├── closing_loc: ∅ │ │ │ │ │ └── unescaped: "l\n" │ │ │ │ └── @ StringNode (location: (38,0)-(38,1)) - │ │ │ │ ├── flags: frozen + │ │ │ │ ├── flags: ∅ │ │ │ │ ├── opening_loc: ∅ │ │ │ │ ├── content_loc: (38,0)-(38,1) = "l" │ │ │ │ ├── closing_loc: ∅ @@ -299,13 +299,13 @@ │ │ │ ├── opening_loc: ∅ │ │ │ ├── parts: (length: 2) │ │ │ │ ├── @ StringNode (location: (48,12)-(48,14)) - │ │ │ │ │ ├── flags: frozen + │ │ │ │ │ ├── flags: ∅ │ │ │ │ │ ├── opening_loc: ∅ │ │ │ │ │ ├── content_loc: (48,12)-(48,14) = "p\\" │ │ │ │ │ ├── closing_loc: ∅ │ │ │ │ │ └── unescaped: "p\n" │ │ │ │ └── @ StringNode (location: (48,12)-(48,14)) - │ │ │ │ ├── flags: frozen + │ │ │ │ ├── flags: ∅ │ │ │ │ ├── opening_loc: ∅ │ │ │ │ ├── content_loc: (48,12)-(48,14) = "p\\" │ │ │ │ ├── closing_loc: ∅ @@ -331,13 +331,13 @@ │ │ │ ├── opening_loc: (53,5)-(53,6) = "/" │ │ │ ├── parts: (length: 2) │ │ │ │ ├── @ StringNode (location: (53,6)-(53,7)) - │ │ │ │ │ ├── flags: frozen + │ │ │ │ │ ├── flags: ∅ │ │ │ │ │ ├── opening_loc: ∅ │ │ │ │ │ ├── content_loc: (53,6)-(53,7) = "\\" │ │ │ │ │ ├── closing_loc: ∅ │ │ │ │ │ └── unescaped: "" │ │ │ │ └── @ StringNode (location: (55,0)-(55,6)) - │ │ │ │ ├── flags: frozen + │ │ │ │ ├── flags: ∅ │ │ │ │ ├── opening_loc: ∅ │ │ │ │ ├── content_loc: (55,0)-(55,6) = "(?)" │ │ │ │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/strings.txt b/test/prism/snapshots/strings.txt index 471b371332c957..40a05ce09d5bbf 100644 --- a/test/prism/snapshots/strings.txt +++ b/test/prism/snapshots/strings.txt @@ -337,7 +337,7 @@ │ │ │ ├── opening_loc: ∅ │ │ │ ├── parts: (length: 3) │ │ │ │ ├── @ StringNode (location: (67,5)-(67,6)) - │ │ │ │ │ ├── flags: frozen + │ │ │ │ │ ├── flags: ∅ │ │ │ │ │ ├── opening_loc: ∅ │ │ │ │ │ ├── content_loc: (67,5)-(67,6) = "b" │ │ │ │ │ ├── closing_loc: ∅ @@ -359,7 +359,7 @@ │ │ │ │ │ │ └── block: ∅ │ │ │ │ │ └── closing_loc: (67,9)-(67,10) = "}" │ │ │ │ └── @ StringNode (location: (67,10)-(67,11)) - │ │ │ │ ├── flags: frozen + │ │ │ │ ├── flags: ∅ │ │ │ │ ├── opening_loc: ∅ │ │ │ │ ├── content_loc: (67,10)-(67,11) = "d" │ │ │ │ ├── closing_loc: ∅ @@ -495,13 +495,13 @@ │ ├── opening_loc: ∅ │ ├── parts: (length: 2) │ │ ├── @ StringNode (location: (99,0)-(99,2)) - │ │ │ ├── flags: frozen + │ │ │ ├── flags: ∅ │ │ │ ├── opening_loc: (99,0)-(99,1) = "?" │ │ │ ├── content_loc: (99,1)-(99,2) = "a" │ │ │ ├── closing_loc: ∅ │ │ │ └── unescaped: "a" │ │ └── @ StringNode (location: (99,3)-(99,6)) - │ │ ├── flags: frozen + │ │ ├── flags: ∅ │ │ ├── opening_loc: (99,3)-(99,4) = "\"" │ │ ├── content_loc: (99,4)-(99,5) = "a" │ │ ├── closing_loc: (99,5)-(99,6) = "\"" diff --git a/test/prism/snapshots/symbols.txt b/test/prism/snapshots/symbols.txt index b6a854db0b2069..c34be74b37d506 100644 --- a/test/prism/snapshots/symbols.txt +++ b/test/prism/snapshots/symbols.txt @@ -234,7 +234,7 @@ │ │ │ ├── opening_loc: ∅ │ │ │ ├── parts: (length: 2) │ │ │ │ ├── @ StringNode (location: (39,5)-(39,6)) - │ │ │ │ │ ├── flags: frozen + │ │ │ │ │ ├── flags: ∅ │ │ │ │ │ ├── opening_loc: ∅ │ │ │ │ │ ├── content_loc: (39,5)-(39,6) = "b" │ │ │ │ │ ├── closing_loc: ∅ @@ -262,7 +262,7 @@ │ │ │ │ │ │ └── value: 2 │ │ │ │ │ └── closing_loc: (39,14)-(39,15) = "}" │ │ │ │ └── @ StringNode (location: (39,15)-(39,16)) - │ │ │ │ ├── flags: frozen + │ │ │ │ ├── flags: ∅ │ │ │ │ ├── opening_loc: ∅ │ │ │ │ ├── content_loc: (39,15)-(39,16) = "c" │ │ │ │ ├── closing_loc: ∅ @@ -272,7 +272,7 @@ │ │ ├── opening_loc: ∅ │ │ ├── parts: (length: 3) │ │ │ ├── @ StringNode (location: (39,17)-(39,18)) - │ │ │ │ ├── flags: frozen + │ │ │ │ ├── flags: ∅ │ │ │ │ ├── opening_loc: ∅ │ │ │ │ ├── content_loc: (39,17)-(39,18) = "d" │ │ │ │ ├── closing_loc: ∅ @@ -287,7 +287,7 @@ │ │ │ │ │ └── value: 3 │ │ │ │ └── closing_loc: (39,21)-(39,22) = "}" │ │ │ └── @ StringNode (location: (39,22)-(39,23)) - │ │ │ ├── flags: frozen + │ │ │ ├── flags: ∅ │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (39,22)-(39,23) = "f" │ │ │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/unparser/corpus/literal/literal.txt b/test/prism/snapshots/unparser/corpus/literal/literal.txt index 8ecf613e7ce9ad..ba7dd70b5b717c 100644 --- a/test/prism/snapshots/unparser/corpus/literal/literal.txt +++ b/test/prism/snapshots/unparser/corpus/literal/literal.txt @@ -358,13 +358,13 @@ │ ├── opening_loc: ∅ │ ├── parts: (length: 2) │ │ ├── @ StringNode (location: (28,0)-(28,5)) - │ │ │ ├── flags: frozen + │ │ │ ├── flags: ∅ │ │ │ ├── opening_loc: (28,0)-(28,1) = "\"" │ │ │ ├── content_loc: (28,1)-(28,4) = "foo" │ │ │ ├── closing_loc: (28,4)-(28,5) = "\"" │ │ │ └── unescaped: "foo" │ │ └── @ StringNode (location: (28,6)-(28,11)) - │ │ ├── flags: frozen + │ │ ├── flags: ∅ │ │ ├── opening_loc: (28,6)-(28,7) = "\"" │ │ ├── content_loc: (28,7)-(28,10) = "bar" │ │ ├── closing_loc: (28,10)-(28,11) = "\"" @@ -497,7 +497,7 @@ │ ├── opening_loc: (39,0)-(39,1) = "`" │ ├── parts: (length: 2) │ │ ├── @ StringNode (location: (39,1)-(39,4)) - │ │ │ ├── flags: frozen + │ │ │ ├── flags: ∅ │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (39,1)-(39,4) = "foo" │ │ │ ├── closing_loc: ∅ @@ -582,7 +582,7 @@ │ ├── opening_loc: (51,0)-(51,1) = "/" │ ├── parts: (length: 2) │ │ ├── @ StringNode (location: (51,1)-(51,4)) - │ │ │ ├── flags: frozen + │ │ │ ├── flags: ∅ │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (51,1)-(51,4) = "foo" │ │ │ ├── closing_loc: ∅ @@ -601,7 +601,7 @@ │ ├── opening_loc: (52,0)-(52,1) = "/" │ ├── parts: (length: 2) │ │ ├── @ StringNode (location: (52,1)-(52,4)) - │ │ │ ├── flags: frozen + │ │ │ ├── flags: ∅ │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (52,1)-(52,4) = "foo" │ │ │ ├── closing_loc: ∅ @@ -1156,7 +1156,7 @@ ├── opening_loc: (89,0)-(89,1) = "`" ├── parts: (length: 3) │ ├── @ StringNode (location: (89,1)-(90,0)) - │ │ ├── flags: frozen + │ │ ├── flags: ∅ │ │ ├── opening_loc: ∅ │ │ ├── content_loc: (89,1)-(90,0) = " x\n" │ │ ├── closing_loc: ∅ @@ -1178,7 +1178,7 @@ │ │ │ └── block: ∅ │ │ └── closing_loc: (90,5)-(90,6) = "}" │ └── @ StringNode (location: (90,6)-(91,1)) - │ ├── flags: frozen + │ ├── flags: ∅ │ ├── opening_loc: ∅ │ ├── content_loc: (90,6)-(91,1) = "\n#" │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/unparser/corpus/semantic/dstr.txt b/test/prism/snapshots/unparser/corpus/semantic/dstr.txt index c5337729692b09..5ab954b6d4119d 100644 --- a/test/prism/snapshots/unparser/corpus/semantic/dstr.txt +++ b/test/prism/snapshots/unparser/corpus/semantic/dstr.txt @@ -429,7 +429,7 @@ │ ├── opening_loc: ∅ │ ├── parts: (length: 2) │ │ ├── @ StringNode (location: (119,0)-(119,3)) - │ │ │ ├── flags: frozen + │ │ │ ├── flags: ∅ │ │ │ ├── opening_loc: (119,0)-(119,1) = "'" │ │ │ ├── content_loc: (119,1)-(119,2) = "a" │ │ │ ├── closing_loc: (119,2)-(119,3) = "'" @@ -447,19 +447,19 @@ │ ├── opening_loc: ∅ │ ├── parts: (length: 3) │ │ ├── @ StringNode (location: (122,0)-(122,2)) - │ │ │ ├── flags: frozen + │ │ │ ├── flags: ∅ │ │ │ ├── opening_loc: (122,0)-(122,1) = "\"" │ │ │ ├── content_loc: (122,1)-(122,1) = "" │ │ │ ├── closing_loc: (122,1)-(122,2) = "\"" │ │ │ └── unescaped: "" │ │ ├── @ StringNode (location: (122,3)-(122,5)) - │ │ │ ├── flags: frozen + │ │ │ ├── flags: ∅ │ │ │ ├── opening_loc: (122,3)-(122,4) = "\"" │ │ │ ├── content_loc: (122,4)-(122,4) = "" │ │ │ ├── closing_loc: (122,4)-(122,5) = "\"" │ │ │ └── unescaped: "" │ │ └── @ StringNode (location: (122,6)-(122,8)) - │ │ ├── flags: frozen + │ │ ├── flags: ∅ │ │ ├── opening_loc: (122,6)-(122,7) = "\"" │ │ ├── content_loc: (122,7)-(122,7) = "" │ │ ├── closing_loc: (122,7)-(122,8) = "\"" @@ -487,7 +487,7 @@ │ │ │ │ └── closing_loc: (124,6)-(124,7) = "}" │ │ │ └── closing_loc: (124,7)-(124,8) = "\"" │ │ └── @ StringNode (location: (124,9)-(124,12)) - │ │ ├── flags: frozen + │ │ ├── flags: ∅ │ │ ├── opening_loc: (124,9)-(124,10) = "\"" │ │ ├── content_loc: (124,10)-(124,11) = "b" │ │ ├── closing_loc: (124,11)-(124,12) = "\"" @@ -512,7 +512,7 @@ │ │ │ │ └── name: :@a │ │ │ └── closing_loc: (125,5)-(125,6) = "\"" │ │ └── @ StringNode (location: (125,7)-(125,10)) - │ │ ├── flags: frozen + │ │ ├── flags: ∅ │ │ ├── opening_loc: (125,7)-(125,8) = "\"" │ │ ├── content_loc: (125,8)-(125,9) = "b" │ │ ├── closing_loc: (125,9)-(125,10) = "\"" @@ -537,7 +537,7 @@ │ │ │ │ └── name: :$a │ │ │ └── closing_loc: (126,5)-(126,6) = "\"" │ │ └── @ StringNode (location: (126,7)-(126,10)) - │ │ ├── flags: frozen + │ │ ├── flags: ∅ │ │ ├── opening_loc: (126,7)-(126,8) = "\"" │ │ ├── content_loc: (126,8)-(126,9) = "b" │ │ ├── closing_loc: (126,9)-(126,10) = "\"" @@ -562,7 +562,7 @@ │ │ │ └── name: :@@a │ │ └── closing_loc: (127,6)-(127,7) = "\"" │ └── @ StringNode (location: (127,8)-(127,11)) - │ ├── flags: frozen + │ ├── flags: ∅ │ ├── opening_loc: (127,8)-(127,9) = "\"" │ ├── content_loc: (127,9)-(127,10) = "b" │ ├── closing_loc: (127,10)-(127,11) = "\"" diff --git a/test/prism/snapshots/unparser/corpus/semantic/literal.txt b/test/prism/snapshots/unparser/corpus/semantic/literal.txt index ef666890be7d34..59e02be64fcfd2 100644 --- a/test/prism/snapshots/unparser/corpus/semantic/literal.txt +++ b/test/prism/snapshots/unparser/corpus/semantic/literal.txt @@ -55,7 +55,7 @@ │ │ │ │ └── name: :@bar │ │ │ └── closing_loc: (11,9)-(11,10) = "}" │ │ └── @ StringNode (location: (11,10)-(11,13)) - │ │ ├── flags: frozen + │ │ ├── flags: ∅ │ │ ├── opening_loc: ∅ │ │ ├── content_loc: (11,10)-(11,13) = "baz" │ │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/whitequark/array_symbols_interp.txt b/test/prism/snapshots/whitequark/array_symbols_interp.txt index 2437486b43e250..7583ad8833e17d 100644 --- a/test/prism/snapshots/whitequark/array_symbols_interp.txt +++ b/test/prism/snapshots/whitequark/array_symbols_interp.txt @@ -41,7 +41,7 @@ │ ├── opening_loc: ∅ │ ├── parts: (length: 2) │ │ ├── @ StringNode (location: (3,3)-(3,6)) - │ │ │ ├── flags: frozen + │ │ │ ├── flags: ∅ │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (3,3)-(3,6) = "foo" │ │ │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/whitequark/array_words_interp.txt b/test/prism/snapshots/whitequark/array_words_interp.txt index 00fd05e7f7befc..c2f23d2bf19d78 100644 --- a/test/prism/snapshots/whitequark/array_words_interp.txt +++ b/test/prism/snapshots/whitequark/array_words_interp.txt @@ -63,7 +63,7 @@ │ │ │ │ └── block: ∅ │ │ │ └── closing_loc: (3,12)-(3,13) = "}" │ │ ├── @ StringNode (location: (3,13)-(3,16)) - │ │ │ ├── flags: frozen + │ │ │ ├── flags: ∅ │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (3,13)-(3,16) = "foo" │ │ │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/whitequark/non_lvar_injecting_match.txt b/test/prism/snapshots/whitequark/non_lvar_injecting_match.txt index 584e997df2c6da..2df571e8dc1890 100644 --- a/test/prism/snapshots/whitequark/non_lvar_injecting_match.txt +++ b/test/prism/snapshots/whitequark/non_lvar_injecting_match.txt @@ -20,7 +20,7 @@ │ │ │ │ └── value: 1 │ │ │ └── closing_loc: (1,4)-(1,5) = "}" │ │ └── @ StringNode (location: (1,5)-(1,18)) - │ │ ├── flags: frozen + │ │ ├── flags: ∅ │ │ ├── opening_loc: ∅ │ │ ├── content_loc: (1,5)-(1,18) = "(?bar)" │ │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/whitequark/regex_interp.txt b/test/prism/snapshots/whitequark/regex_interp.txt index 0a6db4cfdf4b7b..ee8caf22b9ee55 100644 --- a/test/prism/snapshots/whitequark/regex_interp.txt +++ b/test/prism/snapshots/whitequark/regex_interp.txt @@ -8,7 +8,7 @@ ├── opening_loc: (1,0)-(1,1) = "/" ├── parts: (length: 3) │ ├── @ StringNode (location: (1,1)-(1,4)) - │ │ ├── flags: frozen + │ │ ├── flags: ∅ │ │ ├── opening_loc: ∅ │ │ ├── content_loc: (1,1)-(1,4) = "foo" │ │ ├── closing_loc: ∅ @@ -30,7 +30,7 @@ │ │ │ └── block: ∅ │ │ └── closing_loc: (1,9)-(1,10) = "}" │ └── @ StringNode (location: (1,10)-(1,13)) - │ ├── flags: frozen + │ ├── flags: ∅ │ ├── opening_loc: ∅ │ ├── content_loc: (1,10)-(1,13) = "baz" │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/whitequark/ruby_bug_11990.txt b/test/prism/snapshots/whitequark/ruby_bug_11990.txt index abdb8d9ddd6ca6..43c04e55a23a15 100644 --- a/test/prism/snapshots/whitequark/ruby_bug_11990.txt +++ b/test/prism/snapshots/whitequark/ruby_bug_11990.txt @@ -18,13 +18,13 @@ │ ├── opening_loc: ∅ │ ├── parts: (length: 2) │ │ ├── @ StringNode (location: (1,2)-(1,6)) - │ │ │ ├── flags: frozen + │ │ │ ├── flags: ∅ │ │ │ ├── opening_loc: (1,2)-(1,6) = "<<~E" │ │ │ ├── content_loc: (2,0)-(3,0) = " x\n" │ │ │ ├── closing_loc: (3,0)-(4,0) = "E\n" │ │ │ └── unescaped: "x\n" │ │ └── @ StringNode (location: (1,7)-(1,12)) - │ │ ├── flags: frozen + │ │ ├── flags: ∅ │ │ ├── opening_loc: (1,7)-(1,8) = "\"" │ │ ├── content_loc: (1,8)-(1,11) = " y" │ │ ├── closing_loc: (1,11)-(1,12) = "\"" diff --git a/test/prism/snapshots/whitequark/string_concat.txt b/test/prism/snapshots/whitequark/string_concat.txt index 1739b2f65bfed4..38ea0c745dd664 100644 --- a/test/prism/snapshots/whitequark/string_concat.txt +++ b/test/prism/snapshots/whitequark/string_concat.txt @@ -22,7 +22,7 @@ │ │ │ └── name: :@a │ │ └── closing_loc: (1,7)-(1,8) = "\"" │ └── @ StringNode (location: (1,9)-(1,14)) - │ ├── flags: frozen + │ ├── flags: ∅ │ ├── opening_loc: (1,9)-(1,10) = "\"" │ ├── content_loc: (1,10)-(1,13) = "bar" │ ├── closing_loc: (1,13)-(1,14) = "\"" diff --git a/test/prism/snapshots/whitequark/xstring_interp.txt b/test/prism/snapshots/whitequark/xstring_interp.txt index 0676ea96837f72..e9543098c2b232 100644 --- a/test/prism/snapshots/whitequark/xstring_interp.txt +++ b/test/prism/snapshots/whitequark/xstring_interp.txt @@ -7,7 +7,7 @@ ├── opening_loc: (1,0)-(1,1) = "`" ├── parts: (length: 3) │ ├── @ StringNode (location: (1,1)-(1,4)) - │ │ ├── flags: frozen + │ │ ├── flags: ∅ │ │ ├── opening_loc: ∅ │ │ ├── content_loc: (1,1)-(1,4) = "foo" │ │ ├── closing_loc: ∅ @@ -29,7 +29,7 @@ │ │ │ └── block: ∅ │ │ └── closing_loc: (1,9)-(1,10) = "}" │ └── @ StringNode (location: (1,10)-(1,13)) - │ ├── flags: frozen + │ ├── flags: ∅ │ ├── opening_loc: ∅ │ ├── content_loc: (1,10)-(1,13) = "baz" │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/xstring.txt b/test/prism/snapshots/xstring.txt index 1a177026db6bc4..04b4cbf6ea7151 100644 --- a/test/prism/snapshots/xstring.txt +++ b/test/prism/snapshots/xstring.txt @@ -13,7 +13,7 @@ │ ├── opening_loc: (3,0)-(3,1) = "`" │ ├── parts: (length: 3) │ │ ├── @ StringNode (location: (3,1)-(3,5)) - │ │ │ ├── flags: frozen + │ │ │ ├── flags: ∅ │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (3,1)-(3,5) = "foo " │ │ │ ├── closing_loc: ∅ @@ -35,7 +35,7 @@ │ │ │ │ └── block: ∅ │ │ │ └── closing_loc: (3,10)-(3,11) = "}" │ │ └── @ StringNode (location: (3,11)-(3,15)) - │ │ ├── flags: frozen + │ │ ├── flags: ∅ │ │ ├── opening_loc: ∅ │ │ ├── content_loc: (3,11)-(3,15) = " baz" │ │ ├── closing_loc: ∅ From 06d5d4f1d0f58bdf73a0978221fe13e17f36712e Mon Sep 17 00:00:00 2001 From: David Rodriguez Date: Mon, 25 Mar 2024 18:02:28 +0100 Subject: [PATCH 056/211] [rubygems/rubygems] Fix resolver bug where ActivationRequest objects were not properly compared They were delegating their `#hash` value to a class not overriding that method, and so were returning inconsistent results. https://github.com/rubygems/rubygems/commit/723e4ee0fc --- lib/rubygems/resolver/spec_specification.rb | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/rubygems/resolver/spec_specification.rb b/lib/rubygems/resolver/spec_specification.rb index 79a34d8063f6fa..00ef9fdba05bc0 100644 --- a/lib/rubygems/resolver/spec_specification.rb +++ b/lib/rubygems/resolver/spec_specification.rb @@ -66,4 +66,11 @@ def platform def version spec.version end + + ## + # The hash value for this specification. + + def hash + spec.hash + end end From aa90013829d9394fa92a423a818fb0d6e2ab89cb Mon Sep 17 00:00:00 2001 From: Martin Emde Date: Thu, 29 Feb 2024 17:05:18 -0800 Subject: [PATCH 057/211] [rubygems/rubygems] Fix: vendor_gem takes a block https://github.com/rubygems/rubygems/commit/50cda56fc3 --- test/rubygems/helper.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/rubygems/helper.rb b/test/rubygems/helper.rb index ed9e34a906faea..82fb50b70492c7 100644 --- a/test/rubygems/helper.rb +++ b/test/rubygems/helper.rb @@ -1367,12 +1367,12 @@ def v(string) # # Yields the +specification+ to the block, if given - def vendor_gem(name = "a", version = 1) + def vendor_gem(name = "a", version = 1, &block) directory = File.join "vendor", name FileUtils.mkdir_p directory - save_gemspec name, version, directory + save_gemspec name, version, directory, &block end ## From de742b425fde96283e6a5ac60da8cbdb12291bb9 Mon Sep 17 00:00:00 2001 From: Alan Wu Date: Mon, 25 Mar 2024 15:34:26 -0400 Subject: [PATCH 058/211] YJIT: Inline simple getlocal+leave iseqs This mainly targets things like `T.unsafe()` from Sorbet, which is just an identity function at runtime and only a hint for the static checker. Only deal with simple caller and callees (no keywords and splat etc.). Co-authored-by: Takashi Kokubun (k0kubun) --- bootstraptest/test_yjit.rb | 16 ++++++++++++++++ yjit/src/codegen.rs | 35 ++++++++++++++++++++++++++++++++--- yjit/src/cruby.rs | 1 + 3 files changed, 49 insertions(+), 3 deletions(-) diff --git a/bootstraptest/test_yjit.rb b/bootstraptest/test_yjit.rb index 24253a10c9aa2c..e5a160fa9fa342 100644 --- a/bootstraptest/test_yjit.rb +++ b/bootstraptest/test_yjit.rb @@ -4753,3 +4753,19 @@ def entry(define) entry(false) entry(true) } + +assert_equal '[:ok, :ok, :ok]', %q{ + def identity(x) = x + def foo(x, _) = x + def bar(_, _, _, _, x) = x + + def tests + [ + identity(:ok), + foo(:ok, 2), + bar(1, 2, 3, 4, :ok), + ] + end + + tests +} diff --git a/yjit/src/codegen.rs b/yjit/src/codegen.rs index 7d9085d4b30d4e..0d536907c1abad 100644 --- a/yjit/src/codegen.rs +++ b/yjit/src/codegen.rs @@ -6754,11 +6754,16 @@ fn gen_send_bmethod( /// The kind of a value an ISEQ returns enum IseqReturn { Value(VALUE), + LocalVariable(u32), Receiver, } -/// Return the ISEQ's return value if it consists of only putnil/putobject and leave. -fn iseq_get_return_value(iseq: IseqPtr, captured_opnd: Option) -> Option { +extern { + fn rb_simple_iseq_p(iseq: IseqPtr) -> bool; +} + +/// Return the ISEQ's return value if it consists of one simple instruction and leave. +fn iseq_get_return_value(iseq: IseqPtr, captured_opnd: Option, ci_flags: u32) -> Option { // Expect only two instructions and one possible operand let iseq_size = unsafe { get_iseq_encoded_size(iseq) }; if !(2..=3).contains(&iseq_size) { @@ -6774,6 +6779,17 @@ fn iseq_get_return_value(iseq: IseqPtr, captured_opnd: Option) -> Option { + // Only accept simple positional only cases for both the caller and the callee. + // Reject block ISEQs to avoid autosplat and other block parameter complications. + if captured_opnd.is_none() && unsafe { rb_simple_iseq_p(iseq) } && ci_flags & VM_CALL_ARGS_SIMPLE != 0 { + let ep_offset = unsafe { *rb_iseq_pc_at_idx(iseq, 1) }.as_u32(); + let local_idx = ep_offset_to_local_idx(iseq, ep_offset); + Some(IseqReturn::LocalVariable(local_idx)) + } else { + None + } + } YARVINSN_putnil => Some(IseqReturn::Value(Qnil)), YARVINSN_putobject => Some(IseqReturn::Value(unsafe { *rb_iseq_pc_at_idx(iseq, 1) })), YARVINSN_putobject_INT2FIX_0_ => Some(IseqReturn::Value(VALUE::fixnum_from_usize(0))), @@ -7055,11 +7071,24 @@ fn gen_send_iseq( } // Inline simple ISEQs whose return value is known at compile time - if let (Some(value), None, false) = (iseq_get_return_value(iseq, captured_opnd), block_arg_type, opt_send_call) { + if let (Some(value), None, false) = (iseq_get_return_value(iseq, captured_opnd, flags), block_arg_type, opt_send_call) { asm_comment!(asm, "inlined simple ISEQ"); gen_counter_incr(asm, Counter::num_send_iseq_inline); match value { + IseqReturn::LocalVariable(local_idx) => { + // Put the local variable at the return slot + let stack_local = asm.stack_opnd(argc - 1 - local_idx as i32); + let stack_return = asm.stack_opnd(argc); + asm.mov(stack_return, stack_local); + + // Update the mapping for the return value + let mapping = asm.ctx.get_opnd_mapping(stack_local.into()); + asm.ctx.set_opnd_mapping(stack_return.into(), mapping); + + // Pop everything but the return value + asm.stack_pop(argc as usize); + } IseqReturn::Value(value) => { // Pop receiver and arguments asm.stack_pop(argc as usize + if captured_opnd.is_some() { 0 } else { 1 }); diff --git a/yjit/src/cruby.rs b/yjit/src/cruby.rs index 31d842531f5515..9547e3fa2cc67b 100644 --- a/yjit/src/cruby.rs +++ b/yjit/src/cruby.rs @@ -711,6 +711,7 @@ mod manual_defs { pub const RUBY_FIXNUM_MAX: isize = RUBY_LONG_MAX / 2; // From vm_callinfo.h - uses calculation that seems to confuse bindgen + pub const VM_CALL_ARGS_SIMPLE: u32 = 1 << VM_CALL_ARGS_SIMPLE_bit; pub const VM_CALL_ARGS_SPLAT: u32 = 1 << VM_CALL_ARGS_SPLAT_bit; pub const VM_CALL_ARGS_BLOCKARG: u32 = 1 << VM_CALL_ARGS_BLOCKARG_bit; pub const VM_CALL_FCALL: u32 = 1 << VM_CALL_FCALL_bit; From b39057f32cdcf15d8586bd1629d5fc78038320b7 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Tue, 26 Mar 2024 11:10:20 +0900 Subject: [PATCH 059/211] Fix extension installer for out-of-place build https://github.com/ruby/ruby/pull/9673#issuecomment-2019028293 --- tool/rbinstall.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tool/rbinstall.rb b/tool/rbinstall.rb index cdaeff666832b4..63f4beb9434861 100755 --- a/tool/rbinstall.rb +++ b/tool/rbinstall.rb @@ -588,7 +588,12 @@ def ext_features end def makefile_path - "#{makefile_dir}/Makefile" + if File.exist?("#{makefile_dir}/Makefile") + "#{makefile_dir}/Makefile" + else + # for out-of-place build + "#{$ext_build_dir}/#{relative_base}/Makefile" + end end def makefile_dir From 3680981c7b71df8c3a426164787ccefe5296bb25 Mon Sep 17 00:00:00 2001 From: Koichi Sasada Date: Tue, 26 Mar 2024 16:43:14 +0900 Subject: [PATCH 060/211] skip `test_gc_stress_at_startup` (maybe) from 9cf754b the test fails on some environments: https://rubyci.s3.amazonaws.com/icc-x64/ruby-master/log/20240325T200004Z.fail.html.gz ``` 1) Failure: TestGc#test_gc_stress_at_startup [/home/chkbuild/chkbuild/tmp/build/20240325T200004Z/ruby/test/ruby/test_gc.rb:771]: [Bug #15784] pid 1087168 killed by SIGSEGV (signal 11) (core dumped). 1. [3/3] Assertion for "success?" | Expected # to be success?. ``` https://rubyci.s3.amazonaws.com/freebsd14/ruby-master/log/20240325T203002Z.fail.html.gz https://rubyci.s3.amazonaws.com/osx1200arm-no-yjit/ruby-master/log/20240325T195003Z.fail.html.gz https://rubyci.s3.amazonaws.com/s390x/ruby-master/log/20240325T210003Z.fail.html.gz ... so just skipt it until it works. --- test/ruby/test_gc.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/test/ruby/test_gc.rb b/test/ruby/test_gc.rb index 39b001c3d0fee6..470e53fc1fb3f0 100644 --- a/test/ruby/test_gc.rb +++ b/test/ruby/test_gc.rb @@ -768,6 +768,7 @@ def initialize end def test_gc_stress_at_startup + omit 'TODO: fixme later' assert_in_out_err([{"RUBY_DEBUG"=>"gc_stress"}], '', [], [], '[Bug #15784]', success: true, timeout: 60) end From a850cd1a87bef738c40d9c550fb8823699083f2e Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Tue, 26 Mar 2024 10:04:54 +0900 Subject: [PATCH 061/211] [Bug #20392] Block arguments duplication check at `super` --- parse.y | 3 ++- test/ruby/test_syntax.rb | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/parse.y b/parse.y index 28b03f8cd149b8..585130c3465ed6 100644 --- a/parse.y +++ b/parse.y @@ -2063,8 +2063,9 @@ get_nd_args(struct parser_params *p, NODE *node) return RNODE_FCALL(node)->nd_args; case NODE_QCALL: return RNODE_QCALL(node)->nd_args; - case NODE_VCALL: case NODE_SUPER: + return RNODE_SUPER(node)->nd_args; + case NODE_VCALL: case NODE_ZSUPER: case NODE_YIELD: case NODE_RETURN: diff --git a/test/ruby/test_syntax.rb b/test/ruby/test_syntax.rb index 42108f955f0241..371c41fe37beda 100644 --- a/test/ruby/test_syntax.rb +++ b/test/ruby/test_syntax.rb @@ -2154,6 +2154,13 @@ def obj.bar(*args, **kws, &block) assert_equal 0...1, exp.call(a: 0) end + def test_argument_forwarding_with_super + assert_valid_syntax('def foo(...) super {}; end') + assert_valid_syntax('def foo(...) super() {}; end') + assert_syntax_error('def foo(...) super(...) {}; end', /both block arg and actual block/) + assert_syntax_error('def foo(...) super(1, ...) {}; end', /both block arg and actual block/) + end + def test_class_module_Object_ancestors assert_separately([], <<-RUBY) m = Module.new From 52cf6ec46b744a2ac34d9f6531520bf2aee017f5 Mon Sep 17 00:00:00 2001 From: Koichi ITO Date: Sun, 24 Mar 2024 20:06:35 +0900 Subject: [PATCH 062/211] [ruby/prism] Fix typos After finding the "if if" typo, some additional typos identified by running `codespell` are also being corrected: https://github.com/codespell-project/codespell https://github.com/ruby/prism/commit/e6a34cfeeb --- prism/defines.h | 8 ++++---- prism/extension.c | 2 +- prism/options.h | 4 ++-- prism/prism.c | 14 +++++++------- test/prism/parse_test.rb | 2 +- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/prism/defines.h b/prism/defines.h index 2fe73fe3d8cbd9..849ca6d05160ff 100644 --- a/prism/defines.h +++ b/prism/defines.h @@ -151,7 +151,7 @@ #else #ifndef xmalloc /** - * The malloc function that should be used. This can be overriden with + * The malloc function that should be used. This can be overridden with * the PRISM_XALLOCATOR define. */ #define xmalloc malloc @@ -159,7 +159,7 @@ #ifndef xrealloc /** - * The realloc function that should be used. This can be overriden with + * The realloc function that should be used. This can be overridden with * the PRISM_XALLOCATOR define. */ #define xrealloc realloc @@ -167,7 +167,7 @@ #ifndef xcalloc /** - * The calloc function that should be used. This can be overriden with + * The calloc function that should be used. This can be overridden with * the PRISM_XALLOCATOR define. */ #define xcalloc calloc @@ -175,7 +175,7 @@ #ifndef xfree /** - * The free function that should be used. This can be overriden with the + * The free function that should be used. This can be overridden with the * PRISM_XALLOCATOR define. */ #define xfree free diff --git a/prism/extension.c b/prism/extension.c index 88ba006ac2ec30..5b5c0593c91719 100644 --- a/prism/extension.c +++ b/prism/extension.c @@ -730,7 +730,7 @@ parse_input(pm_string_t *input, const pm_options_t *options) { * parsed. This should be an array of arrays of symbols or nil. Scopes are * ordered from the outermost scope to the innermost one. * * `version` - the version of Ruby syntax that prism should used to parse Ruby - * code. By default prism assumes you want to parse with the latest vesion + * code. By default prism assumes you want to parse with the latest version * of Ruby syntax (which you can trigger with `nil` or `"latest"`). You * may also restrict the syntax to a specific version of Ruby. The * supported values are `"3.3.0"` and `"3.4.0"`. diff --git a/prism/options.h b/prism/options.h index d07f5aa4fa7e72..9a4d6969c3c67b 100644 --- a/prism/options.h +++ b/prism/options.h @@ -286,14 +286,14 @@ PRISM_EXPORTED_FUNCTION void pm_options_free(pm_options_t *options); * | `0` | use the latest version of prism | * | `1` | use the version of prism that is vendored in CRuby 3.3.0 | * - * Each scope is layed out as follows: + * Each scope is laid out as follows: * * | # bytes | field | * | ------- | -------------------------- | * | `4` | the number of locals | * | ... | the locals | * - * Each local is layed out as follows: + * Each local is laid out as follows: * * | # bytes | field | * | ------- | -------------------------- | diff --git a/prism/prism.c b/prism/prism.c index 6f2ab81e9ae3f3..169f9c26d64028 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -5602,7 +5602,7 @@ pm_rescue_modifier_node_create(pm_parser_t *parser, pm_node_t *expression, const } /** - * Allocate and initiliaze a new RescueNode node. + * Allocate and initialize a new RescueNode node. */ static pm_rescue_node_t * pm_rescue_node_create(pm_parser_t *parser, const pm_token_t *keyword) { @@ -10416,7 +10416,7 @@ parser_lex(pm_parser_t *parser) { } default: // If we get to this point, then we have a % that is completely - // unparseable. In this case we'll just drop it from the parser + // unparsable. In this case we'll just drop it from the parser // and skip past it and hope that the next token is something // that we can parse. pm_parser_err_current(parser, PM_ERR_INVALID_PERCENT); @@ -14426,7 +14426,7 @@ parse_heredoc_dedent_string(pm_string_t *string, size_t common_whitespace) { static void parse_heredoc_dedent(pm_parser_t *parser, pm_node_list_t *nodes, size_t common_whitespace) { // The next node should be dedented if it's the first node in the list or if - // if follows a string node. + // it follows a string node. bool dedent_next = true; // Iterate over all nodes, and trim whitespace accordingly. We're going to @@ -17949,7 +17949,7 @@ parse_regular_expression_named_captures(pm_parser_t *parser, const pm_string_t * if (pm_regexp_named_capture_group_names(pm_string_source(content), pm_string_length(content), &named_captures, parser->encoding_changed, parser->encoding) && (named_captures.length > 0)) { // Since we should not create a MatchWriteNode when all capture names - // are invalid, creating a MatchWriteNode is delayed here. + // are invalid, creating a MatchWriteNode is delaid here. pm_match_write_node_t *match = NULL; pm_constant_id_list_t names = { 0 }; @@ -19616,7 +19616,7 @@ pm_parser_errors_format(const pm_parser_t *parser, pm_buffer_t *buffer, bool col // Now we're going to determine how we're going to format line numbers and // blank lines based on the maximum number of digits in the line numbers - // that are going to be displayed. + // that are going to be displaid. pm_error_format_t error_format; int32_t max_line_number = errors[error_list->size - 1].line - start_line; @@ -19707,7 +19707,7 @@ pm_parser_errors_format(const pm_parser_t *parser, pm_buffer_t *buffer, bool col pm_error_t *error = &errors[index]; // Here we determine how many lines of padding of the source to display, - // based on the difference from the last line that was displayed. + // based on the difference from the last line that was displaid. if (error->line - last_line > 1) { if (error->line - last_line > 2) { if ((index != 0) && (error->line - last_line > 3)) { @@ -19739,7 +19739,7 @@ pm_parser_errors_format(const pm_parser_t *parser, pm_buffer_t *buffer, bool col // the width of the error, then the error message itself. // // Note that this doesn't take into account the width of the actual - // character when displayed in the terminal. For some east-asian + // character when displaid in the terminal. For some east-asian // languages or emoji, this means it can be thrown off pretty badly. We // will need to solve this eventually. pm_buffer_append_string(buffer, " ", 2); diff --git a/test/prism/parse_test.rb b/test/prism/parse_test.rb index db66b431ba7ad0..4255553f516afd 100644 --- a/test/prism/parse_test.rb +++ b/test/prism/parse_test.rb @@ -204,7 +204,7 @@ def test_parse_file_comments # Additionally, Ripper cannot parse the %w[] fixture in this file, so set ripper_should_parse to false. ripper_should_parse = false if relative == "spanning_heredoc.txt" - # Ruby < 3.3.0 cannot parse heredocs where there are leading whitespace charactes in the heredoc start. + # Ruby < 3.3.0 cannot parse heredocs where there are leading whitespace characters in the heredoc start. # Example: <<~' EOF' or <<-' EOF' # https://bugs.ruby-lang.org/issues/19539 ripper_should_parse = false if relative == "heredocs_leading_whitespace.txt" && RUBY_VERSION < "3.3.0" From 8cfa8e87b2705fb356bbbb9ef719b5c5a54f9862 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Tue, 26 Mar 2024 20:30:29 +0900 Subject: [PATCH 063/211] [ruby/irb] Fix a typo (https://github.com/ruby/irb/pull/912) https://github.com/ruby/irb/commit/2057248e40 --- lib/irb/cmd/nop.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/irb/cmd/nop.rb b/lib/irb/cmd/nop.rb index 49f89bac95eed8..9d2e3c4d47cd6e 100644 --- a/lib/irb/cmd/nop.rb +++ b/lib/irb/cmd/nop.rb @@ -1,4 +1,4 @@ # frozen_string_literal: true # This file is just a placeholder for backward-compatibility. -# Please require 'irb' and inheirt your command from `IRB::Command::Base` instead. +# Please require 'irb' and inherit your command from `IRB::Command::Base` instead. From 2b08406cd0db0042520fb0346544660e10a4d93c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20Barri=C3=A9?= Date: Mon, 25 Mar 2024 11:18:26 +0100 Subject: [PATCH 064/211] Expose rb_str_chilled_p Some extensions (like stringio) may need to differentiate between chilled strings and frozen strings. They can now use rb_str_chilled_p but must check for its presence since the function will be removed when chilled strings are removed. [Bug #20389] [Feature #20205] Co-authored-by: Jean Boussier --- ext/-test-/string/chilled.c | 13 +++ ext/-test-/string/depend | 159 ++++++++++++++++++++++++++ include/ruby/internal/intern/string.h | 15 +++ string.c | 6 + test/-ext-/string/test_chilled.rb | 19 +++ 5 files changed, 212 insertions(+) create mode 100644 ext/-test-/string/chilled.c create mode 100644 test/-ext-/string/test_chilled.rb diff --git a/ext/-test-/string/chilled.c b/ext/-test-/string/chilled.c new file mode 100644 index 00000000000000..c98fc72c477256 --- /dev/null +++ b/ext/-test-/string/chilled.c @@ -0,0 +1,13 @@ +#include "ruby.h" + +static VALUE +bug_s_rb_str_chilled_p(VALUE self, VALUE str) +{ + return rb_str_chilled_p(str) ? Qtrue : Qfalse; +} + +void +Init_string_chilled(VALUE klass) +{ + rb_define_singleton_method(klass, "rb_str_chilled_p", bug_s_rb_str_chilled_p, 1); +} diff --git a/ext/-test-/string/depend b/ext/-test-/string/depend index eeb4914346bd91..f8f58e7d441ee2 100644 --- a/ext/-test-/string/depend +++ b/ext/-test-/string/depend @@ -173,6 +173,165 @@ capacity.o: $(hdrdir)/ruby/subst.h capacity.o: $(top_srcdir)/internal/compilers.h capacity.o: $(top_srcdir)/internal/string.h capacity.o: capacity.c +chilled.o: $(RUBY_EXTCONF_H) +chilled.o: $(arch_hdrdir)/ruby/config.h +chilled.o: $(hdrdir)/ruby.h +chilled.o: $(hdrdir)/ruby/assert.h +chilled.o: $(hdrdir)/ruby/backward.h +chilled.o: $(hdrdir)/ruby/backward/2/assume.h +chilled.o: $(hdrdir)/ruby/backward/2/attributes.h +chilled.o: $(hdrdir)/ruby/backward/2/bool.h +chilled.o: $(hdrdir)/ruby/backward/2/inttypes.h +chilled.o: $(hdrdir)/ruby/backward/2/limits.h +chilled.o: $(hdrdir)/ruby/backward/2/long_long.h +chilled.o: $(hdrdir)/ruby/backward/2/stdalign.h +chilled.o: $(hdrdir)/ruby/backward/2/stdarg.h +chilled.o: $(hdrdir)/ruby/defines.h +chilled.o: $(hdrdir)/ruby/intern.h +chilled.o: $(hdrdir)/ruby/internal/abi.h +chilled.o: $(hdrdir)/ruby/internal/anyargs.h +chilled.o: $(hdrdir)/ruby/internal/arithmetic.h +chilled.o: $(hdrdir)/ruby/internal/arithmetic/char.h +chilled.o: $(hdrdir)/ruby/internal/arithmetic/double.h +chilled.o: $(hdrdir)/ruby/internal/arithmetic/fixnum.h +chilled.o: $(hdrdir)/ruby/internal/arithmetic/gid_t.h +chilled.o: $(hdrdir)/ruby/internal/arithmetic/int.h +chilled.o: $(hdrdir)/ruby/internal/arithmetic/intptr_t.h +chilled.o: $(hdrdir)/ruby/internal/arithmetic/long.h +chilled.o: $(hdrdir)/ruby/internal/arithmetic/long_long.h +chilled.o: $(hdrdir)/ruby/internal/arithmetic/mode_t.h +chilled.o: $(hdrdir)/ruby/internal/arithmetic/off_t.h +chilled.o: $(hdrdir)/ruby/internal/arithmetic/pid_t.h +chilled.o: $(hdrdir)/ruby/internal/arithmetic/short.h +chilled.o: $(hdrdir)/ruby/internal/arithmetic/size_t.h +chilled.o: $(hdrdir)/ruby/internal/arithmetic/st_data_t.h +chilled.o: $(hdrdir)/ruby/internal/arithmetic/uid_t.h +chilled.o: $(hdrdir)/ruby/internal/assume.h +chilled.o: $(hdrdir)/ruby/internal/attr/alloc_size.h +chilled.o: $(hdrdir)/ruby/internal/attr/artificial.h +chilled.o: $(hdrdir)/ruby/internal/attr/cold.h +chilled.o: $(hdrdir)/ruby/internal/attr/const.h +chilled.o: $(hdrdir)/ruby/internal/attr/constexpr.h +chilled.o: $(hdrdir)/ruby/internal/attr/deprecated.h +chilled.o: $(hdrdir)/ruby/internal/attr/diagnose_if.h +chilled.o: $(hdrdir)/ruby/internal/attr/enum_extensibility.h +chilled.o: $(hdrdir)/ruby/internal/attr/error.h +chilled.o: $(hdrdir)/ruby/internal/attr/flag_enum.h +chilled.o: $(hdrdir)/ruby/internal/attr/forceinline.h +chilled.o: $(hdrdir)/ruby/internal/attr/format.h +chilled.o: $(hdrdir)/ruby/internal/attr/maybe_unused.h +chilled.o: $(hdrdir)/ruby/internal/attr/noalias.h +chilled.o: $(hdrdir)/ruby/internal/attr/nodiscard.h +chilled.o: $(hdrdir)/ruby/internal/attr/noexcept.h +chilled.o: $(hdrdir)/ruby/internal/attr/noinline.h +chilled.o: $(hdrdir)/ruby/internal/attr/nonnull.h +chilled.o: $(hdrdir)/ruby/internal/attr/noreturn.h +chilled.o: $(hdrdir)/ruby/internal/attr/packed_struct.h +chilled.o: $(hdrdir)/ruby/internal/attr/pure.h +chilled.o: $(hdrdir)/ruby/internal/attr/restrict.h +chilled.o: $(hdrdir)/ruby/internal/attr/returns_nonnull.h +chilled.o: $(hdrdir)/ruby/internal/attr/warning.h +chilled.o: $(hdrdir)/ruby/internal/attr/weakref.h +chilled.o: $(hdrdir)/ruby/internal/cast.h +chilled.o: $(hdrdir)/ruby/internal/compiler_is.h +chilled.o: $(hdrdir)/ruby/internal/compiler_is/apple.h +chilled.o: $(hdrdir)/ruby/internal/compiler_is/clang.h +chilled.o: $(hdrdir)/ruby/internal/compiler_is/gcc.h +chilled.o: $(hdrdir)/ruby/internal/compiler_is/intel.h +chilled.o: $(hdrdir)/ruby/internal/compiler_is/msvc.h +chilled.o: $(hdrdir)/ruby/internal/compiler_is/sunpro.h +chilled.o: $(hdrdir)/ruby/internal/compiler_since.h +chilled.o: $(hdrdir)/ruby/internal/config.h +chilled.o: $(hdrdir)/ruby/internal/constant_p.h +chilled.o: $(hdrdir)/ruby/internal/core.h +chilled.o: $(hdrdir)/ruby/internal/core/rarray.h +chilled.o: $(hdrdir)/ruby/internal/core/rbasic.h +chilled.o: $(hdrdir)/ruby/internal/core/rbignum.h +chilled.o: $(hdrdir)/ruby/internal/core/rclass.h +chilled.o: $(hdrdir)/ruby/internal/core/rdata.h +chilled.o: $(hdrdir)/ruby/internal/core/rfile.h +chilled.o: $(hdrdir)/ruby/internal/core/rhash.h +chilled.o: $(hdrdir)/ruby/internal/core/robject.h +chilled.o: $(hdrdir)/ruby/internal/core/rregexp.h +chilled.o: $(hdrdir)/ruby/internal/core/rstring.h +chilled.o: $(hdrdir)/ruby/internal/core/rstruct.h +chilled.o: $(hdrdir)/ruby/internal/core/rtypeddata.h +chilled.o: $(hdrdir)/ruby/internal/ctype.h +chilled.o: $(hdrdir)/ruby/internal/dllexport.h +chilled.o: $(hdrdir)/ruby/internal/dosish.h +chilled.o: $(hdrdir)/ruby/internal/error.h +chilled.o: $(hdrdir)/ruby/internal/eval.h +chilled.o: $(hdrdir)/ruby/internal/event.h +chilled.o: $(hdrdir)/ruby/internal/fl_type.h +chilled.o: $(hdrdir)/ruby/internal/gc.h +chilled.o: $(hdrdir)/ruby/internal/glob.h +chilled.o: $(hdrdir)/ruby/internal/globals.h +chilled.o: $(hdrdir)/ruby/internal/has/attribute.h +chilled.o: $(hdrdir)/ruby/internal/has/builtin.h +chilled.o: $(hdrdir)/ruby/internal/has/c_attribute.h +chilled.o: $(hdrdir)/ruby/internal/has/cpp_attribute.h +chilled.o: $(hdrdir)/ruby/internal/has/declspec_attribute.h +chilled.o: $(hdrdir)/ruby/internal/has/extension.h +chilled.o: $(hdrdir)/ruby/internal/has/feature.h +chilled.o: $(hdrdir)/ruby/internal/has/warning.h +chilled.o: $(hdrdir)/ruby/internal/intern/array.h +chilled.o: $(hdrdir)/ruby/internal/intern/bignum.h +chilled.o: $(hdrdir)/ruby/internal/intern/class.h +chilled.o: $(hdrdir)/ruby/internal/intern/compar.h +chilled.o: $(hdrdir)/ruby/internal/intern/complex.h +chilled.o: $(hdrdir)/ruby/internal/intern/cont.h +chilled.o: $(hdrdir)/ruby/internal/intern/dir.h +chilled.o: $(hdrdir)/ruby/internal/intern/enum.h +chilled.o: $(hdrdir)/ruby/internal/intern/enumerator.h +chilled.o: $(hdrdir)/ruby/internal/intern/error.h +chilled.o: $(hdrdir)/ruby/internal/intern/eval.h +chilled.o: $(hdrdir)/ruby/internal/intern/file.h +chilled.o: $(hdrdir)/ruby/internal/intern/hash.h +chilled.o: $(hdrdir)/ruby/internal/intern/io.h +chilled.o: $(hdrdir)/ruby/internal/intern/load.h +chilled.o: $(hdrdir)/ruby/internal/intern/marshal.h +chilled.o: $(hdrdir)/ruby/internal/intern/numeric.h +chilled.o: $(hdrdir)/ruby/internal/intern/object.h +chilled.o: $(hdrdir)/ruby/internal/intern/parse.h +chilled.o: $(hdrdir)/ruby/internal/intern/proc.h +chilled.o: $(hdrdir)/ruby/internal/intern/process.h +chilled.o: $(hdrdir)/ruby/internal/intern/random.h +chilled.o: $(hdrdir)/ruby/internal/intern/range.h +chilled.o: $(hdrdir)/ruby/internal/intern/rational.h +chilled.o: $(hdrdir)/ruby/internal/intern/re.h +chilled.o: $(hdrdir)/ruby/internal/intern/ruby.h +chilled.o: $(hdrdir)/ruby/internal/intern/select.h +chilled.o: $(hdrdir)/ruby/internal/intern/select/largesize.h +chilled.o: $(hdrdir)/ruby/internal/intern/signal.h +chilled.o: $(hdrdir)/ruby/internal/intern/sprintf.h +chilled.o: $(hdrdir)/ruby/internal/intern/string.h +chilled.o: $(hdrdir)/ruby/internal/intern/struct.h +chilled.o: $(hdrdir)/ruby/internal/intern/thread.h +chilled.o: $(hdrdir)/ruby/internal/intern/time.h +chilled.o: $(hdrdir)/ruby/internal/intern/variable.h +chilled.o: $(hdrdir)/ruby/internal/intern/vm.h +chilled.o: $(hdrdir)/ruby/internal/interpreter.h +chilled.o: $(hdrdir)/ruby/internal/iterator.h +chilled.o: $(hdrdir)/ruby/internal/memory.h +chilled.o: $(hdrdir)/ruby/internal/method.h +chilled.o: $(hdrdir)/ruby/internal/module.h +chilled.o: $(hdrdir)/ruby/internal/newobj.h +chilled.o: $(hdrdir)/ruby/internal/scan_args.h +chilled.o: $(hdrdir)/ruby/internal/special_consts.h +chilled.o: $(hdrdir)/ruby/internal/static_assert.h +chilled.o: $(hdrdir)/ruby/internal/stdalign.h +chilled.o: $(hdrdir)/ruby/internal/stdbool.h +chilled.o: $(hdrdir)/ruby/internal/symbol.h +chilled.o: $(hdrdir)/ruby/internal/value.h +chilled.o: $(hdrdir)/ruby/internal/value_type.h +chilled.o: $(hdrdir)/ruby/internal/variable.h +chilled.o: $(hdrdir)/ruby/internal/warning_push.h +chilled.o: $(hdrdir)/ruby/internal/xmalloc.h +chilled.o: $(hdrdir)/ruby/missing.h +chilled.o: $(hdrdir)/ruby/ruby.h +chilled.o: $(hdrdir)/ruby/st.h +chilled.o: $(hdrdir)/ruby/subst.h +chilled.o: chilled.c coderange.o: $(RUBY_EXTCONF_H) coderange.o: $(arch_hdrdir)/ruby/config.h coderange.o: $(hdrdir)/ruby/assert.h diff --git a/include/ruby/internal/intern/string.h b/include/ruby/internal/intern/string.h index 952dc508c24a67..cfe0454ee88562 100644 --- a/include/ruby/internal/intern/string.h +++ b/include/ruby/internal/intern/string.h @@ -601,6 +601,21 @@ VALUE rb_str_dup(VALUE str); */ VALUE rb_str_resurrect(VALUE str); +/** + * Returns whether a string is chilled or not. + * + * This function is temporary and users must check for its presence using + * #ifdef HAVE_RB_STR_CHILLED_P. If HAVE_RB_STR_CHILLED_P is not defined, then + * strings can't be chilled. + * + * @param[in] str A string. + * @retval 1 The string is chilled. + * @retval 0 Otherwise. + */ +bool rb_str_chilled_p(VALUE str); + +#define HAVE_RB_STR_CHILLED_P 1 + /** * Obtains a "temporary lock" of the string. This advisory locking mechanism * prevents other cooperating threads from tampering the receiver. The same diff --git a/string.c b/string.c index 9d84b16a073ec8..5c29718dff8647 100644 --- a/string.c +++ b/string.c @@ -1833,6 +1833,12 @@ rb_ec_str_resurrect(struct rb_execution_context_struct *ec, VALUE str, bool chil return new_str; } +bool +rb_str_chilled_p(VALUE str) +{ + return CHILLED_STRING_P(str); +} + /* * * call-seq: diff --git a/test/-ext-/string/test_chilled.rb b/test/-ext-/string/test_chilled.rb new file mode 100644 index 00000000000000..dccab61cede766 --- /dev/null +++ b/test/-ext-/string/test_chilled.rb @@ -0,0 +1,19 @@ +require 'test/unit' +require '-test-/string' + +class Test_String_ChilledString < Test::Unit::TestCase + def test_rb_str_chilled_p + str = "" + assert_equal true, Bug::String.rb_str_chilled_p(str) + end + + def test_rb_str_chilled_p_frozen + str = "".freeze + assert_equal false, Bug::String.rb_str_chilled_p(str) + end + + def test_rb_str_chilled_p_mutable + str = "".dup + assert_equal false, Bug::String.rb_str_chilled_p(str) + end +end From e9152bc9da39ad864b755e7bcc682610f8035a98 Mon Sep 17 00:00:00 2001 From: Andrew Konchin Date: Mon, 19 Feb 2024 12:58:20 +0200 Subject: [PATCH 065/211] [ruby/prism] Enable ParametersSignatureTest on TruffleRuby https://github.com/ruby/prism/commit/c7a7af1eac --- test/prism/parameters_signature_test.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/prism/parameters_signature_test.rb b/test/prism/parameters_signature_test.rb index 788ce7b90712e4..0eed8d993d52c9 100644 --- a/test/prism/parameters_signature_test.rb +++ b/test/prism/parameters_signature_test.rb @@ -2,7 +2,7 @@ require_relative "test_helper" -return if RUBY_VERSION < "3.2" || RUBY_ENGINE == "truffleruby" +return if RUBY_VERSION < "3.2" module Prism class ParametersSignatureTest < TestCase @@ -55,6 +55,8 @@ def test_keyrest_anonymous end def test_key_ordering + omit("TruffleRuby returns keys in order they were declared") if RUBY_ENGINE == "truffleruby" + assert_parameters([[:keyreq, :a], [:keyreq, :b], [:key, :c], [:key, :d]], "a:, c: 1, b:, d: 2") end From 19752cf4aa227106212be129507ac1bf339b26c0 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 21 Mar 2024 16:22:46 -0400 Subject: [PATCH 066/211] Use macro SET_WHEN_UINT --- debug.c | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/debug.c b/debug.c index e194472c1aaf9d..546ffc440f155b 100644 --- a/debug.c +++ b/debug.c @@ -219,7 +219,11 @@ ruby_env_debug_option(const char *str, int len, void *arg) } \ } while (0) #define SET_WHEN_UINT(name, vals, num, req) \ - if (NAME_MATCH_VALUE(name)) SET_UINT_LIST(name, vals, num); + if (NAME_MATCH_VALUE(name)) { \ + if (!len) req; \ + else SET_UINT_LIST(name, vals, num); \ + return 1; \ + } if (NAME_MATCH("gc_stress")) { rb_gc_stress_set(Qtrue); @@ -227,22 +231,15 @@ ruby_env_debug_option(const char *str, int len, void *arg) } SET_WHEN("core", ruby_enable_coredump, 1); SET_WHEN("ci", ruby_on_ci, 1); - if (NAME_MATCH_VALUE("rgengc")) { - if (!len) ruby_rgengc_debug = 1; - else SET_UINT_LIST("rgengc", &ruby_rgengc_debug, 1); - return 1; - } + SET_WHEN_UINT("rgengc", &ruby_rgengc_debug, 1, ruby_rgengc_debug = 1); #if defined _WIN32 # if RUBY_MSVCRT_VERSION >= 80 SET_WHEN("rtc_error", ruby_w32_rtc_error, 1); # endif #endif #if defined _WIN32 || defined __CYGWIN__ - if (NAME_MATCH_VALUE("codepage")) { - if (!len) fprintf(stderr, "missing codepage argument"); - else SET_UINT_LIST("codepage", ruby_w32_codepage, numberof(ruby_w32_codepage)); - return 1; - } + SET_WHEN_UINT("codepage", ruby_w32_codepage, numberof(ruby_w32_codepage), + fprintf(stderr, "missing codepage argument")); #endif return 0; } From 240fb3957b02cb9a1cc0d1e01ae40db803390bed Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 25 Mar 2024 15:32:54 -0400 Subject: [PATCH 067/211] [ruby/prism] Freeze internal parts, again https://github.com/ruby/prism/commit/50372fee5c --- lib/prism/node_ext.rb | 1 + prism/config.yml | 10 ++ prism/prism.c | 115 ++++++++++++------ test/prism/snapshots/alias.txt | 2 +- test/prism/snapshots/dash_heredocs.txt | 20 +-- test/prism/snapshots/dos_endings.txt | 10 +- test/prism/snapshots/dstring.txt | 8 +- .../snapshots/heredocs_leading_whitespace.txt | 10 +- test/prism/snapshots/heredocs_nested.txt | 16 ++- .../heredocs_with_ignored_newlines.txt | 19 +-- test/prism/snapshots/lambda.txt | 3 +- test/prism/snapshots/method_calls.txt | 5 +- test/prism/snapshots/methods.txt | 3 +- test/prism/snapshots/modules.txt | 5 +- test/prism/snapshots/regex.txt | 8 +- .../prism/snapshots/seattlerb/difficult0_.txt | 5 +- test/prism/snapshots/seattlerb/dstr_evstr.txt | 1 + .../snapshots/seattlerb/dstr_lex_state.txt | 1 + test/prism/snapshots/seattlerb/dstr_str.txt | 3 +- .../prism/snapshots/seattlerb/evstr_evstr.txt | 1 + test/prism/snapshots/seattlerb/evstr_str.txt | 3 +- .../snapshots/seattlerb/heredoc_nested.txt | 5 +- .../snapshots/seattlerb/heredoc_squiggly.txt | 7 +- ...squiggly_blank_line_plus_interpolation.txt | 5 +- .../heredoc_squiggly_blank_lines.txt | 7 +- .../seattlerb/heredoc_squiggly_interp.txt | 9 +- .../seattlerb/heredoc_squiggly_tabs.txt | 5 +- .../seattlerb/heredoc_squiggly_tabs_extra.txt | 5 +- .../heredoc_squiggly_visually_blank_lines.txt | 7 +- ...erpolation_and_carriage_return_escapes.txt | 5 +- ...on_and_carriage_return_escapes_windows.txt | 5 +- .../parse_line_dstr_escaped_newline.txt | 3 +- .../parse_line_dstr_soft_newline.txt | 3 +- .../parse_line_evstr_after_break.txt | 4 +- .../seattlerb/parse_line_heredoc_evstr.txt | 5 +- .../seattlerb/pct_w_heredoc_interp_nested.txt | 1 + .../snapshots/seattlerb/qsymbols_interp.txt | 2 +- test/prism/snapshots/seattlerb/str_evstr.txt | 3 +- .../snapshots/seattlerb/str_evstr_escape.txt | 5 +- .../seattlerb/str_heredoc_interp.txt | 3 +- .../seattlerb/str_interp_ternary_or_label.txt | 1 + .../str_lit_concat_bad_encodings.txt | 5 +- .../snapshots/seattlerb/str_pct_Q_nested.txt | 5 +- .../seattlerb/str_pct_nested_nested.txt | 6 +- test/prism/snapshots/seattlerb/str_str.txt | 3 +- .../prism/snapshots/seattlerb/str_str_str.txt | 5 +- .../snapshots/seattlerb/words_interp.txt | 3 +- test/prism/snapshots/spanning_heredoc.txt | 40 +++--- test/prism/snapshots/strings.txt | 23 ++-- test/prism/snapshots/symbols.txt | 10 +- test/prism/snapshots/tilde_heredocs.txt | 97 +++++++++------ test/prism/snapshots/undef.txt | 2 +- .../unparser/corpus/literal/assignment.txt | 25 ++-- .../snapshots/unparser/corpus/literal/def.txt | 5 +- .../unparser/corpus/literal/dstr.txt | 47 ++++--- .../unparser/corpus/literal/literal.txt | 66 ++++++---- .../unparser/corpus/semantic/dstr.txt | 114 ++++++++++------- .../unparser/corpus/semantic/literal.txt | 2 +- .../whitequark/array_symbols_interp.txt | 2 +- .../whitequark/array_words_interp.txt | 4 +- test/prism/snapshots/whitequark/bug_435.txt | 1 + test/prism/snapshots/whitequark/bug_466.txt | 1 + test/prism/snapshots/whitequark/bug_473.txt | 1 + test/prism/snapshots/whitequark/bug_480.txt | 1 + .../whitequark/bug_interp_single.txt | 2 + .../whitequark/dedenting_heredoc.txt | 59 +++++---- ...olating_heredoc_fake_line_continuation.txt | 5 +- ...nterpolating_heredoc_line_continuation.txt | 5 +- .../endless_method_command_syntax.txt | 6 +- .../whitequark/non_lvar_injecting_match.txt | 2 +- .../snapshots/whitequark/parser_bug_640.txt | 5 +- ...ps_truncated_parts_of_squiggly_heredoc.txt | 3 +- .../snapshots/whitequark/regex_interp.txt | 4 +- .../snapshots/whitequark/ruby_bug_11990.txt | 5 +- .../whitequark/slash_newline_in_heredocs.txt | 7 +- .../snapshots/whitequark/string_concat.txt | 6 +- .../snapshots/whitequark/string_dvar.txt | 5 +- .../snapshots/whitequark/string_interp.txt | 5 +- .../snapshots/whitequark/symbol_interp.txt | 4 +- test/prism/snapshots/whitequark/undef.txt | 2 +- .../snapshots/whitequark/xstring_interp.txt | 4 +- test/prism/snapshots/xstring.txt | 4 +- 82 files changed, 584 insertions(+), 356 deletions(-) diff --git a/lib/prism/node_ext.rb b/lib/prism/node_ext.rb index 4ec7c3014cb464..86745440659680 100644 --- a/lib/prism/node_ext.rb +++ b/lib/prism/node_ext.rb @@ -55,6 +55,7 @@ class StringNode < Node def to_interpolated InterpolatedStringNode.new( source, + frozen? ? InterpolatedStringNodeFlags::FROZEN : 0, opening_loc, [copy(opening_loc: nil, closing_loc: nil, location: content_loc)], closing_loc, diff --git a/prism/config.yml b/prism/config.yml index d9e39460d12f2c..bb1dd6e6b02ec0 100644 --- a/prism/config.yml +++ b/prism/config.yml @@ -626,6 +626,13 @@ flags: - name: HEXADECIMAL comment: "0x prefix" comment: Flags for integer nodes that correspond to the base of the integer. + - name: InterpolatedStringNodeFlags + values: + - name: FROZEN + comment: "frozen by virtue of a `frozen_string_literal: true` comment or `--enable-frozen-string-literal`" + - name: MUTABLE + comment: "mutable by virtue of a `frozen_string_literal: false` comment or `--disable-frozen-string-literal`" + comment: Flags for interpolated string nodes that indicated mutability if they are also marked as literals. - name: KeywordHashNodeFlags values: - name: SYMBOL_KEYS @@ -2260,6 +2267,9 @@ nodes: ^^^^^^^^^^^^^^^^ - name: InterpolatedStringNode fields: + - name: flags + type: flags + kind: InterpolatedStringNodeFlags - name: opening_loc type: location? - name: parts diff --git a/prism/prism.c b/prism/prism.c index 169f9c26d64028..e97af236769156 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -4281,6 +4281,7 @@ pm_interpolated_regular_expression_node_create(pm_parser_t *parser, const pm_tok *node = (pm_interpolated_regular_expression_node_t) { { .type = PM_INTERPOLATED_REGULAR_EXPRESSION_NODE, + .flags = PM_NODE_FLAG_STATIC_LITERAL, .location = { .start = opening->start, .end = NULL, @@ -4302,6 +4303,15 @@ pm_interpolated_regular_expression_node_append(pm_interpolated_regular_expressio if (node->base.location.end < part->location.end) { node->base.location.end = part->location.end; } + + if (PM_NODE_TYPE_P(part, PM_STRING_NODE)) { + pm_node_flag_set(part, PM_NODE_FLAG_STATIC_LITERAL | PM_STRING_FLAGS_FROZEN); + } + + if (!PM_NODE_FLAG_P(part, PM_NODE_FLAG_STATIC_LITERAL)) { + pm_node_flag_unset((pm_node_t *) node, PM_NODE_FLAG_STATIC_LITERAL); + } + pm_node_list_append(&node->parts, part); } @@ -4312,6 +4322,38 @@ pm_interpolated_regular_expression_node_closing_set(pm_parser_t *parser, pm_inte pm_node_flag_set((pm_node_t *)node, pm_regular_expression_flags_create(parser, closing)); } +/** + * Append a part to an InterpolatedStringNode node. + */ +static inline void +pm_interpolated_string_node_append(pm_parser_t *parser, pm_interpolated_string_node_t *node, pm_node_t *part) { + if (node->parts.size == 0 && node->opening_loc.start == NULL) { + node->base.location.start = part->location.start; + } + + if (PM_NODE_TYPE_P(part, PM_STRING_NODE)) { + pm_node_flag_set(part, PM_NODE_FLAG_STATIC_LITERAL | PM_STRING_FLAGS_FROZEN); + } + + if (!PM_NODE_FLAG_P(part, PM_NODE_FLAG_STATIC_LITERAL)) { + pm_node_flag_unset((pm_node_t *) node, PM_NODE_FLAG_STATIC_LITERAL | PM_INTERPOLATED_STRING_NODE_FLAGS_FROZEN | PM_INTERPOLATED_STRING_NODE_FLAGS_MUTABLE); + } + + pm_node_list_append(&node->parts, part); + node->base.location.end = MAX(node->base.location.end, part->location.end); + + if (PM_NODE_FLAG_P(node, PM_NODE_FLAG_STATIC_LITERAL)) { + switch (parser->frozen_string_literal) { + case PM_OPTIONS_FROZEN_STRING_LITERAL_DISABLED: + pm_node_flag_set((pm_node_t *) node, PM_INTERPOLATED_STRING_NODE_FLAGS_FROZEN); + break; + case PM_OPTIONS_FROZEN_STRING_LITERAL_ENABLED: + pm_node_flag_set((pm_node_t *) node, PM_INTERPOLATED_STRING_NODE_FLAGS_MUTABLE); + break; + } + } +} + /** * Allocate and initialize a new InterpolatedStringNode node. */ @@ -4322,6 +4364,7 @@ pm_interpolated_string_node_create(pm_parser_t *parser, const pm_token_t *openin *node = (pm_interpolated_string_node_t) { { .type = PM_INTERPOLATED_STRING_NODE, + .flags = PM_NODE_FLAG_STATIC_LITERAL, .location = { .start = opening->start, .end = closing->end, @@ -4333,25 +4376,14 @@ pm_interpolated_string_node_create(pm_parser_t *parser, const pm_token_t *openin }; if (parts != NULL) { - node->parts = *parts; + for (size_t index = 0; index < parts->size; index++) { + pm_interpolated_string_node_append(parser, node, parts->nodes[index]); + } } return node; } -/** - * Append a part to an InterpolatedStringNode node. - */ -static inline void -pm_interpolated_string_node_append(pm_interpolated_string_node_t *node, pm_node_t *part) { - if (node->parts.size == 0 && node->opening_loc.start == NULL) { - node->base.location.start = part->location.start; - } - - pm_node_list_append(&node->parts, part); - node->base.location.end = part->location.end; -} - /** * Set the closing token of the given InterpolatedStringNode node. */ @@ -4361,6 +4393,24 @@ pm_interpolated_string_node_closing_set(pm_interpolated_string_node_t *node, con node->base.location.end = closing->end; } +static void +pm_interpolated_symbol_node_append(pm_interpolated_symbol_node_t *node, pm_node_t *part) { + if (node->parts.size == 0 && node->opening_loc.start == NULL) { + node->base.location.start = part->location.start; + } + + if (PM_NODE_TYPE_P(part, PM_STRING_NODE)) { + pm_node_flag_set(part, PM_NODE_FLAG_STATIC_LITERAL | PM_STRING_FLAGS_FROZEN); + } + + if (!PM_NODE_FLAG_P(part, PM_NODE_FLAG_STATIC_LITERAL)) { + pm_node_flag_unset((pm_node_t *) node, PM_NODE_FLAG_STATIC_LITERAL); + } + + pm_node_list_append(&node->parts, part); + node->base.location.end = MAX(node->base.location.end, part->location.end); +} + /** * Allocate and initialize a new InterpolatedSymbolNode node. */ @@ -4371,6 +4421,7 @@ pm_interpolated_symbol_node_create(pm_parser_t *parser, const pm_token_t *openin *node = (pm_interpolated_symbol_node_t) { { .type = PM_INTERPOLATED_SYMBOL_NODE, + .flags = PM_NODE_FLAG_STATIC_LITERAL, .location = { .start = opening->start, .end = closing->end, @@ -4382,22 +4433,14 @@ pm_interpolated_symbol_node_create(pm_parser_t *parser, const pm_token_t *openin }; if (parts != NULL) { - node->parts = *parts; + for (size_t index = 0; index < parts->size; index++) { + pm_interpolated_symbol_node_append(node, parts->nodes[index]); + } } return node; } -static inline void -pm_interpolated_symbol_node_append(pm_interpolated_symbol_node_t *node, pm_node_t *part) { - if (node->parts.size == 0 && node->opening_loc.start == NULL) { - node->base.location.start = part->location.start; - } - - pm_node_list_append(&node->parts, part); - node->base.location.end = part->location.end; -} - /** * Allocate a new InterpolatedXStringNode node. */ @@ -4423,6 +4466,10 @@ pm_interpolated_xstring_node_create(pm_parser_t *parser, const pm_token_t *openi static inline void pm_interpolated_xstring_node_append(pm_interpolated_x_string_node_t *node, pm_node_t *part) { + if (PM_NODE_TYPE_P(part, PM_STRING_NODE)) { + pm_node_flag_set(part, PM_NODE_FLAG_STATIC_LITERAL | PM_STRING_FLAGS_FROZEN); + } + pm_node_list_append(&node->parts, part); node->base.location.end = part->location.end; } @@ -15427,11 +15474,11 @@ parse_strings(pm_parser_t *parser, pm_node_t *current) { pm_token_t bounds = not_provided(parser); pm_interpolated_string_node_t *container = pm_interpolated_string_node_create(parser, &bounds, NULL, &bounds); - pm_interpolated_string_node_append(container, current); + pm_interpolated_string_node_append(parser, container, current); current = (pm_node_t *) container; } - pm_interpolated_string_node_append((pm_interpolated_string_node_t *) current, node); + pm_interpolated_string_node_append(parser, (pm_interpolated_string_node_t *) current, node); } } @@ -17365,15 +17412,15 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b // If we hit string content and the current node is // an interpolated string, then we need to append // the string content to the list of child nodes. - pm_interpolated_string_node_append((pm_interpolated_string_node_t *) current, string); + pm_interpolated_string_node_append(parser, (pm_interpolated_string_node_t *) current, string); } else if (PM_NODE_TYPE_P(current, PM_STRING_NODE)) { // If we hit string content and the current node is // a string node, then we need to convert the // current node into an interpolated string and add // the string content to the list of child nodes. pm_interpolated_string_node_t *interpolated = pm_interpolated_string_node_create(parser, &opening, NULL, &closing); - pm_interpolated_string_node_append(interpolated, current); - pm_interpolated_string_node_append(interpolated, string); + pm_interpolated_string_node_append(parser, interpolated, current); + pm_interpolated_string_node_append(parser, interpolated, string); current = (pm_node_t *) interpolated; } else { assert(false && "unreachable"); @@ -17398,7 +17445,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_token_t opening = not_provided(parser); pm_token_t closing = not_provided(parser); pm_interpolated_string_node_t *interpolated = pm_interpolated_string_node_create(parser, &opening, NULL, &closing); - pm_interpolated_string_node_append(interpolated, current); + pm_interpolated_string_node_append(parser, interpolated, current); current = (pm_node_t *) interpolated; } else { // If we hit an embedded variable and the current @@ -17407,7 +17454,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b } pm_node_t *part = parse_string_part(parser); - pm_interpolated_string_node_append((pm_interpolated_string_node_t *) current, part); + pm_interpolated_string_node_append(parser, (pm_interpolated_string_node_t *) current, part); break; } case PM_TOKEN_EMBEXPR_BEGIN: { @@ -17427,7 +17474,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_token_t opening = not_provided(parser); pm_token_t closing = not_provided(parser); pm_interpolated_string_node_t *interpolated = pm_interpolated_string_node_create(parser, &opening, NULL, &closing); - pm_interpolated_string_node_append(interpolated, current); + pm_interpolated_string_node_append(parser, interpolated, current); current = (pm_node_t *) interpolated; } else if (PM_NODE_TYPE_P(current, PM_INTERPOLATED_STRING_NODE)) { // If we hit an embedded expression and the current @@ -17438,7 +17485,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b } pm_node_t *part = parse_string_part(parser); - pm_interpolated_string_node_append((pm_interpolated_string_node_t *) current, part); + pm_interpolated_string_node_append(parser, (pm_interpolated_string_node_t *) current, part); break; } default: diff --git a/test/prism/snapshots/alias.txt b/test/prism/snapshots/alias.txt index 687082d85e2e31..a952e96f67721f 100644 --- a/test/prism/snapshots/alias.txt +++ b/test/prism/snapshots/alias.txt @@ -57,7 +57,7 @@ │ │ ├── opening_loc: (7,6)-(7,8) = ":\"" │ │ ├── parts: (length: 2) │ │ │ ├── @ StringNode (location: (7,8)-(7,11)) - │ │ │ │ ├── flags: ∅ + │ │ │ │ ├── flags: frozen │ │ │ │ ├── opening_loc: ∅ │ │ │ │ ├── content_loc: (7,8)-(7,11) = "abc" │ │ │ │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/dash_heredocs.txt b/test/prism/snapshots/dash_heredocs.txt index 9af3acf9c2e6e8..bd2b05ea0d0717 100644 --- a/test/prism/snapshots/dash_heredocs.txt +++ b/test/prism/snapshots/dash_heredocs.txt @@ -79,10 +79,11 @@ │ ├── closing_loc: (23,0)-(24,0) = " EOF\n" │ └── unescaped: " a\n b\n" ├── @ InterpolatedStringNode (location: (25,0)-(25,8)) + │ ├── flags: ∅ │ ├── opening_loc: (25,0)-(25,8) = "<<-\"EOF\"" │ ├── parts: (length: 3) │ │ ├── @ StringNode (location: (26,0)-(27,0)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (26,0)-(27,0) = " a\n" │ │ │ ├── closing_loc: ∅ @@ -104,17 +105,18 @@ │ │ │ │ └── block: ∅ │ │ │ └── closing_loc: (27,3)-(27,4) = "}" │ │ └── @ StringNode (location: (27,4)-(28,0)) - │ │ ├── flags: ∅ + │ │ ├── flags: frozen │ │ ├── opening_loc: ∅ │ │ ├── content_loc: (27,4)-(28,0) = "\n" │ │ ├── closing_loc: ∅ │ │ └── unescaped: "\n" │ └── closing_loc: (28,0)-(29,0) = "EOF\n" ├── @ InterpolatedStringNode (location: (30,0)-(30,6)) + │ ├── flags: ∅ │ ├── opening_loc: (30,0)-(30,6) = "<<-EOF" │ ├── parts: (length: 3) │ │ ├── @ StringNode (location: (31,0)-(32,0)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (31,0)-(32,0) = " a\n" │ │ │ ├── closing_loc: ∅ @@ -136,7 +138,7 @@ │ │ │ │ └── block: ∅ │ │ │ └── closing_loc: (32,3)-(32,4) = "}" │ │ └── @ StringNode (location: (32,4)-(33,0)) - │ │ ├── flags: ∅ + │ │ ├── flags: frozen │ │ ├── opening_loc: ∅ │ │ ├── content_loc: (32,4)-(33,0) = "\n" │ │ ├── closing_loc: ∅ @@ -184,10 +186,11 @@ │ │ ├── flags: ∅ │ │ └── arguments: (length: 1) │ │ └── @ InterpolatedStringNode (location: (49,7)-(49,11)) + │ │ ├── flags: ∅ │ │ ├── opening_loc: (49,7)-(49,11) = "<<-B" │ │ ├── parts: (length: 3) │ │ │ ├── @ StringNode (location: (52,0)-(53,2)) - │ │ │ │ ├── flags: ∅ + │ │ │ │ ├── flags: frozen │ │ │ │ ├── opening_loc: ∅ │ │ │ │ ├── content_loc: (52,0)-(53,2) = " b\n " │ │ │ │ ├── closing_loc: ∅ @@ -202,7 +205,7 @@ │ │ │ │ │ └── value: 2 │ │ │ │ └── closing_loc: (54,2)-(54,3) = "}" │ │ │ └── @ StringNode (location: (54,3)-(55,0)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (54,3)-(55,0) = "\n" │ │ │ ├── closing_loc: ∅ @@ -228,10 +231,11 @@ │ ├── flags: ∅ │ └── arguments: (length: 1) │ └── @ InterpolatedStringNode (location: (57,7)-(57,11)) + │ ├── flags: ∅ │ ├── opening_loc: (57,7)-(57,11) = "<<-B" │ ├── parts: (length: 3) │ │ ├── @ StringNode (location: (60,0)-(61,2)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (60,0)-(61,2) = " b\n " │ │ │ ├── closing_loc: ∅ @@ -246,7 +250,7 @@ │ │ │ │ └── value: 2 │ │ │ └── closing_loc: (62,3)-(62,4) = "}" │ │ └── @ StringNode (location: (62,4)-(63,0)) - │ │ ├── flags: ∅ + │ │ ├── flags: frozen │ │ ├── opening_loc: ∅ │ │ ├── content_loc: (62,4)-(63,0) = "\n" │ │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/dos_endings.txt b/test/prism/snapshots/dos_endings.txt index c5b962f2181cf3..1ae15e1e87c081 100644 --- a/test/prism/snapshots/dos_endings.txt +++ b/test/prism/snapshots/dos_endings.txt @@ -15,16 +15,17 @@ │ │ ├── flags: ∅ │ │ └── arguments: (length: 1) │ │ └── @ InterpolatedStringNode (location: (1,5)-(2,12)) + │ │ ├── flags: ∅ │ │ ├── opening_loc: ∅ │ │ ├── parts: (length: 2) │ │ │ ├── @ StringNode (location: (1,5)-(1,9)) - │ │ │ │ ├── flags: ∅ + │ │ │ │ ├── flags: frozen │ │ │ │ ├── opening_loc: (1,5)-(1,6) = "\"" │ │ │ │ ├── content_loc: (1,6)-(1,8) = "hi" │ │ │ │ ├── closing_loc: (1,8)-(1,9) = "\"" │ │ │ │ └── unescaped: "hi" │ │ │ └── @ StringNode (location: (2,5)-(2,12)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: (2,5)-(2,6) = "\"" │ │ │ ├── content_loc: (2,6)-(2,11) = "there" │ │ │ ├── closing_loc: (2,11)-(2,12) = "\"" @@ -81,16 +82,17 @@ │ │ ├── flags: ∅ │ │ ├── receiver: │ │ │ @ InterpolatedStringNode (location: (17,8)-(17,14)) + │ │ │ ├── flags: ∅ │ │ │ ├── opening_loc: (17,8)-(17,14) = "<<~EOF" │ │ │ ├── parts: (length: 2) │ │ │ │ ├── @ StringNode (location: (18,0)-(19,0)) - │ │ │ │ │ ├── flags: ∅ + │ │ │ │ │ ├── flags: frozen │ │ │ │ │ ├── opening_loc: ∅ │ │ │ │ │ ├── content_loc: (18,0)-(19,0) = "\r\n" │ │ │ │ │ ├── closing_loc: ∅ │ │ │ │ │ └── unescaped: "\n" │ │ │ │ └── @ StringNode (location: (19,0)-(20,0)) - │ │ │ │ ├── flags: ∅ + │ │ │ │ ├── flags: frozen │ │ │ │ ├── opening_loc: ∅ │ │ │ │ ├── content_loc: (19,0)-(20,0) = " baz\r\n" │ │ │ │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/dstring.txt b/test/prism/snapshots/dstring.txt index ad395f8a8eb94b..3978a2f0879e50 100644 --- a/test/prism/snapshots/dstring.txt +++ b/test/prism/snapshots/dstring.txt @@ -10,10 +10,11 @@ │ ├── closing_loc: (2,5)-(2,6) = "\"" │ └── unescaped: "foo\n bar" ├── @ InterpolatedStringNode (location: (4,0)-(5,9)) + │ ├── flags: ∅ │ ├── opening_loc: (4,0)-(4,1) = "\"" │ ├── parts: (length: 2) │ │ ├── @ StringNode (location: (4,1)-(5,2)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (4,1)-(5,2) = "foo\n " │ │ │ ├── closing_loc: ∅ @@ -36,16 +37,17 @@ │ │ └── closing_loc: (5,7)-(5,8) = "}" │ └── closing_loc: (5,8)-(5,9) = "\"" ├── @ InterpolatedStringNode (location: (7,0)-(9,2)) + │ ├── flags: ∅ │ ├── opening_loc: ∅ │ ├── parts: (length: 2) │ │ ├── @ StringNode (location: (7,0)-(8,2)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: (7,0)-(7,1) = "\"" │ │ │ ├── content_loc: (7,1)-(8,1) = "fo\no" │ │ │ ├── closing_loc: (8,1)-(8,2) = "\"" │ │ │ └── unescaped: "fo\no" │ │ └── @ StringNode (location: (8,3)-(9,2)) - │ │ ├── flags: ∅ + │ │ ├── flags: frozen │ │ ├── opening_loc: (8,3)-(8,4) = "\"" │ │ ├── content_loc: (8,4)-(9,1) = "ba\nr" │ │ ├── closing_loc: (9,1)-(9,2) = "\"" diff --git a/test/prism/snapshots/heredocs_leading_whitespace.txt b/test/prism/snapshots/heredocs_leading_whitespace.txt index 332dfa29862d0d..dbcb0d5a19aed9 100644 --- a/test/prism/snapshots/heredocs_leading_whitespace.txt +++ b/test/prism/snapshots/heredocs_leading_whitespace.txt @@ -28,32 +28,34 @@ │ ├── closing_loc: (19,0)-(20,0) = " FOO\n" │ └── unescaped: "a\nb\n" ├── @ InterpolatedStringNode (location: (21,0)-(21,10)) + │ ├── flags: ∅ │ ├── opening_loc: (21,0)-(21,10) = "<<~' FOO'" │ ├── parts: (length: 2) │ │ ├── @ StringNode (location: (22,0)-(23,0)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (22,0)-(23,0) = "a\n" │ │ │ ├── closing_loc: ∅ │ │ │ └── unescaped: "a\n" │ │ └── @ StringNode (location: (23,0)-(24,0)) - │ │ ├── flags: ∅ + │ │ ├── flags: frozen │ │ ├── opening_loc: ∅ │ │ ├── content_loc: (23,0)-(24,0) = "b\n" │ │ ├── closing_loc: ∅ │ │ └── unescaped: "b\n" │ └── closing_loc: (24,0)-(25,0) = " FOO\n" └── @ InterpolatedStringNode (location: (26,0)-(26,10)) + ├── flags: ∅ ├── opening_loc: (26,0)-(26,10) = "<<~' FOO'" ├── parts: (length: 2) │ ├── @ StringNode (location: (27,0)-(28,0)) - │ │ ├── flags: ∅ + │ │ ├── flags: frozen │ │ ├── opening_loc: ∅ │ │ ├── content_loc: (27,0)-(28,0) = "a\n" │ │ ├── closing_loc: ∅ │ │ └── unescaped: "a\n" │ └── @ StringNode (location: (28,0)-(29,0)) - │ ├── flags: ∅ + │ ├── flags: frozen │ ├── opening_loc: ∅ │ ├── content_loc: (28,0)-(29,0) = "b\n" │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/heredocs_nested.txt b/test/prism/snapshots/heredocs_nested.txt index 1717aadbbce9c8..f830b028c78622 100644 --- a/test/prism/snapshots/heredocs_nested.txt +++ b/test/prism/snapshots/heredocs_nested.txt @@ -4,10 +4,11 @@ @ StatementsNode (location: (1,0)-(12,4)) └── body: (length: 2) ├── @ InterpolatedStringNode (location: (1,0)-(1,7)) + │ ├── flags: ∅ │ ├── opening_loc: (1,0)-(1,7) = "<<~RUBY" │ ├── parts: (length: 4) │ │ ├── @ StringNode (location: (2,0)-(3,0)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (2,0)-(3,0) = "pre\n" │ │ │ ├── closing_loc: ∅ @@ -25,19 +26,20 @@ │ │ │ │ └── unescaped: " hello\n" │ │ │ └── closing_loc: (7,0)-(7,1) = "}" │ │ ├── @ StringNode (location: (7,1)-(8,0)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (7,1)-(8,0) = "\n" │ │ │ ├── closing_loc: ∅ │ │ │ └── unescaped: "\n" │ │ └── @ StringNode (location: (8,0)-(9,0)) - │ │ ├── flags: ∅ + │ │ ├── flags: frozen │ │ ├── opening_loc: ∅ │ │ ├── content_loc: (8,0)-(9,0) = "post\n" │ │ ├── closing_loc: ∅ │ │ └── unescaped: "post\n" │ └── closing_loc: (9,0)-(10,0) = "RUBY\n" └── @ InterpolatedStringNode (location: (12,0)-(12,4)) + ├── flags: ∅ ├── opening_loc: (12,0)-(12,4) = "<<-A" ├── parts: (length: 2) │ ├── @ EmbeddedStatementsNode (location: (13,0)-(21,1)) @@ -46,6 +48,7 @@ │ │ │ @ StatementsNode (location: (14,0)-(14,4)) │ │ │ └── body: (length: 1) │ │ │ └── @ InterpolatedStringNode (location: (14,0)-(14,4)) + │ │ │ ├── flags: ∅ │ │ │ ├── opening_loc: (14,0)-(14,4) = "<<-B" │ │ │ ├── parts: (length: 2) │ │ │ │ ├── @ EmbeddedStatementsNode (location: (15,0)-(19,1)) @@ -54,6 +57,7 @@ │ │ │ │ │ │ @ StatementsNode (location: (16,0)-(16,4)) │ │ │ │ │ │ └── body: (length: 1) │ │ │ │ │ │ └── @ InterpolatedStringNode (location: (16,0)-(16,4)) + │ │ │ │ │ │ ├── flags: ∅ │ │ │ │ │ │ ├── opening_loc: (16,0)-(16,4) = "<<-C" │ │ │ │ │ │ ├── parts: (length: 2) │ │ │ │ │ │ │ ├── @ EmbeddedStatementsNode (location: (17,0)-(17,4)) @@ -66,7 +70,7 @@ │ │ │ │ │ │ │ │ │ └── value: 3 │ │ │ │ │ │ │ │ └── closing_loc: (17,3)-(17,4) = "}" │ │ │ │ │ │ │ └── @ StringNode (location: (17,4)-(18,0)) - │ │ │ │ │ │ │ ├── flags: ∅ + │ │ │ │ │ │ │ ├── flags: frozen │ │ │ │ │ │ │ ├── opening_loc: ∅ │ │ │ │ │ │ │ ├── content_loc: (17,4)-(18,0) = "\n" │ │ │ │ │ │ │ ├── closing_loc: ∅ @@ -74,7 +78,7 @@ │ │ │ │ │ │ └── closing_loc: (18,0)-(19,0) = "C\n" │ │ │ │ │ └── closing_loc: (19,0)-(19,1) = "}" │ │ │ │ └── @ StringNode (location: (19,1)-(20,0)) - │ │ │ │ ├── flags: ∅ + │ │ │ │ ├── flags: frozen │ │ │ │ ├── opening_loc: ∅ │ │ │ │ ├── content_loc: (19,1)-(20,0) = "\n" │ │ │ │ ├── closing_loc: ∅ @@ -82,7 +86,7 @@ │ │ │ └── closing_loc: (20,0)-(21,0) = "B\n" │ │ └── closing_loc: (21,0)-(21,1) = "}" │ └── @ StringNode (location: (21,1)-(22,0)) - │ ├── flags: ∅ + │ ├── flags: frozen │ ├── opening_loc: ∅ │ ├── content_loc: (21,1)-(22,0) = "\n" │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/heredocs_with_ignored_newlines.txt b/test/prism/snapshots/heredocs_with_ignored_newlines.txt index cdc0b4faab9279..0f964ec294185b 100644 --- a/test/prism/snapshots/heredocs_with_ignored_newlines.txt +++ b/test/prism/snapshots/heredocs_with_ignored_newlines.txt @@ -10,58 +10,59 @@ │ ├── closing_loc: (2,0)-(3,0) = "HERE\n" │ └── unescaped: "" └── @ InterpolatedStringNode (location: (4,0)-(4,8)) + ├── flags: ∅ ├── opening_loc: (4,0)-(4,8) = "<<~THERE" ├── parts: (length: 9) │ ├── @ StringNode (location: (5,0)-(6,0)) - │ │ ├── flags: ∅ + │ │ ├── flags: frozen │ │ ├── opening_loc: ∅ │ │ ├── content_loc: (5,0)-(6,0) = " way over\n" │ │ ├── closing_loc: ∅ │ │ └── unescaped: "way over\n" │ ├── @ StringNode (location: (6,0)-(7,0)) - │ │ ├── flags: ∅ + │ │ ├── flags: frozen │ │ ├── opening_loc: ∅ │ │ ├── content_loc: (6,0)-(7,0) = " < Date: Tue, 26 Mar 2024 09:33:03 -0400 Subject: [PATCH 068/211] [ruby/prism] Handle regexp split between heredocs https://github.com/ruby/prism/commit/c1400d8aed --- prism/prism.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/prism/prism.c b/prism/prism.c index e97af236769156..e02bceed657ad3 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -4345,10 +4345,10 @@ pm_interpolated_string_node_append(pm_parser_t *parser, pm_interpolated_string_n if (PM_NODE_FLAG_P(node, PM_NODE_FLAG_STATIC_LITERAL)) { switch (parser->frozen_string_literal) { case PM_OPTIONS_FROZEN_STRING_LITERAL_DISABLED: - pm_node_flag_set((pm_node_t *) node, PM_INTERPOLATED_STRING_NODE_FLAGS_FROZEN); + pm_node_flag_set((pm_node_t *) node, PM_INTERPOLATED_STRING_NODE_FLAGS_MUTABLE); break; case PM_OPTIONS_FROZEN_STRING_LITERAL_ENABLED: - pm_node_flag_set((pm_node_t *) node, PM_INTERPOLATED_STRING_NODE_FLAGS_MUTABLE); + pm_node_flag_set((pm_node_t *) node, PM_INTERPOLATED_STRING_NODE_FLAGS_FROZEN); break; } } @@ -10900,11 +10900,15 @@ parser_lex(pm_parser_t *parser) { // the list of newlines. if (parser->heredoc_end == NULL) { pm_newline_list_append(&parser->newline_list, breakpoint); + parser->current.end = breakpoint + 1; + breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, false); + break; } parser->current.end = breakpoint + 1; - breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, false); - break; + parser_flush_heredoc_end(parser); + pm_regexp_token_buffer_flush(parser, &token_buffer); + LEX(PM_TOKEN_STRING_CONTENT); case '\\': { // If we hit escapes, then we need to treat the next // token literally. In this case we'll skip past the From 8ec7c3ce3069177a9e8eb9ca93cced9aa0533085 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 26 Mar 2024 10:31:51 -0400 Subject: [PATCH 069/211] [ruby/prism] Properly handle freeing ephemeral node lists https://github.com/ruby/prism/commit/f49261a9b9 --- prism/prism.c | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/prism/prism.c b/prism/prism.c index e02bceed657ad3..4f65e1e0173e33 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -4411,6 +4411,12 @@ pm_interpolated_symbol_node_append(pm_interpolated_symbol_node_t *node, pm_node_ node->base.location.end = MAX(node->base.location.end, part->location.end); } +static void +pm_interpolated_symbol_node_closing_loc_set(pm_interpolated_symbol_node_t *node, const pm_token_t *closing) { + node->closing_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(closing); + node->base.location.end = closing->end; +} + /** * Allocate and initialize a new InterpolatedSymbolNode node. */ @@ -14151,14 +14157,12 @@ parse_symbol(pm_parser_t *parser, pm_lex_mode_t *lex_mode, pm_lex_state_t next_s return (pm_node_t *) pm_string_node_to_symbol_node(parser, (pm_string_node_t *) part, &opening, &parser->previous); } - // Create a node_list first. We'll use this to check if it should be an - // InterpolatedSymbolNode or a SymbolNode. - pm_node_list_t node_list = { 0 }; - if (part) pm_node_list_append(&node_list, part); + pm_interpolated_symbol_node_t *symbol = pm_interpolated_symbol_node_create(parser, &opening, NULL, &opening); + if (part) pm_interpolated_symbol_node_append(symbol, part); while (!match2(parser, PM_TOKEN_STRING_END, PM_TOKEN_EOF)) { if ((part = parse_string_part(parser)) != NULL) { - pm_node_list_append(&node_list, part); + pm_interpolated_symbol_node_append(symbol, part); } } @@ -14169,7 +14173,8 @@ parse_symbol(pm_parser_t *parser, pm_lex_mode_t *lex_mode, pm_lex_state_t next_s expect1(parser, PM_TOKEN_STRING_END, PM_ERR_SYMBOL_TERM_INTERPOLATED); } - return (pm_node_t *) pm_interpolated_symbol_node_create(parser, &opening, &node_list, &parser->previous); + pm_interpolated_symbol_node_closing_loc_set(symbol, &parser->previous); + return (pm_node_t *) symbol; } pm_token_t content; @@ -14190,14 +14195,14 @@ parse_symbol(pm_parser_t *parser, pm_lex_mode_t *lex_mode, pm_lex_state_t next_s // In this case, the best way we have to represent this is as an // interpolated string node, so that's what we'll do here. if (match1(parser, PM_TOKEN_STRING_CONTENT)) { - pm_node_list_t parts = { 0 }; + pm_interpolated_symbol_node_t *symbol = pm_interpolated_symbol_node_create(parser, &opening, NULL, &opening); pm_token_t bounds = not_provided(parser); pm_node_t *part = (pm_node_t *) pm_string_node_create_unescaped(parser, &bounds, &content, &bounds, &unescaped); - pm_node_list_append(&parts, part); + pm_interpolated_symbol_node_append(symbol, part); part = (pm_node_t *) pm_string_node_create_unescaped(parser, &bounds, &parser->current, &bounds, &parser->current_string); - pm_node_list_append(&parts, part); + pm_interpolated_symbol_node_append(symbol, part); if (next_state != PM_LEX_STATE_NONE) { lex_state_set(parser, next_state); @@ -14205,7 +14210,9 @@ parse_symbol(pm_parser_t *parser, pm_lex_mode_t *lex_mode, pm_lex_state_t next_s parser_lex(parser); expect1(parser, PM_TOKEN_STRING_END, PM_ERR_SYMBOL_TERM_DYNAMIC); - return (pm_node_t *) pm_interpolated_symbol_node_create(parser, &opening, &parts, &parser->previous); + + pm_interpolated_symbol_node_closing_loc_set(symbol, &parser->previous); + return (pm_node_t *) symbol; } } else { content = (pm_token_t) { .type = PM_TOKEN_STRING_CONTENT, .start = parser->previous.end, .end = parser->previous.end }; @@ -15373,6 +15380,8 @@ parse_strings(pm_parser_t *parser, pm_node_t *current) { expect1(parser, PM_TOKEN_STRING_END, PM_ERR_STRING_LITERAL_EOF); node = (pm_node_t *) pm_interpolated_string_node_create(parser, &opening, &parts, &parser->previous); + + pm_node_list_free(&parts); } else if (accept1(parser, PM_TOKEN_LABEL_END) && !state_is_arg_labeled) { node = (pm_node_t *) pm_symbol_node_create_unescaped(parser, &opening, &content, &parser->previous, &unescaped, parse_symbol_encoding(parser, &unescaped)); } else if (match1(parser, PM_TOKEN_EOF)) { @@ -15427,6 +15436,8 @@ parse_strings(pm_parser_t *parser, pm_node_t *current) { expect1(parser, PM_TOKEN_STRING_END, PM_ERR_STRING_INTERPOLATED_TERM); node = (pm_node_t *) pm_interpolated_string_node_create(parser, &opening, &parts, &parser->previous); } + + pm_node_list_free(&parts); } } else { // If we get here, then the first part of the string is not plain @@ -15450,6 +15461,8 @@ parse_strings(pm_parser_t *parser, pm_node_t *current) { expect1(parser, PM_TOKEN_STRING_END, PM_ERR_STRING_INTERPOLATED_TERM); node = (pm_node_t *) pm_interpolated_string_node_create(parser, &opening, &parts, &parser->previous); } + + pm_node_list_free(&parts); } if (current == NULL) { @@ -16058,6 +16071,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b node = (pm_node_t *) cast; } else { pm_interpolated_string_node_t *cast = pm_interpolated_string_node_create(parser, &opening, &parts, &opening); + pm_node_list_free(&parts); lex_mode_pop(parser); expect1(parser, PM_TOKEN_HEREDOC_END, PM_ERR_HEREDOC_TERM); From 4300c42a7eb12a0aa81e4ec79891cdba1cfdf3aa Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 26 Mar 2024 09:58:24 -0400 Subject: [PATCH 070/211] [PRISM] Better handle interpolated* nodes with inner frozen parts --- prism_compile.c | 367 ++++++++++++++++++++++++++++++------------------ 1 file changed, 234 insertions(+), 133 deletions(-) diff --git a/prism_compile.c b/prism_compile.c index e919a44c0885ae..db032e2a0530a5 100644 --- a/prism_compile.c +++ b/prism_compile.c @@ -129,7 +129,8 @@ parse_integer(const pm_integer_node_t *node) if (integer->values == NULL) { result = UINT2NUM(integer->value); - } else { + } + else { VALUE string = rb_str_new(NULL, integer->length * 8); unsigned char *bytes = (unsigned char *) RSTRING_PTR(string); @@ -251,9 +252,11 @@ parse_string_encoded(const pm_scope_node_t *scope_node, const pm_node_t *node, c if (node->flags & PM_ENCODING_FLAGS_FORCED_BINARY_ENCODING) { encoding = rb_ascii8bit_encoding(); - } else if (node->flags & PM_ENCODING_FLAGS_FORCED_UTF8_ENCODING) { + } + else if (node->flags & PM_ENCODING_FLAGS_FORCED_UTF8_ENCODING) { encoding = rb_utf8_encoding(); - } else { + } + else { encoding = scope_node->encoding; } @@ -335,21 +338,21 @@ pm_reg_flags(const pm_node_t *node) { } static rb_encoding * -pm_reg_enc(const pm_scope_node_t *scope_node, const pm_regular_expression_node_t *node) +pm_reg_enc(const pm_scope_node_t *scope_node, const pm_node_t *node) { - if (node->base.flags & PM_REGULAR_EXPRESSION_FLAGS_ASCII_8BIT) { + if (PM_NODE_FLAG_P(node, PM_REGULAR_EXPRESSION_FLAGS_ASCII_8BIT)) { return rb_ascii8bit_encoding(); } - if (node->base.flags & PM_REGULAR_EXPRESSION_FLAGS_EUC_JP) { + if (PM_NODE_FLAG_P(node, PM_REGULAR_EXPRESSION_FLAGS_EUC_JP)) { return rb_enc_get_from_index(ENCINDEX_EUC_JP); } - if (node->base.flags & PM_REGULAR_EXPRESSION_FLAGS_WINDOWS_31J) { + if (PM_NODE_FLAG_P(node, PM_REGULAR_EXPRESSION_FLAGS_WINDOWS_31J)) { return rb_enc_get_from_index(ENCINDEX_Windows_31J); } - if (node->base.flags & PM_REGULAR_EXPRESSION_FLAGS_UTF_8) { + if (PM_NODE_FLAG_P(node, PM_REGULAR_EXPRESSION_FLAGS_UTF_8)) { return rb_utf8_encoding(); } @@ -368,12 +371,12 @@ pm_static_literal_p(const pm_node_t *node) } static VALUE -pm_new_regex(const pm_scope_node_t *scope_node, const pm_regular_expression_node_t *node) +pm_new_regex(const pm_scope_node_t *scope_node, const pm_node_t *node, const pm_string_t *unescaped) { - VALUE regex_str = parse_string(scope_node, &node->unescaped); + VALUE regex_str = parse_string(scope_node, unescaped); rb_encoding *enc = pm_reg_enc(scope_node, node); - VALUE regex = rb_enc_reg_new(RSTRING_PTR(regex_str), RSTRING_LEN(regex_str), enc, pm_reg_flags((const pm_node_t *) node)); + VALUE regex = rb_enc_reg_new(RSTRING_PTR(regex_str), RSTRING_LEN(regex_str), enc, pm_reg_flags(node)); RB_GC_GUARD(regex_str); rb_obj_freeze(regex); @@ -381,12 +384,44 @@ pm_new_regex(const pm_scope_node_t *scope_node, const pm_regular_expression_node return regex; } +static VALUE +pm_static_literal_concat(const pm_node_list_t *nodes, const pm_scope_node_t *scope_node, bool top) +{ + VALUE current = Qnil; + + for (size_t index = 0; index < nodes->size; index++) { + const pm_node_t *part = nodes->nodes[index]; + VALUE string; + + switch (PM_NODE_TYPE(part)) { + case PM_STRING_NODE: + string = parse_string_encoded(scope_node, part, &((const pm_string_node_t *) part)->unescaped); + break; + case PM_INTERPOLATED_STRING_NODE: + string = pm_static_literal_concat(&((const pm_interpolated_string_node_t *) part)->parts, scope_node, false); + break; + default: + RUBY_ASSERT(false && "unexpected node type in pm_static_literal_concat"); + return Qnil; + } + + if (current != Qnil) { + current = rb_str_concat(current, string); + } + else { + current = string; + } + } + + return top ? rb_fstring(current) : current; +} + /** * Certain nodes can be compiled literally. This function returns the literal * value described by the given node. For example, an array node with all static * literal values can be compiled into a literal array. */ -static inline VALUE +static VALUE pm_static_literal_value(const pm_node_t *node, const pm_scope_node_t *scope_node) { // Every node that comes into this function should already be marked as @@ -433,12 +468,40 @@ pm_static_literal_value(const pm_node_t *node, const pm_scope_node_t *scope_node return parse_imaginary((pm_imaginary_node_t *) node); case PM_INTEGER_NODE: return parse_integer((const pm_integer_node_t *) node); + case PM_INTERPOLATED_MATCH_LAST_LINE_NODE: { + const pm_interpolated_match_last_line_node_t *cast = (const pm_interpolated_match_last_line_node_t *) node; + VALUE string = pm_static_literal_concat(&cast->parts, scope_node, true); + + rb_encoding *encoding = pm_reg_enc(scope_node, (const pm_node_t *) cast); + return rb_obj_freeze(rb_enc_reg_new(RSTRING_PTR(string), RSTRING_LEN(string), encoding, pm_reg_flags((const pm_node_t *) cast))); + } + case PM_INTERPOLATED_REGULAR_EXPRESSION_NODE: { + const pm_interpolated_regular_expression_node_t *cast = (const pm_interpolated_regular_expression_node_t *) node; + VALUE string = pm_static_literal_concat(&cast->parts, scope_node, true); + + rb_encoding *encoding = pm_reg_enc(scope_node, (const pm_node_t *) cast); + return rb_obj_freeze(rb_enc_reg_new(RSTRING_PTR(string), RSTRING_LEN(string), encoding, pm_reg_flags((const pm_node_t *) cast))); + } + case PM_INTERPOLATED_STRING_NODE: + return pm_static_literal_concat(&((const pm_interpolated_string_node_t *) node)->parts, scope_node, true); + case PM_INTERPOLATED_SYMBOL_NODE: { + const pm_interpolated_symbol_node_t *cast = (const pm_interpolated_symbol_node_t *) node; + VALUE string = pm_static_literal_concat(&cast->parts, scope_node, true); + + return ID2SYM(rb_intern_str(string)); + } + case PM_MATCH_LAST_LINE_NODE: { + const pm_match_last_line_node_t *cast = (const pm_match_last_line_node_t *) node; + return pm_new_regex(scope_node, (const pm_node_t *) cast, &cast->unescaped); + } case PM_NIL_NODE: return Qnil; case PM_RATIONAL_NODE: return parse_rational((const pm_rational_node_t *) node); - case PM_REGULAR_EXPRESSION_NODE: - return pm_new_regex(scope_node, (const pm_regular_expression_node_t *) node); + case PM_REGULAR_EXPRESSION_NODE: { + const pm_regular_expression_node_t *cast = (const pm_regular_expression_node_t *) node; + return pm_new_regex(scope_node, (const pm_node_t *) cast, &cast->unescaped); + } case PM_SOURCE_ENCODING_NODE: return rb_enc_from_encoding(scope_node->encoding); case PM_SOURCE_FILE_NODE: { @@ -757,7 +820,8 @@ pm_compile_loop(rb_iseq_t *iseq, const pm_line_column_t *line_column, pm_node_fl PUSH_LABEL(ret, next_label); if (type == PM_WHILE_NODE) { pm_compile_branch_condition(iseq, ret, predicate, redo_label, end_label, popped, scope_node); - } else if (type == PM_UNTIL_NODE) { + } + else if (type == PM_UNTIL_NODE) { pm_compile_branch_condition(iseq, ret, predicate, end_label, redo_label, popped, scope_node); } @@ -779,7 +843,7 @@ pm_compile_loop(rb_iseq_t *iseq, const pm_line_column_t *line_column, pm_node_fl } static int -pm_interpolated_node_compile(pm_node_list_t *parts, rb_iseq_t *iseq, NODE dummy_line_node, LINK_ANCHOR *const ret, bool popped, pm_scope_node_t *scope_node) +pm_interpolated_node_compile(const pm_node_list_t *parts, rb_iseq_t *iseq, NODE dummy_line_node, LINK_ANCHOR *const ret, bool popped, pm_scope_node_t *scope_node) { int number_of_items_pushed = 0; size_t parts_size = parts->size; @@ -787,44 +851,6 @@ pm_interpolated_node_compile(pm_node_list_t *parts, rb_iseq_t *iseq, NODE dummy_ if (parts_size > 0) { VALUE current_string = Qnil; - bool literal = true; - for (size_t index = 0; index < parts_size; index++) { - const pm_node_t *part = parts->nodes[index]; - - if (!PM_NODE_TYPE_P(part, PM_STRING_NODE)) { - literal = false; - break; - } - } - - if (literal) { - for (size_t index = 0; index < parts_size; index++) { - const pm_node_t *part = parts->nodes[index]; - const pm_string_node_t *string_node = (const pm_string_node_t *)part; - VALUE string_value = parse_string_encoded(scope_node, (pm_node_t *)string_node, &string_node->unescaped); - - if (RTEST(current_string)) { - current_string = rb_str_concat(current_string, string_value); - } - else { - current_string = string_value; - } - } - - const pm_node_t *part = parts->nodes[0]; - current_string = rb_fstring(current_string); - if (PM_NODE_FLAG_P(part, PM_STRING_FLAGS_FROZEN)) { - ADD_INSN1(ret, &dummy_line_node, putobject, current_string); - } - else if (PM_NODE_FLAG_P(part, PM_STRING_FLAGS_MUTABLE)) { - ADD_INSN1(ret, &dummy_line_node, putstring, current_string); - } - else { - ADD_INSN1(ret, &dummy_line_node, putchilledstring, current_string); - } - return 1; - } - for (size_t index = 0; index < parts_size; index++) { const pm_node_t *part = parts->nodes[index]; @@ -882,6 +908,7 @@ pm_interpolated_node_compile(pm_node_list_t *parts, rb_iseq_t *iseq, NODE dummy_ else { PM_PUTNIL; } + return number_of_items_pushed; } @@ -904,7 +931,8 @@ pm_lookup_local_index(rb_iseq_t *iseq, const pm_scope_node_t *scope_node, pm_con if (scope_node->previous) { scope_node = scope_node->previous; - } else { + } + else { // We have recursed up all scope nodes // and have not found the local yet rb_bug("Local with constant_id %u does not exist", (unsigned int) constant_id); @@ -1213,7 +1241,8 @@ pm_setup_args_core(const pm_arguments_node_t *arguments_node, const pm_node_t *b } RUBY_ASSERT(keyword_index == size); - } else { + } + else { // If they aren't all symbol keys then we need to // construct a new hash and pass that as an argument. orig_argc++; @@ -1819,7 +1848,8 @@ pm_compile_pattern_deconstruct(rb_iseq_t *iseq, pm_scope_node_t *scope_node, con ADD_INSN(ret, &line.node, pop); ADD_INSN1(ret, &line.node, topn, INT2FIX(base_index + PM_PATTERN_BASE_INDEX_OFFSET_DECONSTRUCTED_CACHE - 1)); ADD_INSNL(ret, &line.node, jump, deconstructed_label); - } else { + } + else { ADD_INSNL(ret, &line.node, jump, deconstruct_label); } @@ -2006,7 +2036,8 @@ pm_compile_pattern(rb_iseq_t *iseq, pm_scope_node_t *scope_node, const pm_node_t ADD_INSN1(ret, &line.node, setn, INT2FIX(4)); ADD_SEND(ret, &line.node, idAREF, INT2FIX(2)); CHECK(pm_compile_pattern_match(iseq, scope_node, ((const pm_splat_node_t *) cast->rest)->expression, ret, match_failed_label, in_single_pattern, in_alternation_pattern, false, base_index + 1)); - } else if (posts_size > 0) { + } + else if (posts_size > 0) { ADD_INSN(ret, &line.node, dup); ADD_SEND(ret, &line.node, idLength, INT2FIX(0)); ADD_INSN1(ret, &line.node, putobject, INT2FIX(minimum_size)); @@ -2237,7 +2268,8 @@ pm_compile_pattern(rb_iseq_t *iseq, pm_scope_node_t *scope_node, const pm_node_t if (NIL_P(keys)) { ADD_INSN(ret, &line.node, putnil); - } else { + } + else { ADD_INSN1(ret, &line.node, duparray, keys); RB_OBJ_WRITTEN(iseq, Qundef, rb_obj_hide(keys)); } @@ -2301,7 +2333,8 @@ pm_compile_pattern(rb_iseq_t *iseq, pm_scope_node_t *scope_node, const pm_node_t } ADD_SEQ(ret, match_values); - } else { + } + else { ADD_INSN(ret, &line.node, dup); ADD_SEND(ret, &line.node, idEmptyP, INT2FIX(0)); if (in_single_pattern) { @@ -2516,7 +2549,8 @@ pm_compile_pattern(rb_iseq_t *iseq, pm_scope_node_t *scope_node, const pm_node_t RUBY_ASSERT(cast->statements != NULL && cast->statements->body.size == 1); statement = cast->statements->body.nodes[0]; - } else { + } + else { const pm_unless_node_t *cast = (const pm_unless_node_t *) node; predicate = cast->predicate; @@ -2533,7 +2567,8 @@ pm_compile_pattern(rb_iseq_t *iseq, pm_scope_node_t *scope_node, const pm_node_t ADD_INSN(ret, &line.node, dup); if (PM_NODE_TYPE_P(node, PM_IF_NODE)) { ADD_INSNL(ret, &line.node, branchif, match_succeeded_label); - } else { + } + else { ADD_INSNL(ret, &line.node, branchunless, match_succeeded_label); } @@ -2550,7 +2585,8 @@ pm_compile_pattern(rb_iseq_t *iseq, pm_scope_node_t *scope_node, const pm_node_t if (PM_NODE_TYPE_P(node, PM_IF_NODE)) { ADD_INSNL(ret, &line.node, branchunless, unmatched_label); - } else { + } + else { ADD_INSNL(ret, &line.node, branchif, unmatched_label); } @@ -3254,7 +3290,8 @@ pm_compile_destructured_param_locals(const pm_multi_target_node_t *node, st_tabl pm_insert_local_index(((const pm_required_parameter_node_t *) left)->name, local_index, index_lookup_table, local_table_for_iseq, scope_node); local_index++; } - } else { + } + else { RUBY_ASSERT(PM_NODE_TYPE_P(left, PM_MULTI_TARGET_NODE)); local_index = pm_compile_destructured_param_locals((const pm_multi_target_node_t *) left, index_lookup_table, local_table_for_iseq, scope_node, local_index); } @@ -3276,7 +3313,8 @@ pm_compile_destructured_param_locals(const pm_multi_target_node_t *node, st_tabl if (PM_NODE_TYPE_P(right, PM_REQUIRED_PARAMETER_NODE)) { pm_insert_local_index(((const pm_required_parameter_node_t *) right)->name, local_index, index_lookup_table, local_table_for_iseq, scope_node); local_index++; - } else { + } + else { RUBY_ASSERT(PM_NODE_TYPE_P(right, PM_MULTI_TARGET_NODE)); local_index = pm_compile_destructured_param_locals((const pm_multi_target_node_t *) right, index_lookup_table, local_table_for_iseq, scope_node, local_index); } @@ -3324,7 +3362,8 @@ pm_compile_destructured_param_writes(rb_iseq_t *iseq, const pm_multi_target_node if (PM_NODE_TYPE_P(left, PM_REQUIRED_PARAMETER_NODE)) { pm_compile_destructured_param_write(iseq, (const pm_required_parameter_node_t *) left, ret, scope_node); - } else { + } + else { RUBY_ASSERT(PM_NODE_TYPE_P(left, PM_MULTI_TARGET_NODE)); pm_compile_destructured_param_writes(iseq, (const pm_multi_target_node_t *) left, ret, scope_node); } @@ -3351,7 +3390,8 @@ pm_compile_destructured_param_writes(rb_iseq_t *iseq, const pm_multi_target_node if (PM_NODE_TYPE_P(right, PM_REQUIRED_PARAMETER_NODE)) { pm_compile_destructured_param_write(iseq, (const pm_required_parameter_node_t *) right, ret, scope_node); - } else { + } + else { RUBY_ASSERT(PM_NODE_TYPE_P(right, PM_MULTI_TARGET_NODE)); pm_compile_destructured_param_writes(iseq, (const pm_multi_target_node_t *) right, ret, scope_node); } @@ -3420,7 +3460,8 @@ pm_multi_target_state_push(pm_multi_target_state_t *state, INSN *topn, size_t st if (state->head == NULL) { state->head = node; state->tail = node; - } else { + } + else { state->tail->next = node; state->tail = node; } @@ -3585,13 +3626,15 @@ pm_compile_target_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *cons if (cast->parent != NULL) { pm_compile_node(iseq, cast->parent, parents, false, scope_node); - } else { + } + else { ADD_INSN1(parents, &dummy_line_node, putobject, rb_cObject); } if (state == NULL) { ADD_INSN(writes, &dummy_line_node, swap); - } else { + } + else { ADD_INSN1(writes, &dummy_line_node, topn, INT2FIX(1)); pm_multi_target_state_push(state, (INSN *) LAST_ELEMENT(writes), 1); } @@ -3660,7 +3703,8 @@ pm_compile_target_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *cons if (argc == 0) { ADD_INSN(writes, &dummy_line_node, swap); - } else { + } + else { for (int index = 0; index < argc; index++) { ADD_INSN1(writes, &dummy_line_node, topn, INT2FIX(argc + 1)); } @@ -4529,7 +4573,8 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, ADD_ADJUST_RESTORE(ret, splabel); PM_PUTNIL_UNLESS_POPPED; - } else { + } + else { const rb_iseq_t *ip = iseq; while (ip) { @@ -4985,7 +5030,8 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, branch_id++; if (in_node->statements != NULL) { PM_COMPILE_INTO_ANCHOR(body_seq, (const pm_node_t *) in_node->statements); - } else if (!popped) { + } + else if (!popped) { ADD_INSN(body_seq, &in_line.node, putnil); } @@ -5013,7 +5059,8 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, if (else_node->statements != NULL) { PM_COMPILE_INTO_ANCHOR(cond_seq, (const pm_node_t *) else_node->statements); - } else if (!popped) { + } + else if (!popped) { ADD_INSN(cond_seq, &dummy_line_node, putnil); } @@ -5022,7 +5069,8 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, if (popped) { ADD_INSN(cond_seq, &dummy_line_node, putnil); } - } else { + } + else { // Otherwise, if we do not have an `else` clause, we will compile in // the code to handle raising an appropriate error. ADD_LABEL(cond_seq, else_label); @@ -5032,7 +5080,8 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, if (in_single_pattern) { pm_compile_pattern_error_handler(iseq, scope_node, node, cond_seq, end_label, popped); - } else { + } + else { ADD_INSN1(cond_seq, &dummy_line_node, putspecialobject, INT2FIX(VM_SPECIAL_OBJECT_VMCORE)); ADD_INSN1(cond_seq, &dummy_line_node, putobject, rb_eNoMatchingPatternError); ADD_INSN1(cond_seq, &dummy_line_node, topn, INT2FIX(2)); @@ -5846,7 +5895,8 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, ADD_INSN1(ret, &dummy_line_node, duphash, value); RB_OBJ_WRITTEN(iseq, Qundef, value); } - } else { + } + else { // Here since we know there are possible side-effects inside the // hash contents, we're going to build it entirely at runtime. We'll // do this by pushing all of the key-value pairs onto the stack and @@ -6018,71 +6068,114 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, return; } case PM_INTERPOLATED_MATCH_LAST_LINE_NODE: { - pm_interpolated_match_last_line_node_t *cast = (pm_interpolated_match_last_line_node_t *) node; - - int parts_size = (int)cast->parts.size; - - pm_interpolated_node_compile(&cast->parts, iseq, dummy_line_node, ret, popped, scope_node); - - ADD_INSN2(ret, &dummy_line_node, toregexp, INT2FIX(pm_reg_flags(node)), INT2FIX(parts_size)); + // if /foo #{bar}/ then end + // ^^^^^^^^^^^^ + if (PM_NODE_FLAG_P(node, PM_NODE_FLAG_STATIC_LITERAL)) { + if (!popped) { + VALUE regexp = pm_static_literal_value(node, scope_node); + PUSH_INSN1(ret, location, putobject, regexp); + } + } + else { + const pm_interpolated_match_last_line_node_t *cast = (const pm_interpolated_match_last_line_node_t *) node; + int length = pm_interpolated_node_compile(&cast->parts, iseq, dummy_line_node, ret, popped, scope_node); + PUSH_INSN2(ret, location, toregexp, INT2FIX(pm_reg_flags((const pm_node_t *) cast)), INT2FIX(length)); + } - ADD_INSN1(ret, &dummy_line_node, getglobal, rb_id2sym(idLASTLINE)); - ADD_SEND(ret, &dummy_line_node, idEqTilde, INT2NUM(1)); - PM_POP_IF_POPPED; + PUSH_INSN1(ret, location, getglobal, rb_id2sym(idLASTLINE)); + PUSH_SEND(ret, location, idEqTilde, INT2NUM(1)); + if (popped) PUSH_INSN(ret, location, pop); return; } case PM_INTERPOLATED_REGULAR_EXPRESSION_NODE: { - if (node->flags & PM_REGULAR_EXPRESSION_FLAGS_ONCE) { + // /foo #{bar}/ + // ^^^^^^^^^^^^ + if (PM_NODE_FLAG_P(node, PM_REGULAR_EXPRESSION_FLAGS_ONCE)) { const rb_iseq_t *prevblock = ISEQ_COMPILE_DATA(iseq)->current_block; const rb_iseq_t *block_iseq = NULL; - int ic_index = ISEQ_BODY(iseq)->ise_size++; + int ise_index = ISEQ_BODY(iseq)->ise_size++; pm_scope_node_t next_scope_node; - pm_scope_node_init((pm_node_t*)node, &next_scope_node, scope_node); - block_iseq = NEW_CHILD_ISEQ(&next_scope_node, make_name_for_block(iseq), ISEQ_TYPE_BLOCK, lineno); + pm_scope_node_init(node, &next_scope_node, scope_node); + + block_iseq = NEW_CHILD_ISEQ(&next_scope_node, make_name_for_block(iseq), ISEQ_TYPE_BLOCK, location.line); pm_scope_node_destroy(&next_scope_node); ISEQ_COMPILE_DATA(iseq)->current_block = block_iseq; - - ADD_INSN2(ret, &dummy_line_node, once, block_iseq, INT2FIX(ic_index)); - + PUSH_INSN2(ret, location, once, block_iseq, INT2FIX(ise_index)); ISEQ_COMPILE_DATA(iseq)->current_block = prevblock; + return; } - pm_interpolated_regular_expression_node_t *cast = (pm_interpolated_regular_expression_node_t *) node; + if (PM_NODE_FLAG_P(node, PM_NODE_FLAG_STATIC_LITERAL)) { + if (!popped) { + VALUE regexp = pm_static_literal_value(node, scope_node); + PUSH_INSN1(ret, location, putobject, regexp); + } + } + else { + const pm_interpolated_regular_expression_node_t *cast = (const pm_interpolated_regular_expression_node_t *) node; + int length = pm_interpolated_node_compile(&cast->parts, iseq, dummy_line_node, ret, popped, scope_node); - int parts_size = pm_interpolated_node_compile(&cast->parts, iseq, dummy_line_node, ret, popped, scope_node); + PUSH_INSN2(ret, location, toregexp, INT2FIX(pm_reg_flags((const pm_node_t *) cast)), INT2FIX(length)); + if (popped) PUSH_INSN(ret, location, pop); + } - ADD_INSN2(ret, &dummy_line_node, toregexp, INT2FIX(pm_reg_flags(node)), INT2FIX(parts_size)); - PM_POP_IF_POPPED; return; } case PM_INTERPOLATED_STRING_NODE: { - pm_interpolated_string_node_t *interp_string_node = (pm_interpolated_string_node_t *) node; - int number_of_items_pushed = pm_interpolated_node_compile(&interp_string_node->parts, iseq, dummy_line_node, ret, popped, scope_node); + // "foo #{bar}" + // ^^^^^^^^^^^^ + if (PM_NODE_FLAG_P(node, PM_NODE_FLAG_STATIC_LITERAL)) { + if (!popped) { + VALUE string = pm_static_literal_value(node, scope_node); - if (number_of_items_pushed > 1) { - ADD_INSN1(ret, &dummy_line_node, concatstrings, INT2FIX(number_of_items_pushed)); + if (PM_NODE_FLAG_P(node, PM_INTERPOLATED_STRING_NODE_FLAGS_FROZEN)) { + PUSH_INSN1(ret, location, putobject, string); + } + else if (PM_NODE_FLAG_P(node, PM_INTERPOLATED_STRING_NODE_FLAGS_MUTABLE)) { + PUSH_INSN1(ret, location, putstring, string); + } + else { + PUSH_INSN1(ret, location, putchilledstring, string); + } + } + } + else { + const pm_interpolated_string_node_t *cast = (const pm_interpolated_string_node_t *) node; + int length = pm_interpolated_node_compile(&cast->parts, iseq, dummy_line_node, ret, popped, scope_node); + + if (length > 1) PUSH_INSN1(ret, location, concatstrings, INT2FIX(length)); + if (popped) PUSH_INSN(ret, location, pop); } - PM_POP_IF_POPPED; return; } case PM_INTERPOLATED_SYMBOL_NODE: { - pm_interpolated_symbol_node_t *interp_symbol_node = (pm_interpolated_symbol_node_t *) node; - int number_of_items_pushed = pm_interpolated_node_compile(&interp_symbol_node->parts, iseq, dummy_line_node, ret, popped, scope_node); - - if (number_of_items_pushed > 1) { - ADD_INSN1(ret, &dummy_line_node, concatstrings, INT2FIX(number_of_items_pushed)); - } + // :"foo #{bar}" + // ^^^^^^^^^^^^^ + const pm_interpolated_symbol_node_t *cast = (const pm_interpolated_symbol_node_t *) node; + int length; - if (!popped) { - ADD_INSN(ret, &dummy_line_node, intern); + if (PM_NODE_FLAG_P(node, PM_NODE_FLAG_STATIC_LITERAL)) { + if (!popped) { + VALUE symbol = pm_static_literal_value(node, scope_node); + PUSH_INSN1(ret, location, putobject, symbol); + } } else { - PM_POP; + if ((length = pm_interpolated_node_compile(&cast->parts, iseq, dummy_line_node, ret, popped, scope_node)) > 1) { + PUSH_INSN1(ret, location, concatstrings, INT2FIX(length)); + } + + if (!popped) { + PUSH_INSN(ret, location, intern); + } + else { + PUSH_INSN(ret, location, pop); + } } return; @@ -6228,17 +6321,14 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, return; } case PM_MATCH_LAST_LINE_NODE: { - if (!popped) { - pm_match_last_line_node_t *cast = (pm_match_last_line_node_t *) node; - - VALUE regex_str = parse_string(scope_node, &cast->unescaped); - VALUE regex = rb_reg_new(RSTRING_PTR(regex_str), RSTRING_LEN(regex_str), pm_reg_flags(node)); - RB_GC_GUARD(regex_str); + // if /foo/ then end + // ^^^^^ + VALUE regexp = pm_static_literal_value(node, scope_node); - ADD_INSN1(ret, &dummy_line_node, putobject, regex); - ADD_INSN2(ret, &dummy_line_node, getspecial, INT2FIX(0), INT2FIX(0)); - ADD_SEND(ret, &dummy_line_node, idEqTilde, INT2NUM(1)); - } + PUSH_INSN1(ret, location, putobject, regexp); + PUSH_INSN2(ret, location, getspecial, INT2FIX(0), INT2FIX(0)); + PUSH_SEND(ret, location, idEqTilde, INT2NUM(1)); + if (popped) PUSH_INSN(ret, location, pop); return; } @@ -6634,7 +6724,8 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, if (cast->body != NULL) { PM_COMPILE(cast->body); - } else if (!popped) { + } + else if (!popped) { PUSH_INSN(ret, location, putnil); } @@ -6714,13 +6805,15 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, else { if (cast->left == NULL) { PUSH_INSN(ret, location, putnil); - } else { + } + else { PM_COMPILE(cast->left); } if (cast->right == NULL) { PUSH_INSN(ret, location, putnil); - } else { + } + else { PM_COMPILE(cast->right); } @@ -6837,7 +6930,8 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, PUSH_INSN1(ret, location, checkmatch, INT2FIX(checkmatch_flags)); PUSH_INSNL(ret, location, branchif, exception_match_label); } - } else { + } + else { ADD_GETLOCAL(ret, &dummy_line_node, LVAR_ERRINFO, 0); PUSH_INSN1(ret, location, putobject, rb_eStandardError); PUSH_INSN1(ret, location, checkmatch, INT2FIX(VM_CHECKMATCH_TYPE_RESCUE)); @@ -6885,7 +6979,8 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, // Now restore the end_label ISEQ_COMPILE_DATA(iseq)->end_label = prev_end; - } else { + } + else { PUSH_INSN(ret, location, putnil); } @@ -6898,7 +6993,8 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, PUSH_LABEL(ret, rescue_end_label); if (cast->consequent) { PM_COMPILE((pm_node_t *) cast->consequent); - } else { + } + else { ADD_GETLOCAL(ret, &dummy_line_node, 1, 0); } @@ -7005,7 +7101,8 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, PUSH_INSN(ret, location, putnil); PUSH_INSN1(ret, location, throw, INT2FIX(TAG_RETRY)); if (popped) PUSH_INSN(ret, location, pop); - } else { + } + else { COMPILE_ERROR(ERROR_ARGS "Invalid retry"); return; } @@ -7595,7 +7692,8 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, if (PM_NODE_TYPE_P(scope_node->ast_node, PM_FOR_NODE)) { if (PM_NODE_TYPE_P(((const pm_for_node_t *) scope_node->ast_node)->index, PM_LOCAL_VARIABLE_TARGET_NODE)) { body->param.lead_num++; - } else { + } + else { body->param.rest_start = local_index; body->param.flags.has_rest = true; } @@ -7823,7 +7921,8 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, pm_compile_node(iseq, scope_node->body, ret, popped, scope_node); break; } - } else { + } + else { PM_PUTNIL; } @@ -7851,7 +7950,8 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, ADD_TRACE(ret, RUBY_EVENT_CALL); if (scope_node->body) { PM_COMPILE((pm_node_t *)scope_node->body); - } else { + } + else { PM_PUTNIL; } @@ -7886,7 +7986,8 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, default: if (scope_node->body) { PM_COMPILE((pm_node_t *)scope_node->body); - } else { + } + else { PM_PUTNIL; } break; From 4bdb79618b33422551e96723827e50ab20e9abb1 Mon Sep 17 00:00:00 2001 From: Gannon McGibbon Date: Fri, 22 Mar 2024 15:40:42 -0500 Subject: [PATCH 071/211] Mark frame info structs with rb_gc_mark_movable Using rb_gc_mark_movable and a reference update function, we can make frame infos movable in memory, and avoid pinning frame info backtraces. ``` require "objspace" exceptions = [] GC.disable 50_000.times do begin raise "some exception" rescue => exception exception.backtrace_locations exceptions << exception end end GC.enable GC.compact p ObjectSpace.dump_all(output: :string).lines.grep(/"pinned":true/).count ``` Co-authored-by: Peter Zhu --- vm_backtrace.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/vm_backtrace.c b/vm_backtrace.c index 11b26adbf49781..3fbb8d1e3b6c21 100644 --- a/vm_backtrace.c +++ b/vm_backtrace.c @@ -134,7 +134,14 @@ static void location_mark(void *ptr) { struct valued_frame_info *vfi = (struct valued_frame_info *)ptr; - rb_gc_mark(vfi->btobj); + rb_gc_mark_movable(vfi->btobj); +} + +static void +location_ref_update(void *ptr) +{ + struct valued_frame_info *vfi = ptr; + vfi->btobj = rb_gc_location(vfi->btobj); } static void @@ -150,6 +157,7 @@ static const rb_data_type_t location_data_type = { location_mark, RUBY_TYPED_DEFAULT_FREE, NULL, // No external memory to report, + location_ref_update, }, 0, 0, RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_EMBEDDABLE }; From 3e0eea644f31b0a80da143c18149c750712d833d Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Sat, 23 Mar 2024 12:08:48 -0400 Subject: [PATCH 072/211] Don't set RUBY_TYPED_EMBEDDABLE flag on backtrace --- vm_backtrace.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/vm_backtrace.c b/vm_backtrace.c index 3fbb8d1e3b6c21..6f4379ad23e85c 100644 --- a/vm_backtrace.c +++ b/vm_backtrace.c @@ -540,7 +540,10 @@ static const rb_data_type_t backtrace_data_type = { NULL, // No external memory to report, backtrace_update, }, - 0, 0, RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_EMBEDDABLE + /* Cannot set the RUBY_TYPED_EMBEDDABLE flag because the loc of frame_info + * points elements in the backtrace array. This can cause the loc to become + * incorrect if this backtrace object is moved by compaction. */ + 0, 0, RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED }; int From 332f4938cf3adbff8f15b647767dc660583a5bef Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Tue, 26 Mar 2024 11:21:31 -0700 Subject: [PATCH 073/211] [DOC] Fix a description about rb_exec_recursive_outer It gives true/TRUE (int) instead of Qtrue (VALUE). --- thread.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/thread.c b/thread.c index e672172a4deec3..9400c32adc1953 100644 --- a/thread.c +++ b/thread.c @@ -5297,7 +5297,7 @@ rb_exec_recursive_paired(VALUE (*func) (VALUE, VALUE, int), VALUE obj, VALUE pai /* * If recursion is detected on the current method and obj, the outermost - * func will be called with (obj, arg, Qtrue). All inner func will be + * func will be called with (obj, arg, true). All inner func will be * short-circuited using throw. */ From 16cf9047c63aad5483bebe2068cdaa832447ba77 Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Tue, 26 Mar 2024 11:24:40 -0700 Subject: [PATCH 074/211] [DOC] Fix a couple other descriptions similarly to 332f4938cf3adbff8f15b647767dc660583a5bef --- thread.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/thread.c b/thread.c index 9400c32adc1953..c798c5a20467a3 100644 --- a/thread.c +++ b/thread.c @@ -5206,7 +5206,7 @@ exec_recursive_i(RB_BLOCK_CALL_FUNC_ARGLIST(tag, data)) * Calls func(obj, arg, recursive), where recursive is non-zero if the * current method is called recursively on obj, or on the pair * If outer is 0, then the innermost func will be called with recursive set - * to Qtrue, otherwise the outermost func will be called. In the latter case, + * to true, otherwise the outermost func will be called. In the latter case, * all inner func are short-circuited by throw. * Implementation details: the value thrown is the recursive list which is * proper to the current method and unlikely to be caught anywhere else. @@ -5315,7 +5315,7 @@ rb_exec_recursive_outer_mid(VALUE (*func) (VALUE, VALUE, int), VALUE obj, VALUE /* * If recursion is detected on the current method, obj and paired_obj, - * the outermost func will be called with (obj, arg, Qtrue). All inner + * the outermost func will be called with (obj, arg, true). All inner * func will be short-circuited using throw. */ From e16086b7f25d334c8049bd0e237191bdb3300d88 Mon Sep 17 00:00:00 2001 From: eileencodes Date: Mon, 25 Mar 2024 13:53:51 -0400 Subject: [PATCH 075/211] Refactor init_copy gc attributes This PR moves `rb_copy_wb_protected_attribute` and `rb_gc_copy_finalizer` into a single function called `rb_gc_copy_attributes` to be called by `init_copy`. This reduces the surface area of the GC API. Co-authored-by: Peter Zhu --- gc.c | 3 ++- internal/gc.h | 2 +- object.c | 4 +--- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/gc.c b/gc.c index 4b30bb171d6fd3..8812c0fea5e986 100644 --- a/gc.c +++ b/gc.c @@ -8691,11 +8691,12 @@ rb_gc_writebarrier_remember(VALUE obj) } void -rb_copy_wb_protected_attribute(VALUE dest, VALUE obj) +rb_gc_copy_attributes(VALUE dest, VALUE obj) { if (RVALUE_WB_UNPROTECTED(obj)) { rb_gc_writebarrier_unprotect(dest); } + rb_gc_copy_finalizer(dest, obj); } size_t diff --git a/internal/gc.h b/internal/gc.h index 595dbb9ef6064f..82a1b901f8af59 100644 --- a/internal/gc.h +++ b/internal/gc.h @@ -197,7 +197,7 @@ void rb_objspace_set_event_hook(const rb_event_flag_t event); VALUE rb_objspace_gc_enable(struct rb_objspace *); VALUE rb_objspace_gc_disable(struct rb_objspace *); void ruby_gc_set_params(void); -void rb_copy_wb_protected_attribute(VALUE dest, VALUE obj); +void rb_gc_copy_attributes(VALUE dest, VALUE obj); size_t rb_size_mul_or_raise(size_t, size_t, VALUE); /* used in compile.c */ size_t rb_size_mul_add_or_raise(size_t, size_t, size_t, VALUE); /* used in iseq.h */ size_t rb_malloc_grow_capa(size_t current_capacity, size_t type_size); diff --git a/object.c b/object.c index 8eee22fbb4a73c..4673ba9f6965ce 100644 --- a/object.c +++ b/object.c @@ -396,10 +396,8 @@ init_copy(VALUE dest, VALUE obj) RBASIC(dest)->flags &= ~(T_MASK|FL_EXIVAR); // Copies the shape id from obj to dest RBASIC(dest)->flags |= RBASIC(obj)->flags & (T_MASK|FL_EXIVAR); - rb_copy_wb_protected_attribute(dest, obj); + rb_gc_copy_attributes(dest, obj); rb_copy_generic_ivar(dest, obj); - rb_gc_copy_finalizer(dest, obj); - if (RB_TYPE_P(obj, T_OBJECT)) { rb_obj_copy_ivar(dest, obj); } From 696b2716e0ab7c7df983856216d65bb5f06bf956 Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Tue, 26 Mar 2024 11:32:09 -0700 Subject: [PATCH 076/211] Return stdbool from recursive_check() The return value is used as a boolean value in C. Since it's not used as a Ruby object, it just seems confusing that it returns a VALUE. --- thread.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/thread.c b/thread.c index c798c5a20467a3..1859f727ea5c2f 100644 --- a/thread.c +++ b/thread.c @@ -5098,12 +5098,12 @@ recursive_list_access(VALUE sym) } /* - * Returns Qtrue if and only if obj (or the pair ) is already + * Returns true if and only if obj (or the pair ) is already * in the recursion list. * Assumes the recursion list is valid. */ -static VALUE +static bool recursive_check(VALUE list, VALUE obj, VALUE paired_obj_id) { #if SIZEOF_LONG == SIZEOF_VOIDP @@ -5115,18 +5115,18 @@ recursive_check(VALUE list, VALUE obj, VALUE paired_obj_id) VALUE pair_list = rb_hash_lookup2(list, obj, Qundef); if (UNDEF_P(pair_list)) - return Qfalse; + return false; if (paired_obj_id) { if (!RB_TYPE_P(pair_list, T_HASH)) { if (!OBJ_ID_EQL(paired_obj_id, pair_list)) - return Qfalse; + return false; } else { if (NIL_P(rb_hash_lookup(pair_list, paired_obj_id))) - return Qfalse; + return false; } } - return Qtrue; + return true; } /* From 4a78d75213b1aee3bf0422fefaef204d06a459ac Mon Sep 17 00:00:00 2001 From: Koichi ITO Date: Fri, 22 Mar 2024 02:22:20 +0900 Subject: [PATCH 077/211] [ruby/prism] Fix an incorrect range of `Prism::Location` when `PM_ERR_RETURN_INVALID` This PR fixes the following incorrect range of `Prism::Location` when `PM_ERR_RETURN_INVALID`. It may be hard to tell from the text, but this Ruby error highlights `return`: ```console $ ruby -e 'class Foo return end' -e:1: Invalid return in class/module body class Foo return end -e: compile error (SyntaxError) ``` Previously, the error's `Prism::Location` pointed to `end`: ```console $ bundle exec ruby -Ilib -rprism -ve 'p Prism.parse("class Foo return end").errors' ruby 3.3.0 (2023-12-25 revision 5124f9ac75) [x86_64-darwin22] [# @level=:fatal>] After this fix, it will indicate `return`. ```console $ bundle exec ruby -Ilib -rprism -ve 'p Prism.parse("class Foo return end").errors' ruby 3.3.0 (2023-12-25 revision 5124f9ac75) [x86_64-darwin22] [# @level=:fatal>] ``` For reference, here are the before and after of `Prism::Translation::Parser`. Before: ```console $ bundle exec ruby -Ilib -rprism -rprism/translation/parser33 -ve 'p Prism::Translation::Parser33.parse("class Foo return end")' ruby 3.3.0 (2023-12-25 revision 5124f9ac75) [x86_64-darwin22] (string):1:18: error: invalid `return` in a class or module body (string):1: class Foo return end (string):1: ^~~ /Users/koic/.rbenv/versions/3.3.0/lib/ruby/gems/3.3.0/gems/parser-3.3.0.5/lib/parser/diagnostic/engine.rb:72:in `process': invalid `return` in a class or module body (Parser::SyntaxError) from /Users/koic/src/github.com/ruby/prism/lib/prism/translation/parser.rb:220:in `block in unwrap' from /Users/koic/src/github.com/ruby/prism/lib/prism/translation/parser.rb:218:in `each' from /Users/koic/src/github.com/ruby/prism/lib/prism/translation/parser.rb:218:in `unwrap' from /Users/koic/src/github.com/ruby/prism/lib/prism/translation/parser.rb:49:in `parse' from /Users/koic/.rbenv/versions/3.3.0/lib/ruby/gems/3.3.0/gems/parser-3.3.0.5/lib/parser/base.rb:33:in `parse' from -e:1:in `
' ``` After: ```console $ bundle exec ruby -Ilib -rprism -rprism/translation/parser33 -ve 'p Prism::Translation::Parser33.parse("class Foo return end")' ruby 3.3.0 (2023-12-25 revision 5124f9ac75) [x86_64-darwin22] (string):1:11: error: invalid `return` in a class or module body (string):1: class Foo return end (string):1: ^~~~~~ /Users/koic/.rbenv/versions/3.3.0/lib/ruby/gems/3.3.0/gems/parser-3.3.0.5/lib/parser/diagnostic/engine.rb:72:in `process': invalid `return` in a class or module body (Parser::SyntaxError) from /Users/koic/src/github.com/ruby/prism/lib/prism/translation/parser.rb:220:in `block in unwrap' from /Users/koic/src/github.com/ruby/prism/lib/prism/translation/parser.rb:218:in `each' from /Users/koic/src/github.com/ruby/prism/lib/prism/translation/parser.rb:218:in `unwrap' from /Users/koic/src/github.com/ruby/prism/lib/prism/translation/parser.rb:49:in `parse' from /Users/koic/.rbenv/versions/3.3.0/lib/ruby/gems/3.3.0/gems/parser-3.3.0.5/lib/parser/base.rb:33:in `parse' from -e:1:in `
' ``` This PR ensures that the originally intended `return` is highlighted as it should be. https://github.com/ruby/prism/commit/1f9af4d2ad --- prism/prism.c | 2 +- test/prism/errors_test.rb | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/prism/prism.c b/prism/prism.c index 4f65e1e0173e33..368a6c12ab768c 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -16444,7 +16444,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b (parser->current_context->context == PM_CONTEXT_CLASS) || (parser->current_context->context == PM_CONTEXT_MODULE) ) { - pm_parser_err_current(parser, PM_ERR_RETURN_INVALID); + pm_parser_err_previous(parser, PM_ERR_RETURN_INVALID); } return (pm_node_t *) pm_return_node_create(parser, &keyword, arguments.arguments); } diff --git a/test/prism/errors_test.rb b/test/prism/errors_test.rb index 9221d52ef35a4c..6e6e74ee5d5a1d 100644 --- a/test/prism/errors_test.rb +++ b/test/prism/errors_test.rb @@ -1091,7 +1091,7 @@ def test_dont_allow_return_inside_class_body ) assert_errors expected, "class A; return; end", [ - ["invalid `return` in a class or module body", 15..16] + ["invalid `return` in a class or module body", 9..15] ] end @@ -1106,7 +1106,7 @@ def test_dont_allow_return_inside_module_body ) assert_errors expected, "module A; return; end", [ - ["invalid `return` in a class or module body", 16..17] + ["invalid `return` in a class or module body", 10..16] ] end From 0c62eb25b5c7a2ce6764e81bf98ac39a804a3721 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 26 Mar 2024 14:40:34 -0400 Subject: [PATCH 078/211] [PRISM] Use correct encoding for regular expression literals --- .github/workflows/prism.yml | 2 +- prism_compile.c | 423 ++++++++++++---------- test/.excludes-prism/TestRegexp.rb | 7 + test/.excludes-prism/TestUnicodeEscape.rb | 1 - 4 files changed, 233 insertions(+), 200 deletions(-) create mode 100644 test/.excludes-prism/TestRegexp.rb diff --git a/.github/workflows/prism.yml b/.github/workflows/prism.yml index 49058c232f616a..8b295f7f815349 100644 --- a/.github/workflows/prism.yml +++ b/.github/workflows/prism.yml @@ -92,7 +92,7 @@ jobs: timeout-minutes: 40 env: GNUMAKEFLAGS: '' - RUBY_TESTOPTS: '-q --tty=no --excludes-dir="../src/test/.excludes-prism" --exclude="test_ast.rb" --exclude="test_regexp.rb" --exclude="error_highlight/test_error_highlight.rb" --exclude="prism/encoding_test.rb"' + RUBY_TESTOPTS: '-q --tty=no --excludes-dir="../src/test/.excludes-prism" --exclude="test_ast.rb" --exclude="error_highlight/test_error_highlight.rb" --exclude="prism/encoding_test.rb"' RUN_OPTS: ${{ matrix.run_opts }} - name: make test-prism-spec diff --git a/prism_compile.c b/prism_compile.c index db032e2a0530a5..d7eada1538ad38 100644 --- a/prism_compile.c +++ b/prism_compile.c @@ -289,131 +289,243 @@ pm_optimizable_range_item_p(pm_node_t *node) return (!node || PM_NODE_TYPE_P(node, PM_INTEGER_NODE) || PM_NODE_TYPE_P(node, PM_NIL_NODE)); } +static void pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, bool popped, pm_scope_node_t *scope_node); + +static int +pm_interpolated_node_compile(const pm_node_list_t *parts, rb_iseq_t *iseq, NODE dummy_line_node, LINK_ANCHOR *const ret, bool popped, pm_scope_node_t *scope_node) +{ + int number_of_items_pushed = 0; + size_t parts_size = parts->size; + + if (parts_size > 0) { + VALUE current_string = Qnil; + + for (size_t index = 0; index < parts_size; index++) { + const pm_node_t *part = parts->nodes[index]; + + if (PM_NODE_TYPE_P(part, PM_STRING_NODE)) { + const pm_string_node_t *string_node = (const pm_string_node_t *)part; + VALUE string_value = parse_string_encoded(scope_node, (pm_node_t *)string_node, &string_node->unescaped); + + if (RTEST(current_string)) { + current_string = rb_str_concat(current_string, string_value); + } + else { + current_string = string_value; + } + } + else if (PM_NODE_TYPE_P(part, PM_EMBEDDED_STATEMENTS_NODE) && + ((const pm_embedded_statements_node_t *) part)->statements != NULL && + ((const pm_embedded_statements_node_t *) part)->statements->body.size == 1 && + PM_NODE_TYPE_P(((const pm_embedded_statements_node_t *) part)->statements->body.nodes[0], PM_STRING_NODE)) { + const pm_string_node_t *string_node = (const pm_string_node_t *) ((const pm_embedded_statements_node_t *) part)->statements->body.nodes[0]; + VALUE string_value = parse_string_encoded(scope_node, (pm_node_t *)string_node, &string_node->unescaped); + + if (RTEST(current_string)) { + current_string = rb_str_concat(current_string, string_value); + } + else { + current_string = string_value; + } + } + else { + if (!RTEST(current_string)) { + current_string = rb_enc_str_new(NULL, 0, scope_node->encoding); + } + + ADD_INSN1(ret, &dummy_line_node, putobject, rb_fstring(current_string)); + + current_string = Qnil; + number_of_items_pushed++; + + PM_COMPILE_NOT_POPPED(part); + PM_DUP; + ADD_INSN1(ret, &dummy_line_node, objtostring, new_callinfo(iseq, idTo_s, 0, VM_CALL_FCALL | VM_CALL_ARGS_SIMPLE , NULL, FALSE)); + ADD_INSN(ret, &dummy_line_node, anytostring); + + number_of_items_pushed++; + } + } + + if (RTEST(current_string)) { + current_string = rb_fstring(current_string); + ADD_INSN1(ret, &dummy_line_node, putobject, current_string); + current_string = Qnil; + number_of_items_pushed++; + } + } + else { + PM_PUTNIL; + } + + return number_of_items_pushed; +} + +static VALUE +pm_static_literal_concat(const pm_node_list_t *nodes, const pm_scope_node_t *scope_node, bool top) +{ + VALUE current = Qnil; + + for (size_t index = 0; index < nodes->size; index++) { + const pm_node_t *part = nodes->nodes[index]; + VALUE string; + + switch (PM_NODE_TYPE(part)) { + case PM_STRING_NODE: + string = parse_string_encoded(scope_node, part, &((const pm_string_node_t *) part)->unescaped); + break; + case PM_INTERPOLATED_STRING_NODE: + string = pm_static_literal_concat(&((const pm_interpolated_string_node_t *) part)->parts, scope_node, false); + break; + default: + RUBY_ASSERT(false && "unexpected node type in pm_static_literal_concat"); + return Qnil; + } + + if (current != Qnil) { + current = rb_str_concat(current, string); + } + else { + current = string; + } + } + + return top ? rb_fstring(current) : current; +} + #define RE_OPTION_ENCODING_SHIFT 8 +#define RE_OPTION_ENCODING(encoding) (((encoding) & 0xFF) << RE_OPTION_ENCODING_SHIFT) +#define ARG_ENCODING_NONE 32 +#define ARG_ENCODING_FIXED 16 +#define ENC_ASCII8BIT 1 +#define ENC_EUC_JP 2 +#define ENC_Windows_31J 3 +#define ENC_UTF8 4 /** * Check the prism flags of a regular expression-like node and return the flags * that are expected by the CRuby VM. */ static int -pm_reg_flags(const pm_node_t *node) { +parse_regexp_flags(const pm_node_t *node) +{ int flags = 0; - int dummy = 0; // Check "no encoding" first so that flags don't get clobbered // We're calling `rb_char_to_option_kcode` in this case so that // we don't need to have access to `ARG_ENCODING_NONE` - if (node->flags & PM_REGULAR_EXPRESSION_FLAGS_ASCII_8BIT) { - rb_char_to_option_kcode('n', &flags, &dummy); + if (PM_NODE_FLAG_P(node, PM_REGULAR_EXPRESSION_FLAGS_ASCII_8BIT)) { + flags |= ARG_ENCODING_NONE; } - if (node->flags & PM_REGULAR_EXPRESSION_FLAGS_EUC_JP) { - rb_char_to_option_kcode('e', &flags, &dummy); - flags |= ('e' << RE_OPTION_ENCODING_SHIFT); + if (PM_NODE_FLAG_P(node, PM_REGULAR_EXPRESSION_FLAGS_EUC_JP)) { + flags |= (ARG_ENCODING_FIXED | RE_OPTION_ENCODING(ENC_EUC_JP)); } - if (node->flags & PM_REGULAR_EXPRESSION_FLAGS_WINDOWS_31J) { - rb_char_to_option_kcode('s', &flags, &dummy); - flags |= ('s' << RE_OPTION_ENCODING_SHIFT); + if (PM_NODE_FLAG_P(node, PM_REGULAR_EXPRESSION_FLAGS_WINDOWS_31J)) { + flags |= (ARG_ENCODING_FIXED | RE_OPTION_ENCODING(ENC_Windows_31J)); } - if (node->flags & PM_REGULAR_EXPRESSION_FLAGS_UTF_8) { - rb_char_to_option_kcode('u', &flags, &dummy); - flags |= ('u' << RE_OPTION_ENCODING_SHIFT); + if (PM_NODE_FLAG_P(node, PM_REGULAR_EXPRESSION_FLAGS_UTF_8)) { + flags |= (ARG_ENCODING_FIXED | RE_OPTION_ENCODING(ENC_UTF8)); } - if (node->flags & PM_REGULAR_EXPRESSION_FLAGS_IGNORE_CASE) { + if (PM_NODE_FLAG_P(node, PM_REGULAR_EXPRESSION_FLAGS_IGNORE_CASE)) { flags |= ONIG_OPTION_IGNORECASE; } - if (node->flags & PM_REGULAR_EXPRESSION_FLAGS_MULTI_LINE) { + if (PM_NODE_FLAG_P(node, PM_REGULAR_EXPRESSION_FLAGS_MULTI_LINE)) { flags |= ONIG_OPTION_MULTILINE; } - if (node->flags & PM_REGULAR_EXPRESSION_FLAGS_EXTENDED) { + if (PM_NODE_FLAG_P(node, PM_REGULAR_EXPRESSION_FLAGS_EXTENDED)) { flags |= ONIG_OPTION_EXTEND; } return flags; } +#undef RE_OPTION_ENCODING_SHIFT +#undef RE_OPTION_ENCODING +#undef ARG_ENCODING_FIXED +#undef ARG_ENCODING_NONE +#undef ENC_ASCII8BIT +#undef ENC_EUC_JP +#undef ENC_Windows_31J +#undef ENC_UTF8 + static rb_encoding * -pm_reg_enc(const pm_scope_node_t *scope_node, const pm_node_t *node) +parse_regexp_encoding(const pm_scope_node_t *scope_node, const pm_node_t *node) { if (PM_NODE_FLAG_P(node, PM_REGULAR_EXPRESSION_FLAGS_ASCII_8BIT)) { return rb_ascii8bit_encoding(); } - - if (PM_NODE_FLAG_P(node, PM_REGULAR_EXPRESSION_FLAGS_EUC_JP)) { + else if (PM_NODE_FLAG_P(node, PM_REGULAR_EXPRESSION_FLAGS_UTF_8)) { + return rb_utf8_encoding(); + } + else if (PM_NODE_FLAG_P(node, PM_REGULAR_EXPRESSION_FLAGS_EUC_JP)) { return rb_enc_get_from_index(ENCINDEX_EUC_JP); } - - if (PM_NODE_FLAG_P(node, PM_REGULAR_EXPRESSION_FLAGS_WINDOWS_31J)) { + else if (PM_NODE_FLAG_P(node, PM_REGULAR_EXPRESSION_FLAGS_WINDOWS_31J)) { return rb_enc_get_from_index(ENCINDEX_Windows_31J); } - - if (PM_NODE_FLAG_P(node, PM_REGULAR_EXPRESSION_FLAGS_UTF_8)) { - return rb_utf8_encoding(); + else { + return scope_node->encoding; } - - return scope_node->encoding; } -/** - * Certain nodes can be compiled literally, which can lead to further - * optimizations. These nodes will all have the PM_NODE_FLAG_STATIC_LITERAL flag - * set. - */ -static inline bool -pm_static_literal_p(const pm_node_t *node) +/** Raise an error corresponding to the invalid regular expression. */ +static VALUE +parse_regexp_error(rb_iseq_t *iseq, int32_t line_number, const char *fmt, ...) { - return node->flags & PM_NODE_FLAG_STATIC_LITERAL; + va_list args; + va_start(args, fmt); + VALUE error = rb_syntax_error_append(Qnil, rb_iseq_path(iseq), line_number, -1, NULL, "%" PRIsVALUE, args); + va_end(args); + rb_exc_raise(error); } static VALUE -pm_new_regex(const pm_scope_node_t *scope_node, const pm_node_t *node, const pm_string_t *unescaped) +parse_regexp(rb_iseq_t *iseq, const pm_scope_node_t *scope_node, const pm_node_t *node, VALUE string) { - VALUE regex_str = parse_string(scope_node, unescaped); - rb_encoding *enc = pm_reg_enc(scope_node, node); + VALUE errinfo = rb_errinfo(); - VALUE regex = rb_enc_reg_new(RSTRING_PTR(regex_str), RSTRING_LEN(regex_str), enc, pm_reg_flags(node)); - RB_GC_GUARD(regex_str); + int32_t line_number = pm_node_line_number(scope_node->parser, node); + VALUE regexp = rb_reg_compile(string, parse_regexp_flags(node), (const char *) pm_string_source(&scope_node->parser->filepath), line_number); - rb_obj_freeze(regex); + if (NIL_P(regexp)) { + VALUE message = rb_attr_get(rb_errinfo(), idMesg); + rb_set_errinfo(errinfo); - return regex; + parse_regexp_error(iseq, line_number, "%" PRIsVALUE, message); + return Qnil; + } + + rb_obj_freeze(regexp); + return regexp; } -static VALUE -pm_static_literal_concat(const pm_node_list_t *nodes, const pm_scope_node_t *scope_node, bool top) +static inline VALUE +parse_regexp_literal(rb_iseq_t *iseq, const pm_scope_node_t *scope_node, const pm_node_t *node, const pm_string_t *unescaped) { - VALUE current = Qnil; - - for (size_t index = 0; index < nodes->size; index++) { - const pm_node_t *part = nodes->nodes[index]; - VALUE string; - - switch (PM_NODE_TYPE(part)) { - case PM_STRING_NODE: - string = parse_string_encoded(scope_node, part, &((const pm_string_node_t *) part)->unescaped); - break; - case PM_INTERPOLATED_STRING_NODE: - string = pm_static_literal_concat(&((const pm_interpolated_string_node_t *) part)->parts, scope_node, false); - break; - default: - RUBY_ASSERT(false && "unexpected node type in pm_static_literal_concat"); - return Qnil; - } + VALUE string = rb_enc_str_new((const char *) pm_string_source(unescaped), pm_string_length(unescaped), parse_regexp_encoding(scope_node, node)); + return parse_regexp(iseq, scope_node, node, string); +} - if (current != Qnil) { - current = rb_str_concat(current, string); - } - else { - current = string; - } - } +static inline VALUE +parse_regexp_concat(rb_iseq_t *iseq, const pm_scope_node_t *scope_node, const pm_node_t *node, const pm_node_list_t *parts) +{ + VALUE string = pm_static_literal_concat(parts, scope_node, false); + rb_enc_associate(string, parse_regexp_encoding(scope_node, node)); + return parse_regexp(iseq, scope_node, node, string); +} - return top ? rb_fstring(current) : current; +static void +pm_compile_regexp_dynamic(rb_iseq_t *iseq, const pm_node_t *node, const pm_node_list_t *parts, const pm_line_column_t *node_location, LINK_ANCHOR *const ret, bool popped, pm_scope_node_t *scope_node) +{ + NODE dummy_line_node = generate_dummy_line_node(node_location->line, node_location->column); + int length = pm_interpolated_node_compile(parts, iseq, dummy_line_node, ret, popped, scope_node); + PUSH_INSN2(ret, *node_location, toregexp, INT2FIX(parse_regexp_flags(node) & 0xFF), INT2FIX(length)); } /** @@ -422,11 +534,11 @@ pm_static_literal_concat(const pm_node_list_t *nodes, const pm_scope_node_t *sco * literal values can be compiled into a literal array. */ static VALUE -pm_static_literal_value(const pm_node_t *node, const pm_scope_node_t *scope_node) +pm_static_literal_value(rb_iseq_t *iseq, const pm_node_t *node, const pm_scope_node_t *scope_node) { // Every node that comes into this function should already be marked as // static literal. If it's not, then we have a bug somewhere. - RUBY_ASSERT(pm_static_literal_p(node)); + RUBY_ASSERT(PM_NODE_FLAG_P(node, PM_NODE_FLAG_STATIC_LITERAL)); switch (PM_NODE_TYPE(node)) { case PM_ARRAY_NODE: { @@ -435,7 +547,7 @@ pm_static_literal_value(const pm_node_t *node, const pm_scope_node_t *scope_node VALUE value = rb_ary_hidden_new(elements->size); for (size_t index = 0; index < elements->size; index++) { - rb_ary_push(value, pm_static_literal_value(elements->nodes[index], scope_node)); + rb_ary_push(value, pm_static_literal_value(iseq, elements->nodes[index], scope_node)); } OBJ_FREEZE(value); @@ -453,7 +565,7 @@ pm_static_literal_value(const pm_node_t *node, const pm_scope_node_t *scope_node for (size_t index = 0; index < elements->size; index++) { RUBY_ASSERT(PM_NODE_TYPE_P(elements->nodes[index], PM_ASSOC_NODE)); pm_assoc_node_t *cast = (pm_assoc_node_t *) elements->nodes[index]; - VALUE pair[2] = { pm_static_literal_value(cast->key, scope_node), pm_static_literal_value(cast->value, scope_node) }; + VALUE pair[2] = { pm_static_literal_value(iseq, cast->key, scope_node), pm_static_literal_value(iseq, cast->value, scope_node) }; rb_ary_cat(array, pair, 2); } @@ -470,17 +582,11 @@ pm_static_literal_value(const pm_node_t *node, const pm_scope_node_t *scope_node return parse_integer((const pm_integer_node_t *) node); case PM_INTERPOLATED_MATCH_LAST_LINE_NODE: { const pm_interpolated_match_last_line_node_t *cast = (const pm_interpolated_match_last_line_node_t *) node; - VALUE string = pm_static_literal_concat(&cast->parts, scope_node, true); - - rb_encoding *encoding = pm_reg_enc(scope_node, (const pm_node_t *) cast); - return rb_obj_freeze(rb_enc_reg_new(RSTRING_PTR(string), RSTRING_LEN(string), encoding, pm_reg_flags((const pm_node_t *) cast))); + return parse_regexp_concat(iseq, scope_node, (const pm_node_t *) cast, &cast->parts); } case PM_INTERPOLATED_REGULAR_EXPRESSION_NODE: { const pm_interpolated_regular_expression_node_t *cast = (const pm_interpolated_regular_expression_node_t *) node; - VALUE string = pm_static_literal_concat(&cast->parts, scope_node, true); - - rb_encoding *encoding = pm_reg_enc(scope_node, (const pm_node_t *) cast); - return rb_obj_freeze(rb_enc_reg_new(RSTRING_PTR(string), RSTRING_LEN(string), encoding, pm_reg_flags((const pm_node_t *) cast))); + return parse_regexp_concat(iseq, scope_node, (const pm_node_t *) cast, &cast->parts); } case PM_INTERPOLATED_STRING_NODE: return pm_static_literal_concat(&((const pm_interpolated_string_node_t *) node)->parts, scope_node, true); @@ -492,7 +598,7 @@ pm_static_literal_value(const pm_node_t *node, const pm_scope_node_t *scope_node } case PM_MATCH_LAST_LINE_NODE: { const pm_match_last_line_node_t *cast = (const pm_match_last_line_node_t *) node; - return pm_new_regex(scope_node, (const pm_node_t *) cast, &cast->unescaped); + return parse_regexp_literal(iseq, scope_node, (const pm_node_t *) cast, &cast->unescaped); } case PM_NIL_NODE: return Qnil; @@ -500,13 +606,20 @@ pm_static_literal_value(const pm_node_t *node, const pm_scope_node_t *scope_node return parse_rational((const pm_rational_node_t *) node); case PM_REGULAR_EXPRESSION_NODE: { const pm_regular_expression_node_t *cast = (const pm_regular_expression_node_t *) node; - return pm_new_regex(scope_node, (const pm_node_t *) cast, &cast->unescaped); + return parse_regexp_literal(iseq, scope_node, (const pm_node_t *) cast, &cast->unescaped); } case PM_SOURCE_ENCODING_NODE: return rb_enc_from_encoding(scope_node->encoding); case PM_SOURCE_FILE_NODE: { - pm_source_file_node_t *cast = (pm_source_file_node_t *)node; - return cast->filepath.length ? parse_string(scope_node, &cast->filepath) : rb_fstring_lit(""); + const pm_source_file_node_t *cast = (const pm_source_file_node_t *) node; + size_t length = pm_string_length(&cast->filepath); + + if (length > 0) { + return rb_enc_str_new((const char *) pm_string_source(&cast->filepath), length, scope_node->encoding); + } + else { + return rb_fstring_lit(""); + } } case PM_SOURCE_LINE_NODE: return INT2FIX(pm_node_line_number(scope_node->parser, node)); @@ -601,8 +714,6 @@ pm_compile_logical(rb_iseq_t *iseq, LINK_ANCHOR *const ret, pm_node_t *cond, LAB return; } -static void pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, bool popped, pm_scope_node_t *scope_node); - static void pm_compile_flip_flop_bound(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, bool popped, pm_scope_node_t *scope_node) { @@ -842,76 +953,6 @@ pm_compile_loop(rb_iseq_t *iseq, const pm_line_column_t *line_column, pm_node_fl return; } -static int -pm_interpolated_node_compile(const pm_node_list_t *parts, rb_iseq_t *iseq, NODE dummy_line_node, LINK_ANCHOR *const ret, bool popped, pm_scope_node_t *scope_node) -{ - int number_of_items_pushed = 0; - size_t parts_size = parts->size; - - if (parts_size > 0) { - VALUE current_string = Qnil; - - for (size_t index = 0; index < parts_size; index++) { - const pm_node_t *part = parts->nodes[index]; - - if (PM_NODE_TYPE_P(part, PM_STRING_NODE)) { - const pm_string_node_t *string_node = (const pm_string_node_t *)part; - VALUE string_value = parse_string_encoded(scope_node, (pm_node_t *)string_node, &string_node->unescaped); - - if (RTEST(current_string)) { - current_string = rb_str_concat(current_string, string_value); - } - else { - current_string = string_value; - } - } - else if (PM_NODE_TYPE_P(part, PM_EMBEDDED_STATEMENTS_NODE) && - ((const pm_embedded_statements_node_t *) part)->statements != NULL && - ((const pm_embedded_statements_node_t *) part)->statements->body.size == 1 && - PM_NODE_TYPE_P(((const pm_embedded_statements_node_t *) part)->statements->body.nodes[0], PM_STRING_NODE)) { - const pm_string_node_t *string_node = (const pm_string_node_t *) ((const pm_embedded_statements_node_t *) part)->statements->body.nodes[0]; - VALUE string_value = parse_string_encoded(scope_node, (pm_node_t *)string_node, &string_node->unescaped); - - if (RTEST(current_string)) { - current_string = rb_str_concat(current_string, string_value); - } - else { - current_string = string_value; - } - } - else { - if (!RTEST(current_string)) { - current_string = rb_enc_str_new(NULL, 0, scope_node->encoding); - } - - ADD_INSN1(ret, &dummy_line_node, putobject, rb_fstring(current_string)); - - current_string = Qnil; - number_of_items_pushed++; - - PM_COMPILE_NOT_POPPED(part); - PM_DUP; - ADD_INSN1(ret, &dummy_line_node, objtostring, new_callinfo(iseq, idTo_s, 0, VM_CALL_FCALL | VM_CALL_ARGS_SIMPLE , NULL, FALSE)); - ADD_INSN(ret, &dummy_line_node, anytostring); - - number_of_items_pushed++; - } - } - - if (RTEST(current_string)) { - current_string = rb_fstring(current_string); - ADD_INSN1(ret, &dummy_line_node, putobject, current_string); - current_string = Qnil; - number_of_items_pushed++; - } - } - else { - PM_PUTNIL; - } - - return number_of_items_pushed; -} - // This recurses through scopes and finds the local index at any scope level // It also takes a pointer to depth, and increments depth appropriately // according to the depth of the local. @@ -1201,7 +1242,7 @@ pm_setup_args_core(const pm_arguments_node_t *arguments_node, const pm_node_t *b // Retrieve the stored index from the hash for this // keyword. - VALUE keyword = pm_static_literal_value(assoc->key, scope_node); + VALUE keyword = pm_static_literal_value(iseq, assoc->key, scope_node); VALUE stored_index = rb_hash_aref(stored_indices, keyword); // If this keyword was already seen in the hash, @@ -1233,7 +1274,7 @@ pm_setup_args_core(const pm_arguments_node_t *arguments_node, const pm_node_t *b bool popped = true; if (rb_ary_entry(keyword_indices, (long) element_index) == Qtrue) { - keywords[keyword_index++] = pm_static_literal_value(assoc->key, scope_node); + keywords[keyword_index++] = pm_static_literal_value(iseq, assoc->key, scope_node); popped = false; } @@ -4232,13 +4273,13 @@ pm_compile_constant_path(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *co * optimization entirely. */ static VALUE -pm_compile_case_node_dispatch(VALUE dispatch, const pm_node_t *node, LABEL *label, const pm_scope_node_t *scope_node) +pm_compile_case_node_dispatch(rb_iseq_t *iseq, VALUE dispatch, const pm_node_t *node, LABEL *label, const pm_scope_node_t *scope_node) { VALUE key = Qundef; switch (PM_NODE_TYPE(node)) { case PM_FLOAT_NODE: { - key = pm_static_literal_value(node, scope_node); + key = pm_static_literal_value(iseq, node, scope_node); double intptr; if (modf(RFLOAT_VALUE(key), &intptr) == 0.0) { @@ -4254,7 +4295,7 @@ pm_compile_case_node_dispatch(VALUE dispatch, const pm_node_t *node, LABEL *labe case PM_SOURCE_LINE_NODE: case PM_SYMBOL_NODE: case PM_TRUE_NODE: - key = pm_static_literal_value(node, scope_node); + key = pm_static_literal_value(iseq, node, scope_node); break; case PM_STRING_NODE: { const pm_string_node_t *cast = (const pm_string_node_t *) node; @@ -4379,13 +4420,13 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, // If every node in the array is static, then we can compile the entire // array now instead of later. - if (pm_static_literal_p(node)) { + if (PM_NODE_FLAG_P(node, PM_NODE_FLAG_STATIC_LITERAL)) { // We're only going to compile this node if it's not popped. If it // is popped, then we know we don't need to do anything since it's // statically known. if (!popped) { if (elements->size) { - VALUE value = pm_static_literal_value(node, scope_node); + VALUE value = pm_static_literal_value(iseq, node, scope_node); PUSH_INSN1(ret, location, duparray, value); } else { @@ -4868,7 +4909,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, // we're going to try to compile the condition into the // dispatch hash. if (dispatch != Qundef) { - dispatch = pm_compile_case_node_dispatch(dispatch, condition, label, scope_node); + dispatch = pm_compile_case_node_dispatch(iseq, dispatch, condition, label, scope_node); } if (PM_NODE_TYPE_P(condition, PM_SPLAT_NODE)) { @@ -5886,12 +5927,12 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, case PM_HASH_NODE: { // If every node in the hash is static, then we can compile the entire // hash now instead of later. - if (pm_static_literal_p(node)) { + if (PM_NODE_FLAG_P(node, PM_NODE_FLAG_STATIC_LITERAL)) { // We're only going to compile this node if it's not popped. If it // is popped, then we know we don't need to do anything since it's // statically known. if (!popped) { - VALUE value = pm_static_literal_value(node, scope_node); + VALUE value = pm_static_literal_value(iseq, node, scope_node); ADD_INSN1(ret, &dummy_line_node, duphash, value); RB_OBJ_WRITTEN(iseq, Qundef, value); } @@ -6072,14 +6113,12 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, // ^^^^^^^^^^^^ if (PM_NODE_FLAG_P(node, PM_NODE_FLAG_STATIC_LITERAL)) { if (!popped) { - VALUE regexp = pm_static_literal_value(node, scope_node); + VALUE regexp = pm_static_literal_value(iseq, node, scope_node); PUSH_INSN1(ret, location, putobject, regexp); } } else { - const pm_interpolated_match_last_line_node_t *cast = (const pm_interpolated_match_last_line_node_t *) node; - int length = pm_interpolated_node_compile(&cast->parts, iseq, dummy_line_node, ret, popped, scope_node); - PUSH_INSN2(ret, location, toregexp, INT2FIX(pm_reg_flags((const pm_node_t *) cast)), INT2FIX(length)); + pm_compile_regexp_dynamic(iseq, node, &((const pm_interpolated_match_last_line_node_t *) node)->parts, &location, ret, popped, scope_node); } PUSH_INSN1(ret, location, getglobal, rb_id2sym(idLASTLINE)); @@ -6106,20 +6145,18 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, PUSH_INSN2(ret, location, once, block_iseq, INT2FIX(ise_index)); ISEQ_COMPILE_DATA(iseq)->current_block = prevblock; + if (popped) PUSH_INSN(ret, location, pop); return; } if (PM_NODE_FLAG_P(node, PM_NODE_FLAG_STATIC_LITERAL)) { if (!popped) { - VALUE regexp = pm_static_literal_value(node, scope_node); + VALUE regexp = pm_static_literal_value(iseq, node, scope_node); PUSH_INSN1(ret, location, putobject, regexp); } } else { - const pm_interpolated_regular_expression_node_t *cast = (const pm_interpolated_regular_expression_node_t *) node; - int length = pm_interpolated_node_compile(&cast->parts, iseq, dummy_line_node, ret, popped, scope_node); - - PUSH_INSN2(ret, location, toregexp, INT2FIX(pm_reg_flags((const pm_node_t *) cast)), INT2FIX(length)); + pm_compile_regexp_dynamic(iseq, node, &((const pm_interpolated_regular_expression_node_t *) node)->parts, &location, ret, popped, scope_node); if (popped) PUSH_INSN(ret, location, pop); } @@ -6130,7 +6167,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, // ^^^^^^^^^^^^ if (PM_NODE_FLAG_P(node, PM_NODE_FLAG_STATIC_LITERAL)) { if (!popped) { - VALUE string = pm_static_literal_value(node, scope_node); + VALUE string = pm_static_literal_value(iseq, node, scope_node); if (PM_NODE_FLAG_P(node, PM_INTERPOLATED_STRING_NODE_FLAGS_FROZEN)) { PUSH_INSN1(ret, location, putobject, string); @@ -6161,7 +6198,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, if (PM_NODE_FLAG_P(node, PM_NODE_FLAG_STATIC_LITERAL)) { if (!popped) { - VALUE symbol = pm_static_literal_value(node, scope_node); + VALUE symbol = pm_static_literal_value(iseq, node, scope_node); PUSH_INSN1(ret, location, putobject, symbol); } } @@ -6323,7 +6360,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, case PM_MATCH_LAST_LINE_NODE: { // if /foo/ then end // ^^^^^ - VALUE regexp = pm_static_literal_value(node, scope_node); + VALUE regexp = pm_static_literal_value(iseq, node, scope_node); PUSH_INSN1(ret, location, putobject, regexp); PUSH_INSN2(ret, location, getspecial, INT2FIX(0), INT2FIX(0)); @@ -6895,8 +6932,8 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, // /foo/ // ^^^^^ if (!popped) { - VALUE regex = pm_static_literal_value(node, scope_node); - PUSH_INSN1(ret, location, putobject, regex); + VALUE regexp = pm_static_literal_value(iseq, node, scope_node); + PUSH_INSN1(ret, location, putobject, regexp); } return; } @@ -7500,12 +7537,8 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, pm_node_t *value = cast->value; name = cast->name; - if (pm_static_literal_p(value) && - !(PM_NODE_TYPE_P(value, PM_ARRAY_NODE) || - PM_NODE_TYPE_P(value, PM_HASH_NODE) || - PM_NODE_TYPE_P(value, PM_RANGE_NODE))) { - - rb_ary_push(default_values, pm_static_literal_value(value, scope_node)); + if (PM_NODE_FLAG_P(value, PM_NODE_FLAG_STATIC_LITERAL) && !(PM_NODE_TYPE_P(value, PM_ARRAY_NODE) || PM_NODE_TYPE_P(value, PM_HASH_NODE) || PM_NODE_TYPE_P(value, PM_RANGE_NODE))) { + rb_ary_push(default_values, pm_static_literal_value(iseq, value, scope_node)); } else { rb_ary_push(default_values, complex_mark); @@ -7814,10 +7847,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, pm_node_t *value = cast->value; name = cast->name; - if (!(pm_static_literal_p(value)) || - PM_NODE_TYPE_P(value, PM_ARRAY_NODE) || - PM_NODE_TYPE_P(value, PM_HASH_NODE) || - PM_NODE_TYPE_P(value, PM_RANGE_NODE)) { + if (!PM_NODE_FLAG_P(value, PM_NODE_FLAG_STATIC_LITERAL) || PM_NODE_TYPE_P(value, PM_ARRAY_NODE) || PM_NODE_TYPE_P(value, PM_HASH_NODE) || PM_NODE_TYPE_P(value, PM_RANGE_NODE)) { LABEL *end_label = NEW_LABEL(nd_line(&dummy_line_node)); pm_local_index_t index = pm_lookup_local_index(iseq, scope_node, name, 0); @@ -7910,11 +7940,8 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, break; } case PM_INTERPOLATED_REGULAR_EXPRESSION_NODE: { - pm_interpolated_regular_expression_node_t *cast = (pm_interpolated_regular_expression_node_t *) scope_node->ast_node; - - int parts_size = pm_interpolated_node_compile(&cast->parts, iseq, dummy_line_node, ret, popped, scope_node); - - ADD_INSN2(ret, &dummy_line_node, toregexp, INT2FIX(pm_reg_flags((pm_node_t *)cast)), INT2FIX(parts_size)); + const pm_interpolated_regular_expression_node_t *cast = (const pm_interpolated_regular_expression_node_t *) scope_node->ast_node; + pm_compile_regexp_dynamic(iseq, (const pm_node_t *) cast, &cast->parts, &location, ret, popped, scope_node); break; } default: @@ -8037,7 +8064,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, // __ENCODING__ // ^^^^^^^^^^^^ if (!popped) { - VALUE value = pm_static_literal_value(node, scope_node); + VALUE value = pm_static_literal_value(iseq, node, scope_node); PUSH_INSN1(ret, location, putobject, value); } return; @@ -8065,7 +8092,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, // __LINE__ // ^^^^^^^^ if (!popped) { - VALUE value = pm_static_literal_value(node, scope_node); + VALUE value = pm_static_literal_value(iseq, node, scope_node); PUSH_INSN1(ret, location, putobject, value); } return; @@ -8156,7 +8183,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, // :foo // ^^^^ if (!popped) { - VALUE value = pm_static_literal_value(node, scope_node); + VALUE value = pm_static_literal_value(iseq, node, scope_node); PUSH_INSN1(ret, location, putobject, value); } return; diff --git a/test/.excludes-prism/TestRegexp.rb b/test/.excludes-prism/TestRegexp.rb new file mode 100644 index 00000000000000..f2b817d79a7266 --- /dev/null +++ b/test/.excludes-prism/TestRegexp.rb @@ -0,0 +1,7 @@ +exclude(:test_unicode_age_14_0, "unknown") +exclude(:test_invalid_fragment, "unknown") +exclude(:test_assign_named_capture_to_reserved_word, "unknown") +exclude(:test_unicode_age_15_0, "unknown") +exclude(:test_unescape, "unknown") +exclude(:test_invalid_escape_error, "unknown") +exclude(:test_unicode_age, "unknown") diff --git a/test/.excludes-prism/TestUnicodeEscape.rb b/test/.excludes-prism/TestUnicodeEscape.rb index 93ed9fcb4514a5..add4911bc2f0aa 100644 --- a/test/.excludes-prism/TestUnicodeEscape.rb +++ b/test/.excludes-prism/TestUnicodeEscape.rb @@ -1,2 +1 @@ exclude(:test_fail, "unknown") -exclude(:test_regexp, "unknown") From a7ff264477105b5dc0ade6facad4176a1b73df0b Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Wed, 27 Mar 2024 10:10:07 +1300 Subject: [PATCH 079/211] Don't clear pending interrupts in the parent process. (#10365) --- process.c | 1 - test/ruby/test_process.rb | 21 +++++++++++++++++++++ thread.c | 1 + 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/process.c b/process.c index e5415dd170a362..8d08da76505635 100644 --- a/process.c +++ b/process.c @@ -1682,7 +1682,6 @@ before_fork_ruby(void) static void after_fork_ruby(rb_pid_t pid) { - rb_threadptr_pending_interrupt_clear(GET_THREAD()); if (pid == 0) { // child clear_pid_cache(); diff --git a/test/ruby/test_process.rb b/test/ruby/test_process.rb index 8982ab8b9a6a2d..59140ba664541c 100644 --- a/test/ruby/test_process.rb +++ b/test/ruby/test_process.rb @@ -2828,4 +2828,25 @@ def test_concurrent_group_and_pid_wait [t1, t2, t3].each { _1&.join rescue nil } [long_rpipe, long_wpipe, short_rpipe, short_wpipe].each { _1&.close rescue nil } end if defined?(fork) + + def test_handle_interrupt_with_fork + Thread.handle_interrupt(RuntimeError => :never) do + Thread.current.raise(RuntimeError, "Queued error") + + assert_predicate Thread, :pending_interrupt? + + pid = Process.fork do + if Thread.pending_interrupt? + exit 1 + end + end + + _, status = Process.waitpid2(pid) + assert_predicate status, :success? + + assert_predicate Thread, :pending_interrupt? + end + rescue RuntimeError + # Ignore. + end if defined?(fork) end diff --git a/thread.c b/thread.c index 1859f727ea5c2f..196f9092b49ebb 100644 --- a/thread.c +++ b/thread.c @@ -4725,6 +4725,7 @@ void rb_thread_atfork(void) { rb_thread_t *th = GET_THREAD(); + rb_threadptr_pending_interrupt_clear(th); rb_thread_atfork_internal(th, terminate_atfork_i); th->join_list = NULL; rb_fiber_atfork(th); From b2b665eba59e3fc9ad9656d9c74509a975db6fe8 Mon Sep 17 00:00:00 2001 From: crazeteam Date: Wed, 27 Mar 2024 00:15:40 +0800 Subject: [PATCH 080/211] [DOC] remove repetitive words in comments Signed-off-by: crazeteam --- doc/syntax/calling_methods.rdoc | 2 +- include/ruby/internal/intern/string.h | 2 +- spec/ruby/optional/capi/util_spec.rb | 2 +- test/ruby/test_string.rb | 2 +- warning.rb | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/syntax/calling_methods.rdoc b/doc/syntax/calling_methods.rdoc index 6cc8678450582f..c2c6c61a1039f4 100644 --- a/doc/syntax/calling_methods.rdoc +++ b/doc/syntax/calling_methods.rdoc @@ -210,7 +210,7 @@ definition. If a keyword argument is given that the method did not list, and the method definition does not accept arbitrary keyword arguments, an ArgumentError will be raised. -Keyword argument value can be omitted, meaning the value will be be fetched +Keyword argument value can be omitted, meaning the value will be fetched from the context by the name of the key keyword1 = 'some value' diff --git a/include/ruby/internal/intern/string.h b/include/ruby/internal/intern/string.h index cfe0454ee88562..6827563e8dc3e0 100644 --- a/include/ruby/internal/intern/string.h +++ b/include/ruby/internal/intern/string.h @@ -454,7 +454,7 @@ VALUE rb_interned_str(const char *ptr, long len); RBIMPL_ATTR_NONNULL(()) /** * Identical to rb_interned_str(), except it assumes the passed pointer is a - * pointer to a C's string. It can also be seen as a routine identical to to + * pointer to a C's string. It can also be seen as a routine identical to * rb_str_to_interned_str(), except it takes a C's string instead of Ruby's. * Or it can also be seen as a routine identical to rb_str_new_cstr(), except * it returns an infamous "f"string. diff --git a/spec/ruby/optional/capi/util_spec.rb b/spec/ruby/optional/capi/util_spec.rb index 2c16999cdc3589..9ff8b4760a83f7 100644 --- a/spec/ruby/optional/capi/util_spec.rb +++ b/spec/ruby/optional/capi/util_spec.rb @@ -48,7 +48,7 @@ ScratchPad.recorded.should == [1, 2, [3, 4]] end - it "assigns the required and optional arguments and and empty Array when there are no arguments to splat" do + it "assigns the required and optional arguments and empty Array when there are no arguments to splat" do @o.rb_scan_args([1, 2], "11*", 3, @acc).should == 2 ScratchPad.recorded.should == [1, 2, []] end diff --git a/test/ruby/test_string.rb b/test/ruby/test_string.rb index 8ec80d06fc7f2f..0dae1c5a8e49ea 100644 --- a/test/ruby/test_string.rb +++ b/test/ruby/test_string.rb @@ -3623,7 +3623,7 @@ def test_chilled_string assert_not_predicate +chilled_string, :frozen? assert_not_same chilled_string, +chilled_string - # @- the the original string as mutable + # @- the original string as mutable assert_predicate -chilled_string, :frozen? assert_not_same chilled_string, -chilled_string end diff --git a/warning.rb b/warning.rb index 4e34c6383393e5..aab5e7c2c6965d 100644 --- a/warning.rb +++ b/warning.rb @@ -40,7 +40,7 @@ module Kernel # baz.rb:6: warning: invalid call to foo # # If category keyword argument is given, passes the category - # to Warning.warn. The category given must be be one of the + # to Warning.warn. The category given must be one of the # following categories: # # :deprecated :: Used for warning for deprecated functionality that may From 6e34386794db69949b13f055fa338431527910eb Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 21 Mar 2024 12:17:15 -0400 Subject: [PATCH 081/211] [flori/json] Fix memory leak when exception is raised during JSON generation If an exception is raised the FBuffer is leaked. For example, the following script leaks memory: o = Object.new def o.to_json(a) = raise 10.times do 100_000.times do begin JSON(o) rescue end end puts `ps -o rss= -p #{$$}` end Before: 31824 35696 40240 44304 47424 50944 54000 58384 62416 65296 After: 24416 24640 24640 24736 24736 24736 24736 24736 24736 24736 https://github.com/flori/json/commit/44df509dc2 --- ext/json/generator/generator.c | 40 +++++++++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/ext/json/generator/generator.c b/ext/json/generator/generator.c index a33df848df638c..6d78284bc444cc 100644 --- a/ext/json/generator/generator.c +++ b/ext/json/generator/generator.c @@ -892,7 +892,6 @@ static void generate_json_object(FBuffer *buffer, VALUE Vstate, JSON_Generator_S struct hash_foreach_arg arg; if (max_nesting != 0 && depth > max_nesting) { - fbuffer_free(buffer); rb_raise(eNestingError, "nesting of %ld is too deep", --state->depth); } fbuffer_append_char(buffer, '{'); @@ -927,7 +926,6 @@ static void generate_json_array(FBuffer *buffer, VALUE Vstate, JSON_Generator_St long depth = ++state->depth; int i, j; if (max_nesting != 0 && depth > max_nesting) { - fbuffer_free(buffer); rb_raise(eNestingError, "nesting of %ld is too deep", --state->depth); } fbuffer_append_char(buffer, '['); @@ -1020,10 +1018,8 @@ static void generate_json_float(FBuffer *buffer, VALUE Vstate, JSON_Generator_St VALUE tmp = rb_funcall(obj, i_to_s, 0); if (!allow_nan) { if (isinf(value)) { - fbuffer_free(buffer); rb_raise(eGeneratorError, "%"PRIsVALUE" not allowed in JSON", RB_OBJ_STRING(tmp)); } else if (isnan(value)) { - fbuffer_free(buffer); rb_raise(eGeneratorError, "%"PRIsVALUE" not allowed in JSON", RB_OBJ_STRING(tmp)); } } @@ -1096,11 +1092,45 @@ static FBuffer *cState_prepare_buffer(VALUE self) return buffer; } +struct generate_json_data { + FBuffer *buffer; + VALUE vstate; + JSON_Generator_State *state; + VALUE obj; +}; + +static VALUE generate_json_try(VALUE d) +{ + struct generate_json_data *data = (struct generate_json_data *)d; + + generate_json(data->buffer, data->vstate, data->state, data->obj); + + return Qnil; +} + +static VALUE generate_json_rescue(VALUE d, VALUE exc) +{ + struct generate_json_data *data = (struct generate_json_data *)d; + fbuffer_free(data->buffer); + + rb_exc_raise(exc); + + return Qundef; +} + static VALUE cState_partial_generate(VALUE self, VALUE obj) { FBuffer *buffer = cState_prepare_buffer(self); GET_STATE(self); - generate_json(buffer, self, state, obj); + + struct generate_json_data data = { + .buffer = buffer, + .vstate = self, + .state = state, + .obj = obj + }; + rb_rescue(generate_json_try, (VALUE)&data, generate_json_rescue, (VALUE)&data); + return fbuffer_to_s(buffer); } From 8896ac0289dcd7a6c9c4a7fd6ccd4cc2dae30507 Mon Sep 17 00:00:00 2001 From: Jun Aruga Date: Tue, 12 Mar 2024 17:12:08 +0100 Subject: [PATCH 082/211] [ruby/openssl] Fix test_pkey_dsa.rb in FIPS. Note that I created the `dsa2048.pem` and signature text (`signature_encoded.txt`), that is used as a text to create the `signature0` in the `test_sign_verify` by the following steps with the `openssl` CLI on FIPS module. ``` $ OPENSSL_DIR="${HOME}/.local/openssl-3.3.0-dev-fips-debug-1f03d33ef5" $ export OPENSSL_CONF="${OPENSSL_DIR}/ssl/openssl_fips.cnf" $ "${OPENSSL_DIR}/bin/openssl" dsaparam -out dsaparam2048.pem 2048 $ "${OPENSSL_DIR}/bin/openssl" gendsa -out dsa2048.pem dsaparam2048.pem $ echo -n "Sign me!" > data.txt $ "${OPENSSL_DIR}/bin/openssl" dgst -sha256 -sign dsa2048.pem data.txt > signature.txt $ cat signature.txt | base64 > signature_encoded.txt ``` Skip the `test_DSAPrivateKey_encrypted` on FIPS because AES-128-CBC, the password based encryption used in the PEM format uses MD5 for deriving the encryption key from the password, and MD5 is not FIPS-approved. See also the comment on the `test/openssl/utils.rb#omit_on_fips`. https://github.com/ruby/openssl/commit/4bdcb419a9 --- test/openssl/fixtures/pkey/dsa2048.pem | 15 ++++++++++ test/openssl/test_pkey_dsa.rb | 40 ++++++++++++++++---------- 2 files changed, 40 insertions(+), 15 deletions(-) create mode 100644 test/openssl/fixtures/pkey/dsa2048.pem diff --git a/test/openssl/fixtures/pkey/dsa2048.pem b/test/openssl/fixtures/pkey/dsa2048.pem new file mode 100644 index 00000000000000..3f22b22b58bd0a --- /dev/null +++ b/test/openssl/fixtures/pkey/dsa2048.pem @@ -0,0 +1,15 @@ +-----BEGIN PRIVATE KEY----- +MIICXgIBADCCAjYGByqGSM44BAEwggIpAoIBAQDXZhJ/dQoWkQELzjzlx8FtIp96 +voCYe5NY0H8j0jz7GyHpXt41+MteqkZK3/Ah+cNR9uG8iEYArAZ71LcWotfee2Gz +xdxozr9bRt0POYhO2YIsfMpBrEskPsDH2g/2nFV8l4OJgxU2qZUrF4PN5ha+Mu6u +sVtN8hjvAvnbf4Pxn0b8NN9f4PJncroUa8acv5WsV85E1RW7NYCefggU4LytYIHg +euRF9eY9gVCX5MkUgW2xODHIYJhwk/+5lJxG7qUsSahD/nPHO/yoWgdVHq2DkdTq +KYXkAxx2PJcTBOHTglhE6mgCbEKp8vcfElnBWyCT6QykclZiPXXD2JV829J/Ah0A +vYa+/G/gUZiomyejVje6UsGoCc+vInxmovOL8QKCAQEAhnKEigYPw6u8JY7v5iGo +Ylz8qiMFYmaJCwevf3KCjWeEXuNO4OrKdfzkQl1tPuGLioYFfP1A2yGosjdUdLEB +0JqnzlKxUp+G6RfBj+WYzbgc5hr7t0M+reAJh09/hDzqfxjcgiHstq7mpRXBP8Y7 +iu27s7TRYJNSAYRvWcXNSBEUym3mHBBbZn7VszYooSrn60/iZ8I+VY1UF/fgqhbj +JfaaZNQCDO9K3Vb3rsXoYd8+bOZIen9uHB+pNjMqhpl4waysqrlpGFeeqdxivH6S +vkrHLs6/eWVMnS08RdcryoCrI3Bm8mMBKQglDwKLnWLfzG565qEhslzyCd/l9k9a +cwQfAh0Ao8/g72fSFmo04FizM7DZJSIPqDLjfZu9hLvUFA== +-----END PRIVATE KEY----- diff --git a/test/openssl/test_pkey_dsa.rb b/test/openssl/test_pkey_dsa.rb index 3f64a80e324814..4c93f2869d8040 100644 --- a/test/openssl/test_pkey_dsa.rb +++ b/test/openssl/test_pkey_dsa.rb @@ -31,11 +31,6 @@ def test_new_break def test_generate # DSA.generate used to call DSA_generate_parameters_ex(), which adjusts the # size of q according to the size of p - key1024 = OpenSSL::PKey::DSA.generate(1024) - assert_predicate key1024, :private? - assert_equal 1024, key1024.p.num_bits - assert_equal 160, key1024.q.num_bits - key2048 = OpenSSL::PKey::DSA.generate(2048) assert_equal 2048, key2048.p.num_bits assert_equal 256, key2048.q.num_bits @@ -47,28 +42,41 @@ def test_generate end end + def test_generate_on_non_fips + # DSA with 1024 bits is invalid on FIPS 186-4. + # https://github.com/openssl/openssl/commit/49ed5ba8f62875074f04417189147fd3dda072ab + omit_on_fips + + key1024 = OpenSSL::PKey::DSA.generate(1024) + assert_predicate key1024, :private? + assert_equal 1024, key1024.p.num_bits + assert_equal 160, key1024.q.num_bits + end + def test_sign_verify - dsa512 = Fixtures.pkey("dsa512") + # The DSA valid size is 2048 or 3072 on FIPS. + # https://github.com/openssl/openssl/blob/7649b5548e5c0352b91d9d3ed695e42a2ac1e99c/providers/common/securitycheck.c#L185-L188 + dsa = Fixtures.pkey("dsa2048") data = "Sign me!" if defined?(OpenSSL::Digest::DSS1) - signature = dsa512.sign(OpenSSL::Digest.new('DSS1'), data) - assert_equal true, dsa512.verify(OpenSSL::Digest.new('DSS1'), signature, data) + signature = dsa.sign(OpenSSL::Digest.new('DSS1'), data) + assert_equal true, dsa.verify(OpenSSL::Digest.new('DSS1'), signature, data) end - signature = dsa512.sign("SHA256", data) - assert_equal true, dsa512.verify("SHA256", signature, data) + signature = dsa.sign("SHA256", data) + assert_equal true, dsa.verify("SHA256", signature, data) signature0 = (<<~'end;').unpack1("m") - MCwCFH5h40plgU5Fh0Z4wvEEpz0eE9SnAhRPbkRB8ggsN/vsSEYMXvJwjGg/ - 6g== + MD4CHQC0zmRkVOAHJTm28fS5PVUv+4LtBeNaKqr/yfmVAh0AsTcLqofWHoW8X5oWu8AOvngOcFVZ + cLTvhY3XNw== end; - assert_equal true, dsa512.verify("SHA256", signature0, data) + assert_equal true, dsa.verify("SHA256", signature0, data) signature1 = signature0.succ - assert_equal false, dsa512.verify("SHA256", signature1, data) + assert_equal false, dsa.verify("SHA256", signature1, data) end def test_sign_verify_raw - key = Fixtures.pkey("dsa512") + key = Fixtures.pkey("dsa2048") data = 'Sign me!' digest = OpenSSL::Digest.digest('SHA1', data) @@ -127,6 +135,8 @@ def test_DSAPrivateKey end def test_DSAPrivateKey_encrypted + omit_on_fips + # key = abcdef dsa512 = Fixtures.pkey("dsa512") pem = <<~EOF From 8fa6c364925bff4e704d4c0fd73555fb33aa7029 Mon Sep 17 00:00:00 2001 From: Andrii Konchyn Date: Wed, 27 Mar 2024 02:39:16 +0200 Subject: [PATCH 083/211] [ruby/strscan] Omit tests for `#scan_byte` and `#peek_byte` on TruffleRuby temporary (https://github.com/ruby/strscan/pull/91) The methods were added in #89 but they aren't implemented in TruffleRuby yet. So let's omit them for now to have CI green. https://github.com/ruby/strscan/commit/844d963b56 --- test/strscan/test_stringscanner.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/strscan/test_stringscanner.rb b/test/strscan/test_stringscanner.rb index 2884b8ef054ff7..143cf7197df673 100644 --- a/test/strscan/test_stringscanner.rb +++ b/test/strscan/test_stringscanner.rb @@ -9,6 +9,7 @@ module StringScannerTests def test_peek_byte + omit("not implemented on TruffleRuby") if RUBY_ENGINE == "truffleruby" s = create_string_scanner('ab') assert_equal 97, s.peek_byte assert_equal 97, s.scan_byte @@ -19,6 +20,7 @@ def test_peek_byte end def test_scan_byte + omit("not implemented on TruffleRuby") if RUBY_ENGINE == "truffleruby" s = create_string_scanner('ab') assert_equal 97, s.scan_byte assert_equal 98, s.scan_byte From e51435177e88fc845528dff7cf2bc2b75dd36144 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Tue, 19 Mar 2024 11:37:11 +0900 Subject: [PATCH 084/211] Update vendored resolv to 0.4.0 --- lib/rubygems/vendor/resolv/lib/resolv.rb | 79 ++++++++++++++++++++---- tool/bundler/vendor_gems.rb | 2 +- 2 files changed, 68 insertions(+), 13 deletions(-) diff --git a/lib/rubygems/vendor/resolv/lib/resolv.rb b/lib/rubygems/vendor/resolv/lib/resolv.rb index 8e31eb1bee97df..ac0ba0b3136a79 100644 --- a/lib/rubygems/vendor/resolv/lib/resolv.rb +++ b/lib/rubygems/vendor/resolv/lib/resolv.rb @@ -37,7 +37,7 @@ class Gem::Resolv - VERSION = "0.3.0" + VERSION = "0.4.0" ## # Looks up the first IP address for +name+. @@ -194,17 +194,10 @@ def lazy_initialize # :nodoc: File.open(@filename, 'rb') {|f| f.each {|line| line.sub!(/#.*/, '') - addr, hostname, *aliases = line.split(/\s+/) + addr, *hostnames = line.split(/\s+/) next unless addr - @addr2name[addr] = [] unless @addr2name.include? addr - @addr2name[addr] << hostname - @addr2name[addr].concat(aliases) - @name2addr[hostname] = [] unless @name2addr.include? hostname - @name2addr[hostname] << addr - aliases.each {|n| - @name2addr[n] = [] unless @name2addr.include? n - @name2addr[n] << addr - } + (@addr2name[addr] ||= []).concat(hostnames) + hostnames.each {|hostname| (@name2addr[hostname] ||= []) << addr} } } @name2addr.each {|name, arr| arr.reverse!} @@ -2544,8 +2537,70 @@ class ANY < Query TypeValue = 255 # :nodoc: end + ## + # CAA resource record defined in RFC 8659 + # + # These records identify certificate authority allowed to issue + # certificates for the given domain. + + class CAA < Resource + TypeValue = 257 + + ## + # Creates a new CAA for +flags+, +tag+ and +value+. + + def initialize(flags, tag, value) + unless (0..255) === flags + raise ArgumentError.new('flags must be an Integer between 0 and 255') + end + unless (1..15) === tag.bytesize + raise ArgumentError.new('length of tag must be between 1 and 15') + end + + @flags = flags + @tag = tag + @value = value + end + + ## + # Flags for this proprty: + # - Bit 0 : 0 = not critical, 1 = critical + + attr_reader :flags + + ## + # Property tag ("issue", "issuewild", "iodef"...). + + attr_reader :tag + + ## + # Property value. + + attr_reader :value + + ## + # Whether the critical flag is set on this property. + + def critical? + flags & 0x80 != 0 + end + + def encode_rdata(msg) # :nodoc: + msg.put_pack('C', @flags) + msg.put_string(@tag) + msg.put_bytes(@value) + end + + def self.decode_rdata(msg) # :nodoc: + flags, = msg.get_unpack('C') + tag = msg.get_string + value = msg.get_bytes + self.new flags, tag, value + end + end + ClassInsensitiveTypes = [ # :nodoc: - NS, CNAME, SOA, PTR, HINFO, MINFO, MX, TXT, LOC, ANY + NS, CNAME, SOA, PTR, HINFO, MINFO, MX, TXT, LOC, ANY, CAA ] ## diff --git a/tool/bundler/vendor_gems.rb b/tool/bundler/vendor_gems.rb index 2500e6c80028d3..f02d02656d2e45 100644 --- a/tool/bundler/vendor_gems.rb +++ b/tool/bundler/vendor_gems.rb @@ -9,7 +9,7 @@ gem "net-protocol", "0.2.2" gem "optparse", "0.4.0" gem "pub_grub", github: "jhawthorn/pub_grub" -gem "resolv", "0.3.0" +gem "resolv", "0.4.0" gem "timeout", "0.4.1" gem "thor", "1.3.0" gem "tsort", "0.2.0" From 0c114dfcc79cb4690705ec88ebf9147e5c03702d Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Wed, 27 Mar 2024 11:45:01 +0900 Subject: [PATCH 085/211] Check existing ISeq wrapper --- iseq.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/iseq.c b/iseq.c index d00cc9e9cc495b..3e45c7b6218902 100644 --- a/iseq.c +++ b/iseq.c @@ -1417,6 +1417,10 @@ static VALUE iseqw_new(const rb_iseq_t *iseq) { if (iseq->wrapper) { + if (rb_check_typeddata(iseq->wrapper, &iseqw_data_type) != iseq) { + rb_raise(rb_eTypeError, "wrong iseq wrapper: %" PRIsVALUE " for %p", + iseq->wrapper, (void *)iseq); + } return iseq->wrapper; } else { From 16c18eafb579cf2263c7e0057c4c81358fe62075 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Wed, 27 Mar 2024 12:53:43 +0900 Subject: [PATCH 086/211] Revert "Mark iseq structs with rb_gc_mark_movable" This reverts commit a31ca3500d995b6706f94ff72166d699c5faeb27 which broke debug inspector API. --- iseq.c | 30 +++++++++--------------------- 1 file changed, 9 insertions(+), 21 deletions(-) diff --git a/iseq.c b/iseq.c index 3e45c7b6218902..c2657f29d7fab9 100644 --- a/iseq.c +++ b/iseq.c @@ -1386,30 +1386,18 @@ rb_iseq_remove_coverage_all(void) static void iseqw_mark(void *ptr) { - rb_gc_mark_movable(*(VALUE *)ptr); + rb_gc_mark((VALUE)ptr); } static size_t iseqw_memsize(const void *ptr) { - return rb_iseq_memsize(*(const rb_iseq_t **)ptr); -} - -static void -iseqw_ref_update(void *ptr) -{ - VALUE *vptr = ptr; - *vptr = rb_gc_location(*vptr); + return rb_iseq_memsize((const rb_iseq_t *)ptr); } static const rb_data_type_t iseqw_data_type = { "T_IMEMO/iseq", - { - iseqw_mark, - RUBY_TYPED_DEFAULT_FREE, - iseqw_memsize, - iseqw_ref_update, - }, + {iseqw_mark, NULL, iseqw_memsize,}, 0, 0, RUBY_TYPED_FREE_IMMEDIATELY|RUBY_TYPED_WB_PROTECTED }; @@ -1424,9 +1412,11 @@ iseqw_new(const rb_iseq_t *iseq) return iseq->wrapper; } else { - rb_iseq_t **ptr; - VALUE obj = TypedData_Make_Struct(rb_cISeq, rb_iseq_t *, &iseqw_data_type, ptr); - RB_OBJ_WRITE(obj, ptr, iseq); + union { const rb_iseq_t *in; void *out; } deconst; + VALUE obj; + deconst.in = iseq; + obj = TypedData_Wrap_Struct(rb_cISeq, &iseqw_data_type, deconst.out); + RB_OBJ_WRITTEN(obj, Qundef, iseq); /* cache a wrapper object */ RB_OBJ_WRITE((VALUE)iseq, &iseq->wrapper, obj); @@ -1750,9 +1740,7 @@ iseqw_s_compile_option_get(VALUE self) static const rb_iseq_t * iseqw_check(VALUE iseqw) { - rb_iseq_t **iseq_ptr; - TypedData_Get_Struct(iseqw, rb_iseq_t *, &iseqw_data_type, iseq_ptr); - rb_iseq_t *iseq = *iseq_ptr; + rb_iseq_t *iseq = DATA_PTR(iseqw); if (!ISEQ_BODY(iseq)) { rb_ibf_load_iseq_complete(iseq); From 66a0e8b008acb07c7eb2f31d65100bac1f361ff9 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Wed, 27 Mar 2024 11:07:25 +0900 Subject: [PATCH 087/211] d9234ba87b7e48381c8c44ef4a302ef368ee0ee7 is done to fix at related gems --- .github/workflows/macos.yml | 2 +- .github/workflows/ubuntu.yml | 2 +- .github/workflows/yjit-ubuntu.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index c5d88e30d70a6c..4bb41de22e799f 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -105,7 +105,7 @@ jobs: timeout-minutes: 60 env: RUBY_TESTOPTS: '-q --tty=no' - TEST_BUNDLED_GEMS_ALLOW_FAILURES: 'minitest,test-unit,debug,bigdecimal,drb,typeprof' + TEST_BUNDLED_GEMS_ALLOW_FAILURES: '' PRECHECK_BUNDLED_GEMS: 'no' - name: make skipped tests diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 62b55845b9e8d8..62e1b564b8828b 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -118,7 +118,7 @@ jobs: timeout-minutes: 40 env: RUBY_TESTOPTS: '-q --tty=no' - TEST_BUNDLED_GEMS_ALLOW_FAILURES: 'minitest,test-unit,debug,bigdecimal,drb,typeprof' + TEST_BUNDLED_GEMS_ALLOW_FAILURES: '' PRECHECK_BUNDLED_GEMS: 'no' - name: make skipped tests diff --git a/.github/workflows/yjit-ubuntu.yml b/.github/workflows/yjit-ubuntu.yml index d93fb24e419bac..b74629340f0a7e 100644 --- a/.github/workflows/yjit-ubuntu.yml +++ b/.github/workflows/yjit-ubuntu.yml @@ -185,7 +185,7 @@ jobs: timeout-minutes: 60 env: RUBY_TESTOPTS: '-q --tty=no' - TEST_BUNDLED_GEMS_ALLOW_FAILURES: 'minitest,test-unit,debug,bigdecimal,drb,typeprof' + TEST_BUNDLED_GEMS_ALLOW_FAILURES: '' PRECHECK_BUNDLED_GEMS: 'no' SYNTAX_SUGGEST_TIMEOUT: '5' YJIT_BINDGEN_DIFF_OPTS: '--exit-code' From 6498c4399511ddf4b953be4c0745d5395406ed01 Mon Sep 17 00:00:00 2001 From: git Date: Wed, 27 Mar 2024 06:59:42 +0000 Subject: [PATCH 088/211] Update bundled gems list as of 2024-03-26 --- NEWS.md | 2 +- gems/bundled_gems | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/NEWS.md b/NEWS.md index adc0ac5daaa0ef..ea274937758cba 100644 --- a/NEWS.md +++ b/NEWS.md @@ -57,7 +57,7 @@ The following bundled gems are updated. * test-unit 3.6.2 * net-ftp 0.3.4 * net-imap 0.4.10 -* net-smtp 0.4.0.1 +* net-smtp 0.5.0 * rbs 3.4.4 * typeprof 0.21.11 * debug 1.9.1 diff --git a/gems/bundled_gems b/gems/bundled_gems index 00a49537de3f86..5dfd755706a379 100644 --- a/gems/bundled_gems +++ b/gems/bundled_gems @@ -15,7 +15,7 @@ rss 0.3.0 https://github.com/ruby/rss net-ftp 0.3.4 https://github.com/ruby/net-ftp net-imap 0.4.10 https://github.com/ruby/net-imap net-pop 0.1.2 https://github.com/ruby/net-pop -net-smtp 0.4.0.1 https://github.com/ruby/net-smtp +net-smtp 0.5.0 https://github.com/ruby/net-smtp matrix 0.4.2 https://github.com/ruby/matrix prime 0.1.2 https://github.com/ruby/prime rbs 3.4.4 https://github.com/ruby/rbs ba7872795d5de04adb8ff500c0e6afdc81a041dd From cbc11bcb63ccebd8fc3f12362b8d6dbcf06d7fdd Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Wed, 27 Mar 2024 15:48:41 +0900 Subject: [PATCH 089/211] Ignore errors on prerelease gems --- tool/downloader.rb | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tool/downloader.rb b/tool/downloader.rb index 59bfd55f17c192..3a91ea0b938aed 100644 --- a/tool/downloader.rb +++ b/tool/downloader.rb @@ -79,6 +79,9 @@ def self.download(name, dir = nil, since = true, options = {}) require 'rubygems' options = options.dup options[:ssl_ca_cert] = Dir.glob(File.expand_path("../lib/rubygems/ssl_certs/**/*.pem", File.dirname(__FILE__))) + if Gem::Version.new(name[/-\K[^-]*(?=\.gem\z)/]).prerelease? + options[:ignore_http_client_errors] = true + end super("https://rubygems.org/downloads/#{name}", name, dir, since, options) end end @@ -237,6 +240,7 @@ def self.download(url, name, dir = nil, since = true, options = {}) $stdout.flush end mtime = nil + ignore_http_client_errors = options.delete(:ignore_http_client_errors) options = options.merge(http_options(file, since.nil? ? true : since)) begin data = with_retry(10) do @@ -247,12 +251,18 @@ def self.download(url, name, dir = nil, since = true, options = {}) data end rescue OpenURI::HTTPError => http_error - if http_error.message =~ /^304 / # 304 Not Modified + case http_error.message + when /^304 / # 304 Not Modified if $VERBOSE $stdout.puts "#{name} not modified" $stdout.flush end return file.to_path + when /^40/ # Net::HTTPClientError: 403 Forbidden, 404 Not Found + if ignore_http_client_errors + puts "Ignore #{url}: #{http_error.message}" + return file.to_path + end end raise rescue Timeout::Error From 44b5c912daae6d4e6dba6e40f13a2840c144cfa8 Mon Sep 17 00:00:00 2001 From: Cody Cutrer Date: Thu, 14 Sep 2023 11:35:50 -0600 Subject: [PATCH 090/211] [rubygems/rubygems] Allow installing plugins from path via CLI Also bring the man page up to date. https://github.com/rubygems/rubygems/commit/a849bd6947 --- lib/bundler/cli/plugin.rb | 3 ++- lib/bundler/man/bundle-add.1 | 2 +- lib/bundler/man/bundle-binstubs.1 | 2 +- lib/bundler/man/bundle-cache.1 | 2 +- lib/bundler/man/bundle-check.1 | 2 +- lib/bundler/man/bundle-clean.1 | 2 +- lib/bundler/man/bundle-config.1 | 2 +- lib/bundler/man/bundle-console.1 | 2 +- lib/bundler/man/bundle-doctor.1 | 2 +- lib/bundler/man/bundle-exec.1 | 2 +- lib/bundler/man/bundle-gem.1 | 2 +- lib/bundler/man/bundle-help.1 | 2 +- lib/bundler/man/bundle-info.1 | 2 +- lib/bundler/man/bundle-init.1 | 2 +- lib/bundler/man/bundle-inject.1 | 2 +- lib/bundler/man/bundle-install.1 | 2 +- lib/bundler/man/bundle-list.1 | 2 +- lib/bundler/man/bundle-lock.1 | 2 +- lib/bundler/man/bundle-open.1 | 2 +- lib/bundler/man/bundle-outdated.1 | 2 +- lib/bundler/man/bundle-platform.1 | 2 +- lib/bundler/man/bundle-plugin.1 | 9 ++++--- lib/bundler/man/bundle-plugin.1.ronn | 8 +++++-- lib/bundler/man/bundle-pristine.1 | 2 +- lib/bundler/man/bundle-remove.1 | 2 +- lib/bundler/man/bundle-show.1 | 2 +- lib/bundler/man/bundle-update.1 | 2 +- lib/bundler/man/bundle-version.1 | 2 +- lib/bundler/man/bundle-viz.1 | 2 +- lib/bundler/man/bundle.1 | 2 +- lib/bundler/man/gemfile.5 | 2 +- lib/bundler/plugin/installer.rb | 35 ++++++++++++++++++---------- lib/bundler/plugin/installer/path.rb | 18 ++++++++++++++ lib/bundler/plugin/source_list.rb | 8 +++---- spec/bundler/plugins/install_spec.rb | 10 ++++++++ 35 files changed, 97 insertions(+), 50 deletions(-) create mode 100644 lib/bundler/plugin/installer/path.rb diff --git a/lib/bundler/cli/plugin.rb b/lib/bundler/cli/plugin.rb index d946e495e93532..fd61ef0d954dea 100644 --- a/lib/bundler/cli/plugin.rb +++ b/lib/bundler/cli/plugin.rb @@ -5,7 +5,7 @@ module Bundler class CLI::Plugin < Thor desc "install PLUGINS", "Install the plugin from the source" long_desc <<-D - Install plugins either from the rubygems source provided (with --source option) or from a git source provided with --git. If no sources are provided, it uses Gem.sources + Install plugins either from the rubygems source provided (with --source option), from a git source provided with --git, or a local path provided with --path. If no sources are provided, it uses Gem.sources D method_option "source", type: :string, default: nil, banner: "URL of the RubyGems source to fetch the plugin from" method_option "version", type: :string, default: nil, banner: "The version of the plugin to fetch" @@ -13,6 +13,7 @@ class CLI::Plugin < Thor method_option "local_git", type: :string, default: nil, banner: "Path of the local git repo to fetch from (deprecated)" method_option "branch", type: :string, default: nil, banner: "The git branch to checkout" method_option "ref", type: :string, default: nil, banner: "The git revision to check out" + method_option "path", type: :string, default: nil, banner: "Path of a local gem to directly use" def install(*plugins) Bundler::Plugin.install(plugins, options) end diff --git a/lib/bundler/man/bundle-add.1 b/lib/bundler/man/bundle-add.1 index f85d21f9599b83..a6cbc88f344e27 100644 --- a/lib/bundler/man/bundle-add.1 +++ b/lib/bundler/man/bundle-add.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-ADD" "1" "February 2024" "" +.TH "BUNDLE\-ADD" "1" "March 2024" "" .SH "NAME" \fBbundle\-add\fR \- Add gem to the Gemfile and run bundle install .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-binstubs.1 b/lib/bundler/man/bundle-binstubs.1 index dd0b3cd4e899bc..2b35bc956a7837 100644 --- a/lib/bundler/man/bundle-binstubs.1 +++ b/lib/bundler/man/bundle-binstubs.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-BINSTUBS" "1" "February 2024" "" +.TH "BUNDLE\-BINSTUBS" "1" "March 2024" "" .SH "NAME" \fBbundle\-binstubs\fR \- Install the binstubs of the listed gems .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-cache.1 b/lib/bundler/man/bundle-cache.1 index 8e39bb92c3c464..3b86b995a62689 100644 --- a/lib/bundler/man/bundle-cache.1 +++ b/lib/bundler/man/bundle-cache.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-CACHE" "1" "February 2024" "" +.TH "BUNDLE\-CACHE" "1" "March 2024" "" .SH "NAME" \fBbundle\-cache\fR \- Package your needed \fB\.gem\fR files into your application .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-check.1 b/lib/bundler/man/bundle-check.1 index 82920d71887425..7f18e265375288 100644 --- a/lib/bundler/man/bundle-check.1 +++ b/lib/bundler/man/bundle-check.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-CHECK" "1" "February 2024" "" +.TH "BUNDLE\-CHECK" "1" "March 2024" "" .SH "NAME" \fBbundle\-check\fR \- Verifies if dependencies are satisfied by installed gems .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-clean.1 b/lib/bundler/man/bundle-clean.1 index 04cf55275caea2..0180eb38a213ac 100644 --- a/lib/bundler/man/bundle-clean.1 +++ b/lib/bundler/man/bundle-clean.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-CLEAN" "1" "February 2024" "" +.TH "BUNDLE\-CLEAN" "1" "March 2024" "" .SH "NAME" \fBbundle\-clean\fR \- Cleans up unused gems in your bundler directory .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-config.1 b/lib/bundler/man/bundle-config.1 index 4b0728c213c2e1..b768f1e3d29ec6 100644 --- a/lib/bundler/man/bundle-config.1 +++ b/lib/bundler/man/bundle-config.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-CONFIG" "1" "February 2024" "" +.TH "BUNDLE\-CONFIG" "1" "March 2024" "" .SH "NAME" \fBbundle\-config\fR \- Set bundler configuration options .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-console.1 b/lib/bundler/man/bundle-console.1 index 467a375f89de92..1368a50eb163fa 100644 --- a/lib/bundler/man/bundle-console.1 +++ b/lib/bundler/man/bundle-console.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-CONSOLE" "1" "February 2024" "" +.TH "BUNDLE\-CONSOLE" "1" "March 2024" "" .SH "NAME" \fBbundle\-console\fR \- Deprecated way to open an IRB session with the bundle pre\-loaded .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-doctor.1 b/lib/bundler/man/bundle-doctor.1 index e77bafec29b46c..80eaf2a8882524 100644 --- a/lib/bundler/man/bundle-doctor.1 +++ b/lib/bundler/man/bundle-doctor.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-DOCTOR" "1" "February 2024" "" +.TH "BUNDLE\-DOCTOR" "1" "March 2024" "" .SH "NAME" \fBbundle\-doctor\fR \- Checks the bundle for common problems .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-exec.1 b/lib/bundler/man/bundle-exec.1 index 926675aa5fdb06..191863c045ba5f 100644 --- a/lib/bundler/man/bundle-exec.1 +++ b/lib/bundler/man/bundle-exec.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-EXEC" "1" "February 2024" "" +.TH "BUNDLE\-EXEC" "1" "March 2024" "" .SH "NAME" \fBbundle\-exec\fR \- Execute a command in the context of the bundle .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-gem.1 b/lib/bundler/man/bundle-gem.1 index 744c4917c7e152..464d8d11264553 100644 --- a/lib/bundler/man/bundle-gem.1 +++ b/lib/bundler/man/bundle-gem.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-GEM" "1" "February 2024" "" +.TH "BUNDLE\-GEM" "1" "March 2024" "" .SH "NAME" \fBbundle\-gem\fR \- Generate a project skeleton for creating a rubygem .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-help.1 b/lib/bundler/man/bundle-help.1 index 613ff261dde66a..3604ad6127d6bd 100644 --- a/lib/bundler/man/bundle-help.1 +++ b/lib/bundler/man/bundle-help.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-HELP" "1" "February 2024" "" +.TH "BUNDLE\-HELP" "1" "March 2024" "" .SH "NAME" \fBbundle\-help\fR \- Displays detailed help for each subcommand .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-info.1 b/lib/bundler/man/bundle-info.1 index 7df617d859cea3..647f5987befbbc 100644 --- a/lib/bundler/man/bundle-info.1 +++ b/lib/bundler/man/bundle-info.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-INFO" "1" "February 2024" "" +.TH "BUNDLE\-INFO" "1" "March 2024" "" .SH "NAME" \fBbundle\-info\fR \- Show information for the given gem in your bundle .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-init.1 b/lib/bundler/man/bundle-init.1 index 4a9c6b01a1b58f..2c41a3c7deff9a 100644 --- a/lib/bundler/man/bundle-init.1 +++ b/lib/bundler/man/bundle-init.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-INIT" "1" "February 2024" "" +.TH "BUNDLE\-INIT" "1" "March 2024" "" .SH "NAME" \fBbundle\-init\fR \- Generates a Gemfile into the current working directory .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-inject.1 b/lib/bundler/man/bundle-inject.1 index f88f6dbe605253..c7269db34d0a21 100644 --- a/lib/bundler/man/bundle-inject.1 +++ b/lib/bundler/man/bundle-inject.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-INJECT" "1" "February 2024" "" +.TH "BUNDLE\-INJECT" "1" "March 2024" "" .SH "NAME" \fBbundle\-inject\fR \- Add named gem(s) with version requirements to Gemfile .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-install.1 b/lib/bundler/man/bundle-install.1 index f41def305fe71b..3fa1a467e2b7ce 100644 --- a/lib/bundler/man/bundle-install.1 +++ b/lib/bundler/man/bundle-install.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-INSTALL" "1" "February 2024" "" +.TH "BUNDLE\-INSTALL" "1" "March 2024" "" .SH "NAME" \fBbundle\-install\fR \- Install the dependencies specified in your Gemfile .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-list.1 b/lib/bundler/man/bundle-list.1 index e5b56767518278..f91fd95739351f 100644 --- a/lib/bundler/man/bundle-list.1 +++ b/lib/bundler/man/bundle-list.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-LIST" "1" "February 2024" "" +.TH "BUNDLE\-LIST" "1" "March 2024" "" .SH "NAME" \fBbundle\-list\fR \- List all the gems in the bundle .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-lock.1 b/lib/bundler/man/bundle-lock.1 index a63c6fd5a5283a..f992f5ee5f0a78 100644 --- a/lib/bundler/man/bundle-lock.1 +++ b/lib/bundler/man/bundle-lock.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-LOCK" "1" "February 2024" "" +.TH "BUNDLE\-LOCK" "1" "March 2024" "" .SH "NAME" \fBbundle\-lock\fR \- Creates / Updates a lockfile without installing .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-open.1 b/lib/bundler/man/bundle-open.1 index 8cf0ec0e364805..53d3541555153b 100644 --- a/lib/bundler/man/bundle-open.1 +++ b/lib/bundler/man/bundle-open.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-OPEN" "1" "February 2024" "" +.TH "BUNDLE\-OPEN" "1" "March 2024" "" .SH "NAME" \fBbundle\-open\fR \- Opens the source directory for a gem in your bundle .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-outdated.1 b/lib/bundler/man/bundle-outdated.1 index 30e795748c0d24..f79eff5ae99808 100644 --- a/lib/bundler/man/bundle-outdated.1 +++ b/lib/bundler/man/bundle-outdated.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-OUTDATED" "1" "February 2024" "" +.TH "BUNDLE\-OUTDATED" "1" "March 2024" "" .SH "NAME" \fBbundle\-outdated\fR \- List installed gems with newer versions available .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-platform.1 b/lib/bundler/man/bundle-platform.1 index 4f5240c242c430..d2133ec4d3fecb 100644 --- a/lib/bundler/man/bundle-platform.1 +++ b/lib/bundler/man/bundle-platform.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-PLATFORM" "1" "February 2024" "" +.TH "BUNDLE\-PLATFORM" "1" "March 2024" "" .SH "NAME" \fBbundle\-platform\fR \- Displays platform compatibility information .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-plugin.1 b/lib/bundler/man/bundle-plugin.1 index 29db99376010ba..cbdfac11b6bfb5 100644 --- a/lib/bundler/man/bundle-plugin.1 +++ b/lib/bundler/man/bundle-plugin.1 @@ -1,10 +1,10 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-PLUGIN" "1" "February 2024" "" +.TH "BUNDLE\-PLUGIN" "1" "March 2024" "" .SH "NAME" \fBbundle\-plugin\fR \- Manage Bundler plugins .SH "SYNOPSIS" -\fBbundle plugin\fR install PLUGINS [\-\-source=\fISOURCE\fR] [\-\-version=\fIversion\fR] [\-\-git=\fIgit\-url\fR] [\-\-branch=\fIbranch\fR|\-\-ref=\fIrev\fR] +\fBbundle plugin\fR install PLUGINS [\-\-source=\fISOURCE\fR] [\-\-version=\fIversion\fR] [\-\-git=\fIgit\-url\fR] [\-\-branch=\fIbranch\fR|\-\-ref=\fIrev\fR] [\-\-path=\fIpath\fR] .br \fBbundle plugin\fR uninstall PLUGINS .br @@ -37,7 +37,10 @@ Install bundler\-graph gem from Git repository\. You can use standard Git URLs l .br \fBfile:///path/to/repo\fR .IP -When you specify \fB\-\-git\fR, you can use \fB\-\-branch\fR or \fB\-\-ref\fR to specify any branch, tag, or commit hash (revision) to use\. When you specify both, only the latter is used\. +When you specify \fB\-\-git\fR, you can use \fB\-\-branch\fR or \fB\-\-ref\fR to specify any branch, tag, or commit hash (revision) to use\. +.TP +\fBbundle plugin install bundler\-graph \-\-path \.\./bundler\-graph\fR +Install bundler\-graph gem from a local path\. .SS "uninstall" Uninstall the plugin(s) specified in PLUGINS\. .SS "list" diff --git a/lib/bundler/man/bundle-plugin.1.ronn b/lib/bundler/man/bundle-plugin.1.ronn index e38776a47cd7a5..b0a34660ea84da 100644 --- a/lib/bundler/man/bundle-plugin.1.ronn +++ b/lib/bundler/man/bundle-plugin.1.ronn @@ -4,7 +4,8 @@ bundle-plugin(1) -- Manage Bundler plugins ## SYNOPSIS `bundle plugin` install PLUGINS [--source=] [--version=] - [--git=] [--branch=|--ref=]
+ [--git=] [--branch=|--ref=] + [--path=]
`bundle plugin` uninstall PLUGINS
`bundle plugin` list
`bundle plugin` help [COMMAND] @@ -36,7 +37,10 @@ Install the given plugin(s). `/path/to/repo`
`file:///path/to/repo` - When you specify `--git`, you can use `--branch` or `--ref` to specify any branch, tag, or commit hash (revision) to use. When you specify both, only the latter is used. + When you specify `--git`, you can use `--branch` or `--ref` to specify any branch, tag, or commit hash (revision) to use. + +* `bundle plugin install bundler-graph --path ../bundler-graph`: + Install bundler-graph gem from a local path. ### uninstall diff --git a/lib/bundler/man/bundle-pristine.1 b/lib/bundler/man/bundle-pristine.1 index 81a4758374fa86..faa04d76762a30 100644 --- a/lib/bundler/man/bundle-pristine.1 +++ b/lib/bundler/man/bundle-pristine.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-PRISTINE" "1" "February 2024" "" +.TH "BUNDLE\-PRISTINE" "1" "March 2024" "" .SH "NAME" \fBbundle\-pristine\fR \- Restores installed gems to their pristine condition .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-remove.1 b/lib/bundler/man/bundle-remove.1 index 1172577bca9d68..3f8cbbd9b684bf 100644 --- a/lib/bundler/man/bundle-remove.1 +++ b/lib/bundler/man/bundle-remove.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-REMOVE" "1" "February 2024" "" +.TH "BUNDLE\-REMOVE" "1" "March 2024" "" .SH "NAME" \fBbundle\-remove\fR \- Removes gems from the Gemfile .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-show.1 b/lib/bundler/man/bundle-show.1 index 338502ae358d18..bc72c6e3b6baf3 100644 --- a/lib/bundler/man/bundle-show.1 +++ b/lib/bundler/man/bundle-show.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-SHOW" "1" "February 2024" "" +.TH "BUNDLE\-SHOW" "1" "March 2024" "" .SH "NAME" \fBbundle\-show\fR \- Shows all the gems in your bundle, or the path to a gem .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-update.1 b/lib/bundler/man/bundle-update.1 index 8174d9b84133e9..d1284c2e72b54f 100644 --- a/lib/bundler/man/bundle-update.1 +++ b/lib/bundler/man/bundle-update.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-UPDATE" "1" "February 2024" "" +.TH "BUNDLE\-UPDATE" "1" "March 2024" "" .SH "NAME" \fBbundle\-update\fR \- Update your gems to the latest available versions .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-version.1 b/lib/bundler/man/bundle-version.1 index 5361b0493e2fee..05905e1347fc35 100644 --- a/lib/bundler/man/bundle-version.1 +++ b/lib/bundler/man/bundle-version.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-VERSION" "1" "February 2024" "" +.TH "BUNDLE\-VERSION" "1" "March 2024" "" .SH "NAME" \fBbundle\-version\fR \- Prints Bundler version information .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-viz.1 b/lib/bundler/man/bundle-viz.1 index ea726ceb499537..681563cd4c3d7f 100644 --- a/lib/bundler/man/bundle-viz.1 +++ b/lib/bundler/man/bundle-viz.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-VIZ" "1" "February 2024" "" +.TH "BUNDLE\-VIZ" "1" "March 2024" "" .SH "NAME" \fBbundle\-viz\fR \- Generates a visual dependency graph for your Gemfile .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle.1 b/lib/bundler/man/bundle.1 index 053b56d00a3b9c..1d2c780060f7c1 100644 --- a/lib/bundler/man/bundle.1 +++ b/lib/bundler/man/bundle.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE" "1" "February 2024" "" +.TH "BUNDLE" "1" "March 2024" "" .SH "NAME" \fBbundle\fR \- Ruby Dependency Management .SH "SYNOPSIS" diff --git a/lib/bundler/man/gemfile.5 b/lib/bundler/man/gemfile.5 index 6db532c34ae96d..39503f22a66233 100644 --- a/lib/bundler/man/gemfile.5 +++ b/lib/bundler/man/gemfile.5 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "GEMFILE" "5" "February 2024" "" +.TH "GEMFILE" "5" "March 2024" "" .SH "NAME" \fBGemfile\fR \- A format for describing gem dependencies for Ruby programs .SH "SYNOPSIS" diff --git a/lib/bundler/plugin/installer.rb b/lib/bundler/plugin/installer.rb index 7267f58f5d5ea9..6771f3f1537698 100644 --- a/lib/bundler/plugin/installer.rb +++ b/lib/bundler/plugin/installer.rb @@ -10,6 +10,7 @@ module Plugin class Installer autoload :Rubygems, File.expand_path("installer/rubygems", __dir__) autoload :Git, File.expand_path("installer/git", __dir__) + autoload :Path, File.expand_path("installer/path", __dir__) def install(names, options) check_sources_consistency!(options) @@ -18,6 +19,8 @@ def install(names, options) if options[:git] install_git(names, version, options) + elsif options[:path] + install_path(names, version, options[:path]) else sources = options[:source] || Gem.sources install_rubygems(names, version, sources) @@ -50,8 +53,8 @@ def check_sources_consistency!(options) options[:git] = options.delete(:local_git) end - if (options.keys & [:source, :git]).length > 1 - raise InvalidOption, "Only one of --source, or --git may be specified" + if (options.keys & [:source, :git, :path]).length > 1 + raise InvalidOption, "Only one of --source, --git, or --path may be specified" end if (options.key?(:branch) || options.key?(:ref)) && !options.key?(:git) @@ -64,10 +67,19 @@ def check_sources_consistency!(options) end def install_git(names, version, options) - uri = options.delete(:git) - options["uri"] = uri + source_list = SourceList.new + source = source_list.add_git_source({ "uri" => options[:git], + "branch" => options[:branch], + "ref" => options[:ref] }) + + install_all_sources(names, version, source_list, source) + end + + def install_path(names, version, path) + source_list = SourceList.new + source = source_list.add_path_source({ "path" => path }) - install_all_sources(names, version, options, options[:source]) + install_all_sources(names, version, source_list, source) end # Installs the plugin from rubygems source and returns the path where the @@ -79,16 +91,15 @@ def install_git(names, version, options) # # @return [Hash] map of names to the specs of plugins installed def install_rubygems(names, version, sources) - install_all_sources(names, version, nil, sources) - end - - def install_all_sources(names, version, git_source_options, rubygems_source) source_list = SourceList.new - source_list.add_git_source(git_source_options) if git_source_options - Array(rubygems_source).each {|remote| source_list.add_global_rubygems_remote(remote) } if rubygems_source + Array(sources).each {|remote| source_list.add_global_rubygems_remote(remote) } + + install_all_sources(names, version, source_list) + end - deps = names.map {|name| Dependency.new name, version } + def install_all_sources(names, version, source_list, source = nil) + deps = names.map {|name| Dependency.new(name, version, { "source" => source }) } Bundler.configure_gem_home_and_path(Plugin.root) diff --git a/lib/bundler/plugin/installer/path.rb b/lib/bundler/plugin/installer/path.rb new file mode 100644 index 00000000000000..1b60724b5e4ea3 --- /dev/null +++ b/lib/bundler/plugin/installer/path.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Bundler + module Plugin + class Installer + class Path < Bundler::Source::Path + def root + Plugin.root + end + + def generate_bin(spec, disable_extensions = false) + # Need to find a way without code duplication + # For now, we can ignore this + end + end + end + end +end diff --git a/lib/bundler/plugin/source_list.rb b/lib/bundler/plugin/source_list.rb index 547661cf2f73da..746996de5548ac 100644 --- a/lib/bundler/plugin/source_list.rb +++ b/lib/bundler/plugin/source_list.rb @@ -9,6 +9,10 @@ def add_git_source(options = {}) add_source_to_list Plugin::Installer::Git.new(options), git_sources end + def add_path_source(options = {}) + add_source_to_list Plugin::Installer::Path.new(options), path_sources + end + def add_rubygems_source(options = {}) add_source_to_list Plugin::Installer::Rubygems.new(options), @rubygems_sources end @@ -17,10 +21,6 @@ def all_sources path_sources + git_sources + rubygems_sources + [metadata_source] end - def default_source - git_sources.first || global_rubygems_source - end - private def rubygems_aggregate_class diff --git a/spec/bundler/plugins/install_spec.rb b/spec/bundler/plugins/install_spec.rb index 86eb4e584c6a8b..61c513ed723d76 100644 --- a/spec/bundler/plugins/install_spec.rb +++ b/spec/bundler/plugins/install_spec.rb @@ -212,6 +212,16 @@ def exec(command, args) end end + it "installs from a path source" do + build_lib "path_plugin" do |s| + s.write "plugins.rb" + end + bundle "plugin install path_plugin --path #{lib_path("path_plugin-1.0")}" + + expect(out).to include("Installed plugin path_plugin") + plugin_should_be_installed("path_plugin") + end + context "Gemfile eval" do before do allow(Bundler::SharedHelpers).to receive(:find_gemfile).and_return(bundled_app_gemfile) From e4b210906564cb5e922e8331a6b3724624377fdb Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 26 Mar 2024 14:54:50 -0400 Subject: [PATCH 091/211] [PRISM] Fix ASCII-compatible check for eval encoding --- prism_compile.c | 7 +++++-- test/.excludes-prism/TestEval.rb | 1 - 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/prism_compile.c b/prism_compile.c index d7eada1538ad38..a5b1219d6c1743 100644 --- a/prism_compile.c +++ b/prism_compile.c @@ -8611,9 +8611,12 @@ pm_load_parse_file(pm_parse_result_t *result, VALUE filepath) VALUE pm_parse_string(pm_parse_result_t *result, VALUE source, VALUE filepath) { - pm_string_constant_init(&result->input, RSTRING_PTR(source), RSTRING_LEN(source)); - rb_encoding *encoding = rb_enc_get(source); + if (!rb_enc_asciicompat(encoding)) { + return rb_exc_new_cstr(rb_eArgError, "invalid source encoding"); + } + + pm_string_constant_init(&result->input, RSTRING_PTR(source), RSTRING_LEN(source)); pm_options_encoding_set(&result->options, rb_enc_name(encoding)); pm_options_filepath_set(&result->options, RSTRING_PTR(filepath)); diff --git a/test/.excludes-prism/TestEval.rb b/test/.excludes-prism/TestEval.rb index 83f3e38fc9721f..6cc6bdfb1d65e6 100644 --- a/test/.excludes-prism/TestEval.rb +++ b/test/.excludes-prism/TestEval.rb @@ -1,2 +1 @@ -exclude(:test_eval_ascii_incompatible, "incorrect encoding") exclude(:test_file_encoding, "incorrect encoding") From 6f8a252e96715a72f5aef20e081a0efe7d66a850 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 26 Mar 2024 14:55:29 -0400 Subject: [PATCH 092/211] [PRISM] Enable passing heredoc test --- test/.excludes-prism/TestParse.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/test/.excludes-prism/TestParse.rb b/test/.excludes-prism/TestParse.rb index b5c4c6202d8cff..87694ac15a9851 100644 --- a/test/.excludes-prism/TestParse.rb +++ b/test/.excludes-prism/TestParse.rb @@ -1,5 +1,4 @@ exclude(:test_assign_in_conditional, "missing warning") -exclude(:test_here_document, "incorrect heredoc") exclude(:test_magic_comment, "incorrect encoding") exclude(:test_shareable_constant_value_nested, "ractor support") exclude(:test_shareable_constant_value_nonliteral, "ractor support") From 8b2fc859703fe3be0408f05b8734cc7624017f96 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 26 Mar 2024 14:56:17 -0400 Subject: [PATCH 093/211] [PRISM] Enable passing frozen string in array test --- test/.excludes-prism/TestRubyLiteral.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/test/.excludes-prism/TestRubyLiteral.rb b/test/.excludes-prism/TestRubyLiteral.rb index 1f78ea55a4172b..4cfe25215e1190 100644 --- a/test/.excludes-prism/TestRubyLiteral.rb +++ b/test/.excludes-prism/TestRubyLiteral.rb @@ -1,4 +1,3 @@ exclude(:test_dregexp, "unknown") -exclude(:test_frozen_string_in_array_literal, "incorrect frozen value") exclude(:test_hash_duplicated_key, "parser implementation dependent") exclude(:test_string, "https://github.com/ruby/prism/issues/2331") From 843c760a0ff7857d9ded3352877ca7d22f69e065 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 26 Mar 2024 14:57:28 -0400 Subject: [PATCH 094/211] [PRISM] Enable passing syntax tests --- test/.excludes-prism/TestSyntax.rb | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/.excludes-prism/TestSyntax.rb b/test/.excludes-prism/TestSyntax.rb index 23c755738e196c..680974906bcb6f 100644 --- a/test/.excludes-prism/TestSyntax.rb +++ b/test/.excludes-prism/TestSyntax.rb @@ -1,8 +1,4 @@ exclude(:test_dedented_heredoc_concatenation, "unknown") exclude(:test_dedented_heredoc_continued_line, "unknown") exclude(:test_duplicated_when, "unknown") -exclude(:test_integer_suffix, "unknown") exclude(:test_it, "https://github.com/ruby/prism/issues/2323") -exclude(:test_keyword_duplicated, "unknown") -exclude(:test_numbered_parameter, "unknown") -exclude(:test_too_big_nth_ref, "unknown") From a1ae29e87dd6d68da99df68420cd451b63b2211d Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 26 Mar 2024 15:14:51 -0400 Subject: [PATCH 095/211] [PRISM] Enable other passing specs --- spec/prism.mspec | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/spec/prism.mspec b/spec/prism.mspec index 99af52c0f1601c..490de9e348339b 100644 --- a/spec/prism.mspec +++ b/spec/prism.mspec @@ -2,28 +2,33 @@ ## Command line MSpec.register(:exclude, "The -S command line option runs launcher found in PATH, but only code after the first /#!.*ruby.*/-ish line in target file") -MSpec.register(:exclude, /^The -x command line option/) -MSpec.register(:exclude, "The --enable and --disable flags can be used with frozen-string-literal") -MSpec.register(:exclude, /^The --enable-frozen-string-literal flag/) +MSpec.register(:exclude, "The -x command line option runs code after the first /#!.*ruby.*/-ish line in target file") +MSpec.register(:exclude, "The -x command line option fails when /#!.*ruby.*/-ish line in target file is not found") +MSpec.register(:exclude, "The -x command line option behaves as -x was set when non-ruby shebang is encountered on first line") +MSpec.register(:exclude, "The --enable-frozen-string-literal flag causes string literals to produce the same object for literals with the same content in different files") MSpec.register(:exclude, "The --debug flag produces debugging info on attempted frozen string modification") ## Language MSpec.register(:exclude, "The BEGIN keyword runs multiple begins in FIFO order") MSpec.register(:exclude, "Executing break from within a block works when passing through a super call") MSpec.register(:exclude, "The defined? keyword when called with a method name in a void context warns about the void context when parsing it") -MSpec.register(:exclude, /^Hash literal expands an '\*\*\{\}'/) +MSpec.register(:exclude, "Hash literal expands an '**{}' or '**obj' element with the last key/value pair taking precedence") +MSpec.register(:exclude, "Hash literal expands an '**{}' and warns when finding an additional duplicate key afterwards") MSpec.register(:exclude, "Hash literal merges multiple nested '**obj' in Hash literals") -MSpec.register(:exclude, /^Hash literal raises a SyntaxError at parse time when Symbol key with invalid bytes/) +MSpec.register(:exclude, "Hash literal raises a SyntaxError at parse time when Symbol key with invalid bytes") +MSpec.register(:exclude, "Hash literal raises a SyntaxError at parse time when Symbol key with invalid bytes and 'key: value' syntax used") MSpec.register(:exclude, "The next statement in a method is invalid and raises a SyntaxError") MSpec.register(:exclude, "Pattern matching variable pattern does not support using variable name (except _) several times") MSpec.register(:exclude, "Pattern matching Hash pattern raise SyntaxError when keys duplicate in pattern") -MSpec.register(:exclude, "Regexp with character classes supports [[:alpha:][:digit:][:etc:]] (predefined character classes)") -MSpec.register(:exclude, /^Regexps with encoding modifiers/) -MSpec.register(:exclude, "Regexps with grouping raises a SyntaxError when parentheses aren't balanced") -MSpec.register(:exclude, "Regexps with modifiers supports (?imx-imx) (inline modifiers)") -MSpec.register(:exclude, "Regexps with modifiers supports (?imx-imx:expr) (scoped inline modifiers)") -MSpec.register(:exclude, "Literal Regexps throws SyntaxError for malformed literals") -MSpec.register(:exclude, "The rescue keyword raises SyntaxError when else is used without rescue and ensure") +MSpec.register(:exclude, "Regexps with encoding modifiers supports /e (EUC encoding) with interpolation") +MSpec.register(:exclude, "Regexps with encoding modifiers supports /e (EUC encoding) with interpolation /o") +MSpec.register(:exclude, "Regexps with encoding modifiers preserves EUC-JP as /e encoding through interpolation") +MSpec.register(:exclude, "Regexps with encoding modifiers supports /s (Windows_31J encoding) with interpolation") +MSpec.register(:exclude, "Regexps with encoding modifiers supports /s (Windows_31J encoding) with interpolation and /o") +MSpec.register(:exclude, "Regexps with encoding modifiers preserves Windows-31J as /s encoding through interpolation") +MSpec.register(:exclude, "Regexps with encoding modifiers supports /u (UTF8 encoding) with interpolation") +MSpec.register(:exclude, "Regexps with encoding modifiers supports /u (UTF8 encoding) with interpolation and /o") +MSpec.register(:exclude, "Regexps with encoding modifiers preserves UTF-8 as /u encoding through interpolation") MSpec.register(:exclude, "A Symbol literal raises an SyntaxError at parse time when Symbol with invalid bytes") ## Core @@ -33,8 +38,6 @@ MSpec.register(:exclude, "Kernel#eval includes file and line information in synt MSpec.register(:exclude, "Kernel#eval evaluates string with given filename and negative linenumber") MSpec.register(:exclude, "Kernel#eval with a magic encoding comment allows spaces before the magic encoding comment") MSpec.register(:exclude, "Kernel#eval with a magic encoding comment allows a shebang line and some spaces before the magic encoding comment") -MSpec.register(:exclude, "Kernel#eval with a magic encoding comment ignores the frozen_string_literal magic comment if it appears after a token and warns if $VERBOSE is true") -MSpec.register(:exclude, "Regexp#source has US-ASCII encoding when created from an ASCII-only \\u{} literal") MSpec.register(:exclude, "TracePoint#eval_script is the evald source code") MSpec.register(:exclude, "TracePoint#event returns the type of event") MSpec.register(:exclude, "TracePoint#inspect returns a String showing the event, method, path and line for a :return event") @@ -58,9 +61,6 @@ MSpec.register(:exclude, "Coverage.result does not clear counters when stop: fal MSpec.register(:exclude, "Coverage.result clears counters (sets 0 values) when stop: false and clear: true specified") MSpec.register(:exclude, "Coverage.result does not clear counters when stop: false and clear: false specified") MSpec.register(:exclude, "Coverage.start measures coverage within eval") -MSpec.register(:exclude, "Digest::SHA256.file when passed a path to a file that exists can be used with frozen-string-literal") MSpec.register(:exclude, "ERB#filename raises an exception if there are errors processing content") MSpec.register(:exclude, "ERB#filename uses '(erb)' as filename when filename is not set") -MSpec.register(:exclude, "mkmf can be required with --enable-frozen-string-literal") -MSpec.register(:exclude, "RbConfig::CONFIG contains no frozen strings even with --enable-frozen-string-literal") MSpec.register(:exclude, "Socket.gethostbyaddr using an IPv6 address with an explicit address family raises SocketError when the address is not supported by the family") From 42d1cd8f7fa62b585283e566d4be9a390631c43d Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 26 Mar 2024 15:32:01 -0400 Subject: [PATCH 096/211] [PRISM] Pass --enable-frozen-string-literal through to evals --- iseq.c | 2 -- prism_compile.c | 46 ++++++++++++++++++++++++++++------------------ prism_compile.h | 1 - ruby.c | 2 -- spec/prism.mspec | 1 - vm_eval.c | 2 -- 6 files changed, 28 insertions(+), 26 deletions(-) diff --git a/iseq.c b/iseq.c index c2657f29d7fab9..2d412dfcc9265d 100644 --- a/iseq.c +++ b/iseq.c @@ -1253,8 +1253,6 @@ pm_iseq_compile_with_option(VALUE src, VALUE file, VALUE realpath, VALUE line, V pm_parse_result_t result = { 0 }; pm_options_line_set(&result.options, NUM2INT(line)); - pm_options_frozen_string_literal_init(&result, option.frozen_string_literal); - VALUE error; if (RB_TYPE_P(src, T_FILE)) { VALUE filepath = rb_io_path(src); diff --git a/prism_compile.c b/prism_compile.c index a5b1219d6c1743..aec2f80f4102dc 100644 --- a/prism_compile.c +++ b/prism_compile.c @@ -8479,6 +8479,30 @@ pm_parse_process(pm_parse_result_t *result, pm_node_t *node) return Qnil; } +/** + * Set the frozen_string_literal option based on the default value used by the + * CRuby compiler. + */ +static void +pm_options_frozen_string_literal_init(pm_options_t *options) +{ + int frozen_string_literal = rb_iseq_opt_frozen_string_literal(); + + switch (frozen_string_literal) { + case ISEQ_FROZEN_STRING_LITERAL_UNSET: + break; + case ISEQ_FROZEN_STRING_LITERAL_DISABLED: + pm_options_frozen_string_literal_set(options, false); + break; + case ISEQ_FROZEN_STRING_LITERAL_ENABLED: + pm_options_frozen_string_literal_set(options, true); + break; + default: + rb_bug("pm_options_frozen_string_literal_init: invalid frozen_string_literal=%d", frozen_string_literal); + break; + } +} + /** * Returns an array of ruby String objects that represent the lines of the * source file that the given parser parsed. @@ -8514,24 +8538,6 @@ pm_parse_file_script_lines(const pm_scope_node_t *scope_node, const pm_parser_t return lines; } -void -pm_options_frozen_string_literal_init(pm_parse_result_t *result, int frozen_string_literal) -{ - switch (frozen_string_literal) { - case ISEQ_FROZEN_STRING_LITERAL_UNSET: - break; - case ISEQ_FROZEN_STRING_LITERAL_DISABLED: - pm_options_frozen_string_literal_set(&result->options, false); - break; - case ISEQ_FROZEN_STRING_LITERAL_ENABLED: - pm_options_frozen_string_literal_set(&result->options, true); - break; - default: - rb_bug("pm_options_frozen_string_literal_init: invalid frozen_string_literal=%d", frozen_string_literal); - break; - } -} - /** * Attempt to load the file into memory. Return a Ruby error if the file cannot * be read. @@ -8551,6 +8557,7 @@ pm_load_file(pm_parse_result_t *result, VALUE filepath) return err; } + pm_options_frozen_string_literal_init(&result->options); return Qnil; } @@ -8616,6 +8623,7 @@ pm_parse_string(pm_parse_result_t *result, VALUE source, VALUE filepath) return rb_exc_new_cstr(rb_eArgError, "invalid source encoding"); } + pm_options_frozen_string_literal_init(&result->options); pm_string_constant_init(&result->input, RSTRING_PTR(source), RSTRING_LEN(source)); pm_options_encoding_set(&result->options, rb_enc_name(encoding)); @@ -8658,6 +8666,8 @@ pm_parse_stdin_fgets(char *string, int size, void *stream) VALUE pm_parse_stdin(pm_parse_result_t *result) { + pm_options_frozen_string_literal_init(&result->options); + pm_buffer_t buffer; pm_node_t *node = pm_parse_stream(&result->parser, &buffer, (void *) rb_stdin, pm_parse_stdin_fgets, &result->options); diff --git a/prism_compile.h b/prism_compile.h index 0c1510d67f9814..427fa54b516013 100644 --- a/prism_compile.h +++ b/prism_compile.h @@ -47,7 +47,6 @@ typedef struct { bool parsed; } pm_parse_result_t; -void pm_options_frozen_string_literal_init(pm_parse_result_t *result, int frozen_string_literal); VALUE pm_load_file(pm_parse_result_t *result, VALUE filepath); VALUE pm_parse_file(pm_parse_result_t *result, VALUE filepath); VALUE pm_load_parse_file(pm_parse_result_t *result, VALUE filepath); diff --git a/ruby.c b/ruby.c index 5c1d44ec6df56c..59cbbd0360b690 100644 --- a/ruby.c +++ b/ruby.c @@ -2116,8 +2116,6 @@ prism_script(ruby_cmdline_options_t *opt, pm_parse_result_t *result) pm_options_t *options = &result->options; pm_options_line_set(options, 1); - pm_options_frozen_string_literal_init(result, rb_iseq_opt_frozen_string_literal()); - if (opt->ext.enc.name != 0) { pm_options_encoding_set(options, StringValueCStr(opt->ext.enc.name)); } diff --git a/spec/prism.mspec b/spec/prism.mspec index 490de9e348339b..bd427392e2f4e3 100644 --- a/spec/prism.mspec +++ b/spec/prism.mspec @@ -5,7 +5,6 @@ MSpec.register(:exclude, "The -S command line option runs launcher found in PATH MSpec.register(:exclude, "The -x command line option runs code after the first /#!.*ruby.*/-ish line in target file") MSpec.register(:exclude, "The -x command line option fails when /#!.*ruby.*/-ish line in target file is not found") MSpec.register(:exclude, "The -x command line option behaves as -x was set when non-ruby shebang is encountered on first line") -MSpec.register(:exclude, "The --enable-frozen-string-literal flag causes string literals to produce the same object for literals with the same content in different files") MSpec.register(:exclude, "The --debug flag produces debugging info on attempted frozen string modification") ## Language diff --git a/vm_eval.c b/vm_eval.c index 50c81f79d25353..a859a76f245bc8 100644 --- a/vm_eval.c +++ b/vm_eval.c @@ -1663,8 +1663,6 @@ pm_eval_make_iseq(VALUE src, VALUE fname, int line, pm_parse_result_t result = { 0 }; pm_options_line_set(&result.options, line); - pm_options_frozen_string_literal_init(&result, rb_iseq_opt_frozen_string_literal()); - // Cout scopes, one for each parent iseq, plus one for our local scope int scopes_count = 0; do { From 2505c27fdfa67c3191f9b5902a30ab3139a8ab6f Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 26 Mar 2024 15:45:43 -0400 Subject: [PATCH 097/211] [PRISM] Fix up some error formatting edge cases --- prism/prism.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/prism/prism.c b/prism/prism.c index 368a6c12ab768c..c6353f451b586c 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -19588,7 +19588,7 @@ typedef struct { #define PM_COLOR_GRAY "\033[38;5;102m" #define PM_COLOR_RED "\033[1;31m" -#define PM_COLOR_RESET "\033[0m" +#define PM_COLOR_RESET "\033[m" static inline pm_error_t * pm_parser_errors_format_sort(const pm_parser_t *parser, const pm_list_t *error_list, const pm_newline_list_t *newline_list) { @@ -19644,7 +19644,11 @@ pm_parser_errors_format_sort(const pm_parser_t *parser, const pm_list_t *error_l static inline void pm_parser_errors_format_line(const pm_parser_t *parser, const pm_newline_list_t *newline_list, const char *number_prefix, int32_t line, pm_buffer_t *buffer) { - size_t index = (size_t) (line - parser->start_line); + int32_t line_delta = line - parser->start_line; + assert(line_delta >= 0); + + size_t index = (size_t) line_delta; + assert(index < newline_list->size); const uint8_t *start = &parser->start[newline_list->offsets[index]]; const uint8_t *end; @@ -19765,7 +19769,7 @@ pm_parser_errors_format(const pm_parser_t *parser, pm_buffer_t *buffer, bool col // the source before the error to give some context. We'll be careful not to // display the same line twice in case the errors are close enough in the // source. - int32_t last_line = 0; + int32_t last_line = parser->start_line - 1; const pm_encoding_t *encoding = parser->encoding; for (size_t index = 0; index < error_list->size; index++) { @@ -19791,13 +19795,16 @@ pm_parser_errors_format(const pm_parser_t *parser, pm_buffer_t *buffer, bool col // the line that has the error in it. if ((index == 0) || (error->line != last_line)) { if (colorize) { - pm_buffer_append_string(buffer, PM_COLOR_RED "> " PM_COLOR_RESET, 13); + pm_buffer_append_string(buffer, PM_COLOR_RED "> " PM_COLOR_RESET, 12); } else { pm_buffer_append_string(buffer, "> ", 2); } pm_parser_errors_format_line(parser, newline_list, error_format.number_prefix, error->line, buffer); } + const uint8_t *start = &parser->start[newline_list->offsets[error->line - start_line]]; + if (start == parser->end) pm_buffer_append_byte(buffer, '\n'); + // Now we'll display the actual error message. We'll do this by first // putting the prefix to the line, then a bunch of blank spaces // depending on the column, then as many carets as we need to display @@ -19811,13 +19818,11 @@ pm_parser_errors_format(const pm_parser_t *parser, pm_buffer_t *buffer, bool col pm_buffer_append_string(buffer, error_format.blank_prefix, error_format.blank_prefix_length); size_t column = 0; - const uint8_t *start = &parser->start[newline_list->offsets[error->line - start_line]]; - while (column < error->column_end) { if (column < error->column_start) { pm_buffer_append_byte(buffer, ' '); } else if (colorize) { - pm_buffer_append_string(buffer, PM_COLOR_RED "^" PM_COLOR_RESET, 12); + pm_buffer_append_string(buffer, PM_COLOR_RED "^" PM_COLOR_RESET, 11); } else { pm_buffer_append_byte(buffer, '^'); } From 9ad175c1ee7137321310c4e87d8ce952b95abc8f Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Tue, 26 Mar 2024 11:34:28 -0400 Subject: [PATCH 098/211] Register rb_fix_to_s_static as global right after creating If a GC runs right during creating a rb_fix_to_s_static, it may cause the previous ones to become swept by the GC because they have not been registered by rb_vm_register_global_object. --- numeric.c | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/numeric.c b/numeric.c index 41beff0919f7f3..f0a0e3c279e9d1 100644 --- a/numeric.c +++ b/numeric.c @@ -6251,19 +6251,25 @@ Init_Numeric(void) rb_define_method(rb_cInteger, "digits", rb_int_digits, -1); - rb_fix_to_s_static[0] = rb_fstring_literal("0"); - rb_fix_to_s_static[1] = rb_fstring_literal("1"); - rb_fix_to_s_static[2] = rb_fstring_literal("2"); - rb_fix_to_s_static[3] = rb_fstring_literal("3"); - rb_fix_to_s_static[4] = rb_fstring_literal("4"); - rb_fix_to_s_static[5] = rb_fstring_literal("5"); - rb_fix_to_s_static[6] = rb_fstring_literal("6"); - rb_fix_to_s_static[7] = rb_fstring_literal("7"); - rb_fix_to_s_static[8] = rb_fstring_literal("8"); - rb_fix_to_s_static[9] = rb_fstring_literal("9"); - for(int i = 0; i < 10; i++) { - rb_vm_register_global_object(rb_fix_to_s_static[i]); - } +#define fix_to_s_static(n) do { \ + VALUE lit = rb_fstring_literal(#n); \ + rb_fix_to_s_static[n] = lit; \ + rb_vm_register_global_object(lit); \ + RB_GC_GUARD(lit); \ + } while (0) + + fix_to_s_static(0); + fix_to_s_static(1); + fix_to_s_static(2); + fix_to_s_static(3); + fix_to_s_static(4); + fix_to_s_static(5); + fix_to_s_static(6); + fix_to_s_static(7); + fix_to_s_static(8); + fix_to_s_static(9); + +#undef fix_to_s_static rb_cFloat = rb_define_class("Float", rb_cNumeric); From 1d99fe430aff34a90544d23edbd707f6f240dafe Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Tue, 26 Mar 2024 11:35:44 -0400 Subject: [PATCH 099/211] Register classpath of FrozenCore before converting to ICLASS Since ICLASS do not mark the classpath, we need to register it as a global object before we convert RubyVM::FrozenCore as a ICLASS. --- vm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm.c b/vm.c index 882514db41e57a..c9d99a34ea7aa1 100644 --- a/vm.c +++ b/vm.c @@ -3908,6 +3908,7 @@ Init_VM(void) /* FrozenCore (hidden) */ fcore = rb_class_new(rb_cBasicObject); rb_set_class_path(fcore, rb_cRubyVM, "FrozenCore"); + rb_vm_register_global_object(rb_class_path_cached(fcore)); RBASIC(fcore)->flags = T_ICLASS; klass = rb_singleton_class(fcore); rb_define_method_id(klass, id_core_set_method_alias, m_core_set_method_alias, 3); @@ -3927,7 +3928,6 @@ Init_VM(void) RBASIC_CLEAR_CLASS(klass); rb_obj_freeze(klass); rb_vm_register_global_object(fcore); - rb_vm_register_global_object(rb_class_path_cached(fcore)); rb_mRubyVMFrozenCore = fcore; /* From f14e52c8c45f1288537ff38c153096d095b5ad20 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Tue, 26 Mar 2024 11:37:21 -0400 Subject: [PATCH 100/211] Fix setting GC stress at boot when objspace not available --- debug.c | 2 +- gc.c | 26 +++++++++++++++----------- internal/gc.h | 2 +- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/debug.c b/debug.c index 546ffc440f155b..4717a0bc9c5e90 100644 --- a/debug.c +++ b/debug.c @@ -226,7 +226,7 @@ ruby_env_debug_option(const char *str, int len, void *arg) } if (NAME_MATCH("gc_stress")) { - rb_gc_stress_set(Qtrue); + rb_gc_initial_stress_set(Qtrue); return 1; } SET_WHEN("core", ruby_enable_coredump, 1); diff --git a/gc.c b/gc.c index 8812c0fea5e986..04df9dc71bf998 100644 --- a/gc.c +++ b/gc.c @@ -1876,10 +1876,22 @@ calloc1(size_t n) return calloc(1, n); } +static VALUE initial_stress = Qfalse; + +void +rb_gc_initial_stress_set(VALUE flag) +{ + initial_stress = flag; +} + rb_objspace_t * rb_objspace_alloc(void) { rb_objspace_t *objspace = calloc1(sizeof(rb_objspace_t)); + + objspace->flags.gc_stressful = RTEST(initial_stress); + objspace->gc_stress_mode = initial_stress; + objspace->flags.measure_gc = 1; malloc_limit = gc_params.malloc_limit_min; objspace->finalize_deferred_pjob = rb_postponed_job_preregister(0, gc_finalize_deferred, objspace); @@ -1898,8 +1910,6 @@ rb_objspace_alloc(void) rb_darray_make_without_gc(&objspace->weak_references, 0); - dont_gc_on(); - return objspace; } @@ -8951,7 +8961,7 @@ gc_start(rb_objspace_t *objspace, unsigned int reason) /* reason may be clobbered, later, so keep set immediate_sweep here */ objspace->flags.immediate_sweep = !!(reason & GPR_FLAG_IMMEDIATE_SWEEP); - if (!heap_allocated_pages) return FALSE; /* heap is not ready */ + if (!heap_allocated_pages) return TRUE; /* heap is not ready */ if (!(reason & GPR_FLAG_METHOD) && !ready_to_gc(objspace)) return TRUE; /* GC is not allowed */ GC_ASSERT(gc_mode(objspace) == gc_mode_none); @@ -11147,20 +11157,14 @@ gc_stress_get(rb_execution_context_t *ec, VALUE self) return ruby_gc_stress_mode; } -void -rb_gc_stress_set(VALUE flag) +static VALUE +gc_stress_set_m(rb_execution_context_t *ec, VALUE self, VALUE flag) { rb_objspace_t *objspace = &rb_objspace; objspace->flags.gc_stressful = RTEST(flag); objspace->gc_stress_mode = flag; -} - -static VALUE -gc_stress_set_m(rb_execution_context_t *ec, VALUE self, VALUE flag) -{ - rb_gc_stress_set(flag); return flag; } diff --git a/internal/gc.h b/internal/gc.h index 82a1b901f8af59..5b1180fd91ccdc 100644 --- a/internal/gc.h +++ b/internal/gc.h @@ -223,7 +223,7 @@ void rb_gc_remove_weak(VALUE parent_obj, VALUE *ptr); void rb_gc_ref_update_table_values_only(st_table *tbl); -void rb_gc_stress_set(VALUE flag); +void rb_gc_initial_stress_set(VALUE flag); #define rb_gc_mark_and_move_ptr(ptr) do { \ VALUE _obj = (VALUE)*(ptr); \ From 19916bac968fa79437649821932a6b8869414002 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Tue, 26 Mar 2024 11:37:54 -0400 Subject: [PATCH 101/211] Revert "skip `test_gc_stress_at_startup`" This reverts commit 3680981c7b71df8c3a426164787ccefe5296bb25. --- test/ruby/test_gc.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/test/ruby/test_gc.rb b/test/ruby/test_gc.rb index 470e53fc1fb3f0..39b001c3d0fee6 100644 --- a/test/ruby/test_gc.rb +++ b/test/ruby/test_gc.rb @@ -768,7 +768,6 @@ def initialize end def test_gc_stress_at_startup - omit 'TODO: fixme later' assert_in_out_err([{"RUBY_DEBUG"=>"gc_stress"}], '', [], [], '[Bug #15784]', success: true, timeout: 60) end From aa794cc5a237bf4b7aa8f1917fa4ed86949d71c6 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Tue, 26 Mar 2024 14:25:41 -0400 Subject: [PATCH 102/211] Turn GC off at boot on Windows This is to stop crashes like: .\miniruby.exe: [BUG] Segmentation fault ruby 3.4.0dev (2024-03-26T15:38:26Z pull/10370/merge 040ea2ae2f) [x64-mswin64_140] -- Control frame information ----------------------------------------------- c:0001 p:0000 s:0003 E:000d00 DUMMY [FINISH] -- Threading information --------------------------------------------------- Total ractor count: 1 Ruby thread count for this ractor: 1 -- C level backtrace information ------------------------------------------- C:\Windows\SYSTEM32\ntdll.dll(NtWaitForSingleObject+0x14) [0x00007FFA091AFC74] C:\Windows\System32\KERNELBASE.dll(WaitForSingleObjectEx+0x93) [0x00007FFA05BB4513] D:\a\ruby\ruby\build\miniruby.exe(rb_print_backtrace+0x3e) [0x00007FF64E536EFE] d:\a\ruby\ruby\src\vm_dump.c:844 D:\a\ruby\ruby\build\miniruby.exe(rb_vm_bugreport+0x1ae) [0x00007FF64E5370B2] d:\a\ruby\ruby\src\vm_dump.c:1154 D:\a\ruby\ruby\build\miniruby.exe(rb_bug_for_fatal_signal+0x77) [0x00007FF64E3FF357] d:\a\ruby\ruby\src\error.c:1087 D:\a\ruby\ruby\build\miniruby.exe(sigsegv+0x71) [0x00007FF64E4C79E5] d:\a\ruby\ruby\src\signal.c:926 C:\Windows\System32\ucrtbase.dll(seh_filter_exe+0x233) [0x00007FFA0521CE03] D:\a\ruby\ruby\build\miniruby.exe(`__scrt_common_main_seh'::`1'::filt$0+0x16) [0x00007FF64E594DA0] f:\dd\vctools\crt\vcstartup\src\startup\exe_common.inl:269 C:\Windows\SYSTEM32\VCRUNTIME140.dll(_C_specific_handler+0x9f) [0x00007FF9E54AF73F] C:\Windows\SYSTEM32\ntdll.dll(_chkstk+0x11f) [0x00007FFA091B4C2F] C:\Windows\SYSTEM32\ntdll.dll(RtlWalkFrameChain+0x14bf) [0x00007FFA09114CEF] C:\Windows\SYSTEM32\ntdll.dll(KiUserExceptionDispatcher+0x2e) [0x00007FFA091B399E] D:\a\ruby\ruby\build\miniruby.exe(newobj_of+0x6d) [0x00007FF64E418615] d:\a\ruby\ruby\src\gc.c:2949 D:\a\ruby\ruby\build\miniruby.exe(rb_wb_protected_newobj_of+0x32) [0x00007FF64E41C7DA] d:\a\ruby\ruby\src\gc.c:2974 D:\a\ruby\ruby\build\miniruby.exe(str_new0+0x64) [0x00007FF64E4E7F48] d:\a\ruby\ruby\src\string.c:887 D:\a\ruby\ruby\build\miniruby.exe(rb_enc_str_new+0x40) [0x00007FF64E4D89B8] d:\a\ruby\ruby\src\string.c:945 D:\a\ruby\ruby\build\miniruby.exe(iseq_compile_each0+0xdd7) [0x00007FF64E3B4A23] d:\a\ruby\ruby\src\compile.c:10368 D:\a\ruby\ruby\build\miniruby.exe(iseq_compile_each+0x74) [0x00007FF64E3B3C40] d:\a\ruby\ruby\src\compile.c:9971 --- gc.c | 5 +++++ vm.c | 2 ++ 2 files changed, 7 insertions(+) diff --git a/gc.c b/gc.c index 04df9dc71bf998..2b363ac7d08abc 100644 --- a/gc.c +++ b/gc.c @@ -1910,6 +1910,11 @@ rb_objspace_alloc(void) rb_darray_make_without_gc(&objspace->weak_references, 0); + // TODO: debug why on Windows Ruby crashes on boot when GC is on. +#ifdef _WIN32 + dont_gc_on(); +#endif + return objspace; } diff --git a/vm.c b/vm.c index c9d99a34ea7aa1..36f6700ad7d459 100644 --- a/vm.c +++ b/vm.c @@ -4198,7 +4198,9 @@ Init_VM(void) */ rb_define_global_const("TOPLEVEL_BINDING", rb_binding_new()); +#ifdef _WIN32 rb_objspace_gc_enable(vm->objspace); +#endif } vm_init_redefined_flag(); From e4d6479730fa87fc420cfe9ae6b83206737b0db5 Mon Sep 17 00:00:00 2001 From: Jeremy Evans Date: Fri, 22 Mar 2024 13:53:35 -0700 Subject: [PATCH 103/211] Add array/hash implicit allocation tests These are designed to prevent allocation regressions (commits that increase the number of implicitly allocated arrays and hashes). We have already had three commits in the last couple weeks to fix allocation regressions: * 15dc3aaa311b32203d8ffb414bcf9b8e55ce5691 * aceee71c35e0b387691836e756b4e008efd84cf1 * c38878494377c94f2425a81e598260ea944ef7f3 This test suite should hopefully allow us to find such regressions in CI before commit, to avoid committing future allocation regressions. This uses assert_separately around each set of tests. Doing it for each individual check was too slow. Failures are gathered and reported at the end of the the suite as a single assertion, with the message describing all failures. --- test/ruby/test_allocation.rb | 656 +++++++++++++++++++++++++++++++++++ 1 file changed, 656 insertions(+) create mode 100644 test/ruby/test_allocation.rb diff --git a/test/ruby/test_allocation.rb b/test/ruby/test_allocation.rb new file mode 100644 index 00000000000000..48348c0fbd514d --- /dev/null +++ b/test/ruby/test_allocation.rb @@ -0,0 +1,656 @@ +# frozen_string_literal: false +require 'test/unit' + +class TestAllocation < Test::Unit::TestCase + def check_allocations(checks) + assert_separately([], <<~RUBY) + $allocations = [0, 0] + $counts = {} + failures = [] + + def self.num_allocations + ObjectSpace.count_objects($counts) + arrays = $counts[:T_ARRAY] + hashes = $counts[:T_HASH] + yield + ObjectSpace.count_objects($counts) + arrays -= $counts[:T_ARRAY] + hashes -= $counts[:T_HASH] + $allocations[0] = -arrays + $allocations[1] = -hashes + end + + define_singleton_method(:check_allocations) do |num_arrays, num_hashes, check_code| + instance_eval <<~RB + empty_array = empty_array = [] + empty_hash = empty_hash = {} + array1 = array1 = [1] + hash1 = hash1 = {a: 2} + nill = nill = nil + block = block = lambda{} + + num_allocations do + \#{check_code} + end + RB + + if num_arrays != $allocations[0] + failures << "Expected \#{num_arrays} array allocations for \#{check_code.inspect}, but \#{$allocations[0]} arrays allocated" + end + if num_hashes != $allocations[1] + failures << "Expected \#{num_hashes} hash allocations for \#{check_code.inspect}, but \#{$allocations[1]} hashes allocated" + end + end + + GC.start + GC.disable + + #{checks} + + unless failures.empty? + assert_equal(true, false, failures.join("\n")) + end + RUBY + end + + class Literal < self + def test_array_literal + check_allocations(<<~RUBY) + check_allocations(1, 0, "[]") + check_allocations(1, 0, "[1]") + check_allocations(1, 0, "[*empty_array]") + check_allocations(1, 0, "[*empty_array, 1, *empty_array]") + check_allocations(1, 0, "[*empty_array, *empty_array]") + check_allocations(1, 0, "[#{'1,'*100000}]") + RUBY + end + + def test_hash_literal + check_allocations(<<~RUBY) + check_allocations(0, 1, "{}") + check_allocations(0, 1, "{a: 1}") + check_allocations(0, 1, "{**empty_hash}") + check_allocations(0, 1, "{**empty_hash, a: 1, **empty_hash}") + check_allocations(0, 1, "{**empty_hash, **empty_hash}") + check_allocations(0, 1, "{#{100000.times.map{|i| "a#{i}: 1"}.join(',')}}") + RUBY + end + end + + class MethodCall < self + def block + '' + end + + def test_no_parameters + only_block = block.empty? ? block : block[2..] + check_allocations(<<~RUBY) + def self.none(#{only_block}); end + + check_allocations(0, 0, "none(#{only_block})") + check_allocations(0, 0, "none(*empty_array#{block})") + check_allocations(0, 0, "none(**empty_hash#{block})") + check_allocations(0, 0, "none(*empty_array, **empty_hash#{block})") + + check_allocations(1, 0, "none(*empty_array, *empty_array#{block})") + check_allocations(0, 1, "none(**empty_hash, **empty_hash#{block})") + check_allocations(1, 1, "none(*empty_array, *empty_array, **empty_hash, **empty_hash#{block})") + RUBY + end + + def test_required_parameter + check_allocations(<<~RUBY) + def self.required(x#{block}); end + + check_allocations(0, 0, "required(1#{block})") + check_allocations(0, 0, "required(1, *empty_array#{block})") + check_allocations(0, 0, "required(1, **empty_hash#{block})") + check_allocations(0, 0, "required(1, *empty_array, **empty_hash#{block})") + + check_allocations(0, 0, "required(*array1#{block})") + check_allocations(0, 1, "required(**hash1#{block})") + + check_allocations(1, 0, "required(*array1, *empty_array#{block})") + check_allocations(0, 1, "required(**hash1, **empty_hash#{block})") + check_allocations(1, 0, "required(*array1, *empty_array, **empty_hash#{block})") + + # Currently allocates 1 array unnecessarily due to splatarray true + check_allocations(1, 1, "required(*empty_array, **hash1, **empty_hash#{block})") + RUBY + end + + def test_optional_parameter + check_allocations(<<~RUBY) + def self.optional(x=nil#{block}); end + + check_allocations(0, 0, "optional(1#{block})") + check_allocations(0, 0, "optional(1, *empty_array#{block})") + check_allocations(0, 0, "optional(1, **empty_hash#{block})") + check_allocations(0, 0, "optional(1, *empty_array, **empty_hash#{block})") + + check_allocations(0, 0, "optional(*array1#{block})") + check_allocations(0, 1, "optional(**hash1#{block})") + + check_allocations(1, 0, "optional(*array1, *empty_array#{block})") + check_allocations(0, 1, "optional(**hash1, **empty_hash#{block})") + check_allocations(1, 0, "optional(*array1, *empty_array, **empty_hash#{block})") + + # Currently allocates 1 array unnecessarily due to splatarray true + check_allocations(1, 1, "optional(*empty_array, **hash1, **empty_hash#{block})") + RUBY + end + + def test_positional_splat_parameter + check_allocations(<<~RUBY) + def self.splat(*x#{block}); end + + check_allocations(1, 0, "splat(1#{block})") + check_allocations(1, 0, "splat(1, *empty_array#{block})") + check_allocations(1, 0, "splat(1, **empty_hash#{block})") + check_allocations(1, 0, "splat(1, *empty_array, **empty_hash#{block})") + + check_allocations(1, 0, "splat(*array1#{block})") + check_allocations(1, 0, "splat(*array1, *empty_array#{block})") + check_allocations(1, 0, "splat(*array1, **empty_hash#{block})") + check_allocations(1, 0, "splat(*array1, *empty_array, **empty_hash#{block})") + + check_allocations(1, 0, "splat(1, *array1#{block})") + check_allocations(1, 0, "splat(1, *array1, *empty_array#{block})") + check_allocations(1, 0, "splat(1, *array1, **empty_hash#{block})") + check_allocations(1, 0, "splat(1, *array1, *empty_array, **empty_hash#{block})") + + check_allocations(1, 0, "splat(*array1#{block})") + check_allocations(1, 1, "splat(**hash1#{block})") + + check_allocations(1, 0, "splat(*array1, *empty_array#{block})") + check_allocations(1, 1, "splat(**hash1, **empty_hash#{block})") + check_allocations(1, 0, "splat(*array1, *empty_array, **empty_hash#{block})") + check_allocations(1, 1, "splat(*empty_array, **hash1, **empty_hash#{block})") + RUBY + end + + def test_required_and_positional_splat_parameters + check_allocations(<<~RUBY) + def self.req_splat(x, *y#{block}); end + + check_allocations(1, 0, "req_splat(1#{block})") + check_allocations(1, 0, "req_splat(1, *empty_array#{block})") + check_allocations(1, 0, "req_splat(1, **empty_hash#{block})") + check_allocations(1, 0, "req_splat(1, *empty_array, **empty_hash#{block})") + + check_allocations(1, 0, "req_splat(*array1#{block})") + check_allocations(1, 0, "req_splat(*array1, *empty_array#{block})") + check_allocations(1, 0, "req_splat(*array1, **empty_hash#{block})") + check_allocations(1, 0, "req_splat(*array1, *empty_array, **empty_hash#{block})") + + check_allocations(1, 0, "req_splat(1, *array1#{block})") + check_allocations(1, 0, "req_splat(1, *array1, *empty_array#{block})") + check_allocations(1, 0, "req_splat(1, *array1, **empty_hash#{block})") + check_allocations(1, 0, "req_splat(1, *array1, *empty_array, **empty_hash#{block})") + + check_allocations(1, 0, "req_splat(*array1#{block})") + check_allocations(1, 1, "req_splat(**hash1#{block})") + + check_allocations(1, 0, "req_splat(*array1, *empty_array#{block})") + check_allocations(1, 1, "req_splat(**hash1, **empty_hash#{block})") + check_allocations(1, 0, "req_splat(*array1, *empty_array, **empty_hash#{block})") + check_allocations(1, 1, "req_splat(*empty_array, **hash1, **empty_hash#{block})") + RUBY + end + + def test_positional_splat_and_post_parameters + check_allocations(<<~RUBY) + def self.splat_post(*x, y#{block}); end + + check_allocations(1, 0, "splat_post(1#{block})") + check_allocations(1, 0, "splat_post(1, *empty_array#{block})") + check_allocations(1, 0, "splat_post(1, **empty_hash#{block})") + check_allocations(1, 0, "splat_post(1, *empty_array, **empty_hash#{block})") + + check_allocations(1, 0, "splat_post(*array1#{block})") + check_allocations(1, 0, "splat_post(*array1, *empty_array#{block})") + check_allocations(1, 0, "splat_post(*array1, **empty_hash#{block})") + check_allocations(1, 0, "splat_post(*array1, *empty_array, **empty_hash#{block})") + + check_allocations(1, 0, "splat_post(1, *array1#{block})") + check_allocations(1, 0, "splat_post(1, *array1, *empty_array#{block})") + check_allocations(1, 0, "splat_post(1, *array1, **empty_hash#{block})") + check_allocations(1, 0, "splat_post(1, *array1, *empty_array, **empty_hash#{block})") + + check_allocations(1, 0, "splat_post(*array1#{block})") + check_allocations(1, 1, "splat_post(**hash1#{block})") + + check_allocations(1, 0, "splat_post(*array1, *empty_array#{block})") + check_allocations(1, 1, "splat_post(**hash1, **empty_hash#{block})") + check_allocations(1, 0, "splat_post(*array1, *empty_array, **empty_hash#{block})") + check_allocations(1, 1, "splat_post(*empty_array, **hash1, **empty_hash#{block})") + RUBY + end + + def test_keyword_parameter + check_allocations(<<~RUBY) + def self.keyword(a: nil#{block}); end + + check_allocations(0, 0, "keyword(a: 2#{block})") + check_allocations(0, 0, "keyword(*empty_array, a: 2#{block})") + check_allocations(0, 1, "keyword(a:2, **empty_hash#{block})") + check_allocations(0, 1, "keyword(**empty_hash, a: 2#{block})") + + check_allocations(0, 0, "keyword(**nil#{block})") + check_allocations(0, 0, "keyword(**empty_hash#{block})") + check_allocations(0, 0, "keyword(**hash1#{block})") + check_allocations(0, 0, "keyword(*empty_array, **hash1#{block})") + check_allocations(0, 1, "keyword(**hash1, **empty_hash#{block})") + check_allocations(0, 1, "keyword(**empty_hash, **hash1#{block})") + + check_allocations(0, 0, "keyword(*empty_array#{block})") + check_allocations(0, 1, "keyword(**hash1, **empty_hash#{block})") + check_allocations(1, 0, "keyword(*empty_array, *empty_array, **empty_hash#{block})") + + check_allocations(0, 0, "keyword(*empty_array#{block})") + check_allocations(0, 1, "keyword(**hash1, **empty_hash#{block})") + check_allocations(1, 0, "keyword(*empty_array, *empty_array, **empty_hash#{block})") + + # Currently allocates 1 array unnecessarily due to splatarray true + check_allocations(1, 1, "keyword(*empty_array, a: 2, **empty_hash#{block})") + check_allocations(1, 1, "keyword(*empty_array, **hash1, **empty_hash#{block})") + RUBY + end + + def test_keyword_splat_parameter + check_allocations(<<~RUBY) + def self.keyword_splat(**kw#{block}); end + + check_allocations(0, 1, "keyword_splat(a: 2#{block})") + check_allocations(0, 1, "keyword_splat(*empty_array, a: 2#{block})") + check_allocations(0, 1, "keyword_splat(a:2, **empty_hash#{block})") + check_allocations(0, 1, "keyword_splat(**empty_hash, a: 2#{block})") + + check_allocations(0, 1, "keyword_splat(**nil#{block})") + check_allocations(0, 1, "keyword_splat(**empty_hash#{block})") + check_allocations(0, 1, "keyword_splat(**hash1#{block})") + check_allocations(0, 1, "keyword_splat(*empty_array, **hash1#{block})") + check_allocations(0, 1, "keyword_splat(**hash1, **empty_hash#{block})") + check_allocations(0, 1, "keyword_splat(**empty_hash, **hash1#{block})") + + check_allocations(0, 1, "keyword_splat(*empty_array#{block})") + check_allocations(0, 1, "keyword_splat(**hash1, **empty_hash#{block})") + check_allocations(1, 1, "keyword_splat(*empty_array, *empty_array, **empty_hash#{block})") + + check_allocations(0, 1, "keyword_splat(*empty_array#{block})") + check_allocations(0, 1, "keyword_splat(**hash1, **empty_hash#{block})") + check_allocations(1, 1, "keyword_splat(*empty_array, *empty_array, **empty_hash#{block})") + + # Currently allocates 1 array unnecessarily due to splatarray true + check_allocations(1, 1, "keyword_splat(*empty_array, a: 2, **empty_hash#{block})") + check_allocations(1, 1, "keyword_splat(*empty_array, **hash1, **empty_hash#{block})") + RUBY + end + + def test_keyword_and_keyword_splat_parameter + check_allocations(<<~RUBY) + def self.keyword_and_keyword_splat(a: 1, **kw#{block}); end + + check_allocations(0, 1, "keyword_and_keyword_splat(a: 2#{block})") + check_allocations(0, 1, "keyword_and_keyword_splat(*empty_array, a: 2#{block})") + check_allocations(0, 1, "keyword_and_keyword_splat(a:2, **empty_hash#{block})") + check_allocations(0, 1, "keyword_and_keyword_splat(**empty_hash, a: 2#{block})") + + check_allocations(0, 1, "keyword_and_keyword_splat(**nil#{block})") + check_allocations(0, 1, "keyword_and_keyword_splat(**empty_hash#{block})") + check_allocations(0, 1, "keyword_and_keyword_splat(**hash1#{block})") + check_allocations(0, 1, "keyword_and_keyword_splat(*empty_array, **hash1#{block})") + check_allocations(0, 1, "keyword_and_keyword_splat(**hash1, **empty_hash#{block})") + check_allocations(0, 1, "keyword_and_keyword_splat(**empty_hash, **hash1#{block})") + + check_allocations(0, 1, "keyword_and_keyword_splat(*empty_array#{block})") + check_allocations(0, 1, "keyword_and_keyword_splat(**hash1, **empty_hash#{block})") + check_allocations(1, 1, "keyword_and_keyword_splat(*empty_array, *empty_array, **empty_hash#{block})") + + check_allocations(0, 1, "keyword_and_keyword_splat(*empty_array#{block})") + check_allocations(0, 1, "keyword_and_keyword_splat(**hash1, **empty_hash#{block})") + check_allocations(1, 1, "keyword_and_keyword_splat(*empty_array, *empty_array, **empty_hash#{block})") + + # Currently allocates 1 array unnecessarily due to splatarray true + check_allocations(1, 1, "keyword_and_keyword_splat(*empty_array, a: 2, **empty_hash#{block})") + check_allocations(1, 1, "keyword_and_keyword_splat(*empty_array, **hash1, **empty_hash#{block})") + RUBY + end + + def test_required_positional_and_keyword_parameter + check_allocations(<<~RUBY) + def self.required_and_keyword(b, a: nil#{block}); end + + check_allocations(0, 0, "required_and_keyword(1, a: 2#{block})") + check_allocations(0, 0, "required_and_keyword(1, *empty_array, a: 2#{block})") + check_allocations(0, 1, "required_and_keyword(1, a:2, **empty_hash#{block})") + check_allocations(0, 1, "required_and_keyword(1, **empty_hash, a: 2#{block})") + + check_allocations(0, 0, "required_and_keyword(1, **nil#{block})") + check_allocations(0, 0, "required_and_keyword(1, **empty_hash#{block})") + check_allocations(0, 0, "required_and_keyword(1, **hash1#{block})") + check_allocations(0, 0, "required_and_keyword(1, *empty_array, **hash1#{block})") + check_allocations(0, 1, "required_and_keyword(1, **hash1, **empty_hash#{block})") + check_allocations(0, 1, "required_and_keyword(1, **empty_hash, **hash1#{block})") + + check_allocations(0, 0, "required_and_keyword(1, *empty_array#{block})") + check_allocations(0, 1, "required_and_keyword(1, **hash1, **empty_hash#{block})") + check_allocations(1, 0, "required_and_keyword(1, *empty_array, *empty_array, **empty_hash#{block})") + + check_allocations(0, 0, "required_and_keyword(*array1, a: 2#{block})") + + check_allocations(0, 0, "required_and_keyword(*array1, **nill#{block})") + check_allocations(0, 0, "required_and_keyword(*array1, **empty_hash#{block})") + check_allocations(0, 0, "required_and_keyword(*array1, **hash1#{block})") + check_allocations(1, 0, "required_and_keyword(*array1, *empty_array, **hash1#{block})") + + check_allocations(1, 0, "required_and_keyword(*array1, *empty_array#{block})") + check_allocations(1, 0, "required_and_keyword(*array1, *empty_array, **empty_hash#{block})") + + check_allocations(1, 1, "required_and_keyword(*array1, *empty_array, a: 2, **empty_hash#{block})") + check_allocations(1, 1, "required_and_keyword(*array1, *empty_array, **hash1, **empty_hash#{block})") + + # Currently allocates 1 array unnecessarily due to splatarray true + check_allocations(1, 1, "required_and_keyword(1, *empty_array, a: 2, **empty_hash#{block})") + check_allocations(1, 1, "required_and_keyword(1, *empty_array, **hash1, **empty_hash#{block})") + check_allocations(1, 1, "required_and_keyword(*array1, **empty_hash, a: 2#{block})") + check_allocations(1, 1, "required_and_keyword(*array1, **hash1, **empty_hash#{block})") + check_allocations(1, 0, "required_and_keyword(*array1, **nil#{block})") + RUBY + end + + def test_positional_splat_and_keyword_parameter + check_allocations(<<~RUBY) + def self.splat_and_keyword(*b, a: nil#{block}); end + + check_allocations(1, 0, "splat_and_keyword(1, a: 2#{block})") + check_allocations(1, 0, "splat_and_keyword(1, *empty_array, a: 2#{block})") + check_allocations(1, 1, "splat_and_keyword(1, a:2, **empty_hash#{block})") + check_allocations(1, 1, "splat_and_keyword(1, **empty_hash, a: 2#{block})") + + check_allocations(1, 0, "splat_and_keyword(1, **nil#{block})") + check_allocations(1, 0, "splat_and_keyword(1, **empty_hash#{block})") + check_allocations(1, 0, "splat_and_keyword(1, **hash1#{block})") + check_allocations(1, 0, "splat_and_keyword(1, *empty_array, **hash1#{block})") + check_allocations(1, 1, "splat_and_keyword(1, **hash1, **empty_hash#{block})") + check_allocations(1, 1, "splat_and_keyword(1, **empty_hash, **hash1#{block})") + + check_allocations(1, 0, "splat_and_keyword(1, *empty_array#{block})") + check_allocations(1, 1, "splat_and_keyword(1, **hash1, **empty_hash#{block})") + check_allocations(1, 0, "splat_and_keyword(1, *empty_array, *empty_array, **empty_hash#{block})") + + check_allocations(1, 0, "splat_and_keyword(*array1, a: 2#{block})") + + check_allocations(1, 0, "splat_and_keyword(*array1, **nill#{block})") + check_allocations(1, 0, "splat_and_keyword(*array1, **empty_hash#{block})") + check_allocations(1, 0, "splat_and_keyword(*array1, **hash1#{block})") + check_allocations(1, 0, "splat_and_keyword(*array1, *empty_array, **hash1#{block})") + + check_allocations(1, 0, "splat_and_keyword(*array1, *empty_array#{block})") + check_allocations(1, 0, "splat_and_keyword(*array1, *empty_array, **empty_hash#{block})") + + check_allocations(1, 1, "splat_and_keyword(*array1, *empty_array, a: 2, **empty_hash#{block})") + check_allocations(1, 1, "splat_and_keyword(*array1, *empty_array, **hash1, **empty_hash#{block})") + + check_allocations(1, 1, "splat_and_keyword(1, *empty_array, a: 2, **empty_hash#{block})") + check_allocations(1, 1, "splat_and_keyword(1, *empty_array, **hash1, **empty_hash#{block})") + check_allocations(1, 1, "splat_and_keyword(*array1, **empty_hash, a: 2#{block})") + check_allocations(1, 1, "splat_and_keyword(*array1, **hash1, **empty_hash#{block})") + check_allocations(1, 0, "splat_and_keyword(*array1, **nil#{block})") + RUBY + end + + def test_required_and_keyword_splat_parameter + check_allocations(<<~RUBY) + def self.required_and_keyword_splat(b, **kw#{block}); end + + check_allocations(0, 1, "required_and_keyword_splat(1, a: 2#{block})") + check_allocations(0, 1, "required_and_keyword_splat(1, *empty_array, a: 2#{block})") + check_allocations(0, 1, "required_and_keyword_splat(1, a:2, **empty_hash#{block})") + check_allocations(0, 1, "required_and_keyword_splat(1, **empty_hash, a: 2#{block})") + + check_allocations(0, 1, "required_and_keyword_splat(1, **nil#{block})") + check_allocations(0, 1, "required_and_keyword_splat(1, **empty_hash#{block})") + check_allocations(0, 1, "required_and_keyword_splat(1, **hash1#{block})") + check_allocations(0, 1, "required_and_keyword_splat(1, *empty_array, **hash1#{block})") + check_allocations(0, 1, "required_and_keyword_splat(1, **hash1, **empty_hash#{block})") + check_allocations(0, 1, "required_and_keyword_splat(1, **empty_hash, **hash1#{block})") + + check_allocations(0, 1, "required_and_keyword_splat(1, *empty_array#{block})") + check_allocations(0, 1, "required_and_keyword_splat(1, **hash1, **empty_hash#{block})") + check_allocations(1, 1, "required_and_keyword_splat(1, *empty_array, *empty_array, **empty_hash#{block})") + + check_allocations(0, 1, "required_and_keyword_splat(*array1, a: 2#{block})") + + check_allocations(0, 1, "required_and_keyword_splat(*array1, **nill#{block})") + check_allocations(0, 1, "required_and_keyword_splat(*array1, **empty_hash#{block})") + check_allocations(0, 1, "required_and_keyword_splat(*array1, **hash1#{block})") + check_allocations(1, 1, "required_and_keyword_splat(*array1, *empty_array, **hash1#{block})") + + check_allocations(1, 1, "required_and_keyword_splat(*array1, *empty_array#{block})") + check_allocations(1, 1, "required_and_keyword_splat(*array1, *empty_array, **empty_hash#{block})") + + check_allocations(1, 1, "required_and_keyword_splat(*array1, *empty_array, a: 2, **empty_hash#{block})") + check_allocations(1, 1, "required_and_keyword_splat(*array1, *empty_array, **hash1, **empty_hash#{block})") + + # Currently allocates 1 array unnecessarily due to splatarray true + check_allocations(1, 1, "required_and_keyword_splat(1, *empty_array, a: 2, **empty_hash#{block})") + check_allocations(1, 1, "required_and_keyword_splat(1, *empty_array, **hash1, **empty_hash#{block})") + check_allocations(1, 1, "required_and_keyword_splat(*array1, **empty_hash, a: 2#{block})") + check_allocations(1, 1, "required_and_keyword_splat(*array1, **hash1, **empty_hash#{block})") + check_allocations(1, 1, "required_and_keyword_splat(*array1, **nil#{block})") + RUBY + end + + def test_positional_splat_and_keyword_splat_parameter + check_allocations(<<~RUBY) + def self.splat_and_keyword_splat(*b, **kw#{block}); end + + check_allocations(1, 1, "splat_and_keyword_splat(1, a: 2#{block})") + check_allocations(1, 1, "splat_and_keyword_splat(1, *empty_array, a: 2#{block})") + check_allocations(1, 1, "splat_and_keyword_splat(1, a:2, **empty_hash#{block})") + check_allocations(1, 1, "splat_and_keyword_splat(1, **empty_hash, a: 2#{block})") + + check_allocations(1, 1, "splat_and_keyword_splat(1, **nil#{block})") + check_allocations(1, 1, "splat_and_keyword_splat(1, **empty_hash#{block})") + check_allocations(1, 1, "splat_and_keyword_splat(1, **hash1#{block})") + check_allocations(1, 1, "splat_and_keyword_splat(1, *empty_array, **hash1#{block})") + check_allocations(1, 1, "splat_and_keyword_splat(1, **hash1, **empty_hash#{block})") + check_allocations(1, 1, "splat_and_keyword_splat(1, **empty_hash, **hash1#{block})") + + check_allocations(1, 1, "splat_and_keyword_splat(1, *empty_array#{block})") + check_allocations(1, 1, "splat_and_keyword_splat(1, **hash1, **empty_hash#{block})") + check_allocations(1, 1, "splat_and_keyword_splat(1, *empty_array, *empty_array, **empty_hash#{block})") + + check_allocations(1, 1, "splat_and_keyword_splat(*array1, a: 2#{block})") + + check_allocations(1, 1, "splat_and_keyword_splat(*array1, **nill#{block})") + check_allocations(1, 1, "splat_and_keyword_splat(*array1, **empty_hash#{block})") + check_allocations(1, 1, "splat_and_keyword_splat(*array1, **hash1#{block})") + check_allocations(1, 1, "splat_and_keyword_splat(*array1, *empty_array, **hash1#{block})") + + check_allocations(1, 1, "splat_and_keyword_splat(*array1, *empty_array#{block})") + check_allocations(1, 1, "splat_and_keyword_splat(*array1, *empty_array, **empty_hash#{block})") + + check_allocations(1, 1, "splat_and_keyword_splat(*array1, *empty_array, a: 2, **empty_hash#{block})") + check_allocations(1, 1, "splat_and_keyword_splat(*array1, *empty_array, **hash1, **empty_hash#{block})") + + check_allocations(1, 1, "splat_and_keyword_splat(1, *empty_array, a: 2, **empty_hash#{block})") + check_allocations(1, 1, "splat_and_keyword_splat(1, *empty_array, **hash1, **empty_hash#{block})") + check_allocations(1, 1, "splat_and_keyword_splat(*array1, **empty_hash, a: 2#{block})") + check_allocations(1, 1, "splat_and_keyword_splat(*array1, **hash1, **empty_hash#{block})") + check_allocations(1, 1, "splat_and_keyword_splat(*array1, **nil#{block})") + RUBY + end + + def test_anonymous_splat_and_anonymous_keyword_splat_parameters + check_allocations(<<~RUBY) + def self.anon_splat_and_anon_keyword_splat(*, **#{block}); end + + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(1, a: 2#{block})") + check_allocations(1, 0, "anon_splat_and_anon_keyword_splat(1, *empty_array, a: 2#{block})") + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(1, a:2, **empty_hash#{block})") + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(1, **empty_hash, a: 2#{block})") + + check_allocations(1, 0, "anon_splat_and_anon_keyword_splat(1, **nil#{block})") + check_allocations(1, 0, "anon_splat_and_anon_keyword_splat(1, **empty_hash#{block})") + check_allocations(1, 0, "anon_splat_and_anon_keyword_splat(1, **hash1#{block})") + check_allocations(1, 0, "anon_splat_and_anon_keyword_splat(1, *empty_array, **hash1#{block})") + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(1, **hash1, **empty_hash#{block})") + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(1, **empty_hash, **hash1#{block})") + + check_allocations(1, 0, "anon_splat_and_anon_keyword_splat(1, *empty_array#{block})") + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(1, **hash1, **empty_hash#{block})") + check_allocations(1, 0, "anon_splat_and_anon_keyword_splat(1, *empty_array, *empty_array, **empty_hash#{block})") + + check_allocations(0, 0, "anon_splat_and_anon_keyword_splat(*array1, a: 2#{block})") + + check_allocations(0, 0, "anon_splat_and_anon_keyword_splat(*array1, **nill#{block})") + check_allocations(0, 0, "anon_splat_and_anon_keyword_splat(*array1, **empty_hash#{block})") + check_allocations(0, 0, "anon_splat_and_anon_keyword_splat(*array1, **hash1#{block})") + check_allocations(1, 0, "anon_splat_and_anon_keyword_splat(*array1, *empty_array, **hash1#{block})") + + check_allocations(1, 0, "anon_splat_and_anon_keyword_splat(*array1, *empty_array#{block})") + check_allocations(1, 0, "anon_splat_and_anon_keyword_splat(*array1, *empty_array, **empty_hash#{block})") + + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(*array1, *empty_array, a: 2, **empty_hash#{block})") + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(*array1, *empty_array, **hash1, **empty_hash#{block})") + + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(1, *empty_array, a: 2, **empty_hash#{block})") + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(1, *empty_array, **hash1, **empty_hash#{block})") + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(*array1, **empty_hash, a: 2#{block})") + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(*array1, **hash1, **empty_hash#{block})") + check_allocations(1, 0, "anon_splat_and_anon_keyword_splat(*array1, **nil#{block})") + RUBY + end + + def test_nested_anonymous_splat_and_anonymous_keyword_splat_parameters + check_allocations(<<~RUBY) + def self.anon_splat_and_anon_keyword_splat(*, **#{block}); t(*, **) end; def self.t(*, **#{block}); end + + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(1, a: 2#{block})") + check_allocations(1, 0, "anon_splat_and_anon_keyword_splat(1, *empty_array, a: 2#{block})") + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(1, a:2, **empty_hash#{block})") + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(1, **empty_hash, a: 2#{block})") + + check_allocations(1, 0, "anon_splat_and_anon_keyword_splat(1, **nil#{block})") + check_allocations(1, 0, "anon_splat_and_anon_keyword_splat(1, **empty_hash#{block})") + check_allocations(1, 0, "anon_splat_and_anon_keyword_splat(1, **hash1#{block})") + check_allocations(1, 0, "anon_splat_and_anon_keyword_splat(1, *empty_array, **hash1#{block})") + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(1, **hash1, **empty_hash#{block})") + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(1, **empty_hash, **hash1#{block})") + + check_allocations(1, 0, "anon_splat_and_anon_keyword_splat(1, *empty_array#{block})") + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(1, **hash1, **empty_hash#{block})") + check_allocations(1, 0, "anon_splat_and_anon_keyword_splat(1, *empty_array, *empty_array, **empty_hash#{block})") + + check_allocations(0, 0, "anon_splat_and_anon_keyword_splat(*array1, a: 2#{block})") + + check_allocations(0, 0, "anon_splat_and_anon_keyword_splat(*array1, **nill#{block})") + check_allocations(0, 0, "anon_splat_and_anon_keyword_splat(*array1, **empty_hash#{block})") + check_allocations(0, 0, "anon_splat_and_anon_keyword_splat(*array1, **hash1#{block})") + check_allocations(1, 0, "anon_splat_and_anon_keyword_splat(*array1, *empty_array, **hash1#{block})") + + check_allocations(1, 0, "anon_splat_and_anon_keyword_splat(*array1, *empty_array#{block})") + check_allocations(1, 0, "anon_splat_and_anon_keyword_splat(*array1, *empty_array, **empty_hash#{block})") + + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(*array1, *empty_array, a: 2, **empty_hash#{block})") + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(*array1, *empty_array, **hash1, **empty_hash#{block})") + + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(1, *empty_array, a: 2, **empty_hash#{block})") + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(1, *empty_array, **hash1, **empty_hash#{block})") + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(*array1, **empty_hash, a: 2#{block})") + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(*array1, **hash1, **empty_hash#{block})") + check_allocations(1, 0, "anon_splat_and_anon_keyword_splat(*array1, **nil#{block})") + RUBY + end + + def test_argument_forwarding + check_allocations(<<~RUBY) + def self.argument_forwarding(...); end + + check_allocations(1, 1, "argument_forwarding(1, a: 2#{block})") + check_allocations(1, 0, "argument_forwarding(1, *empty_array, a: 2#{block})") + check_allocations(1, 1, "argument_forwarding(1, a:2, **empty_hash#{block})") + check_allocations(1, 1, "argument_forwarding(1, **empty_hash, a: 2#{block})") + + check_allocations(1, 0, "argument_forwarding(1, **nil#{block})") + check_allocations(1, 0, "argument_forwarding(1, **empty_hash#{block})") + check_allocations(1, 0, "argument_forwarding(1, **hash1#{block})") + check_allocations(1, 0, "argument_forwarding(1, *empty_array, **hash1#{block})") + check_allocations(1, 1, "argument_forwarding(1, **hash1, **empty_hash#{block})") + check_allocations(1, 1, "argument_forwarding(1, **empty_hash, **hash1#{block})") + + check_allocations(1, 0, "argument_forwarding(1, *empty_array#{block})") + check_allocations(1, 1, "argument_forwarding(1, **hash1, **empty_hash#{block})") + check_allocations(1, 0, "argument_forwarding(1, *empty_array, *empty_array, **empty_hash#{block})") + + check_allocations(0, 0, "argument_forwarding(*array1, a: 2#{block})") + + check_allocations(0, 0, "argument_forwarding(*array1, **nill#{block})") + check_allocations(0, 0, "argument_forwarding(*array1, **empty_hash#{block})") + check_allocations(0, 0, "argument_forwarding(*array1, **hash1#{block})") + check_allocations(1, 0, "argument_forwarding(*array1, *empty_array, **hash1#{block})") + + check_allocations(1, 0, "argument_forwarding(*array1, *empty_array#{block})") + check_allocations(1, 0, "argument_forwarding(*array1, *empty_array, **empty_hash#{block})") + + check_allocations(1, 1, "argument_forwarding(*array1, *empty_array, a: 2, **empty_hash#{block})") + check_allocations(1, 1, "argument_forwarding(*array1, *empty_array, **hash1, **empty_hash#{block})") + + check_allocations(1, 1, "argument_forwarding(1, *empty_array, a: 2, **empty_hash#{block})") + check_allocations(1, 1, "argument_forwarding(1, *empty_array, **hash1, **empty_hash#{block})") + check_allocations(1, 1, "argument_forwarding(*array1, **empty_hash, a: 2#{block})") + check_allocations(1, 1, "argument_forwarding(*array1, **hash1, **empty_hash#{block})") + check_allocations(1, 0, "argument_forwarding(*array1, **nil#{block})") + RUBY + end + + def test_nested_argument_forwarding + check_allocations(<<~RUBY) + def self.argument_forwarding(...); t(...) end; def self.t(...) end + + check_allocations(1, 1, "argument_forwarding(1, a: 2#{block})") + check_allocations(1, 0, "argument_forwarding(1, *empty_array, a: 2#{block})") + check_allocations(1, 1, "argument_forwarding(1, a:2, **empty_hash#{block})") + check_allocations(1, 1, "argument_forwarding(1, **empty_hash, a: 2#{block})") + + check_allocations(1, 0, "argument_forwarding(1, **nil#{block})") + check_allocations(1, 0, "argument_forwarding(1, **empty_hash#{block})") + check_allocations(1, 0, "argument_forwarding(1, **hash1#{block})") + check_allocations(1, 0, "argument_forwarding(1, *empty_array, **hash1#{block})") + check_allocations(1, 1, "argument_forwarding(1, **hash1, **empty_hash#{block})") + check_allocations(1, 1, "argument_forwarding(1, **empty_hash, **hash1#{block})") + + check_allocations(1, 0, "argument_forwarding(1, *empty_array#{block})") + check_allocations(1, 1, "argument_forwarding(1, **hash1, **empty_hash#{block})") + check_allocations(1, 0, "argument_forwarding(1, *empty_array, *empty_array, **empty_hash#{block})") + + check_allocations(0, 0, "argument_forwarding(*array1, a: 2#{block})") + + check_allocations(0, 0, "argument_forwarding(*array1, **nill#{block})") + check_allocations(0, 0, "argument_forwarding(*array1, **empty_hash#{block})") + check_allocations(0, 0, "argument_forwarding(*array1, **hash1#{block})") + check_allocations(1, 0, "argument_forwarding(*array1, *empty_array, **hash1#{block})") + + check_allocations(1, 0, "argument_forwarding(*array1, *empty_array#{block})") + check_allocations(1, 0, "argument_forwarding(*array1, *empty_array, **empty_hash#{block})") + + check_allocations(1, 1, "argument_forwarding(*array1, *empty_array, a: 2, **empty_hash#{block})") + check_allocations(1, 1, "argument_forwarding(*array1, *empty_array, **hash1, **empty_hash#{block})") + + check_allocations(1, 1, "argument_forwarding(1, *empty_array, a: 2, **empty_hash#{block})") + check_allocations(1, 1, "argument_forwarding(1, *empty_array, **hash1, **empty_hash#{block})") + check_allocations(1, 1, "argument_forwarding(*array1, **empty_hash, a: 2#{block})") + check_allocations(1, 1, "argument_forwarding(*array1, **hash1, **empty_hash#{block})") + check_allocations(1, 0, "argument_forwarding(*array1, **nil#{block})") + RUBY + end + + class WithBlock < self + def block + ', &block' + end + end + end +end From 4566843b3e9daf68984e8b051c7b11023099e788 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Tue, 26 Mar 2024 13:54:18 -0400 Subject: [PATCH 104/211] Check FL_SEEN_OBJ_ID before looking up in table This is an optimization for compaction so that we only lookup in the obj_to_id_tbl table only when FL_SEEN_OBJ_ID is set. --- gc.c | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/gc.c b/gc.c index 2b363ac7d08abc..84e9b20d4f2b8c 100644 --- a/gc.c +++ b/gc.c @@ -9559,19 +9559,23 @@ gc_move(rb_objspace_t *objspace, VALUE scan, VALUE free, size_t src_slot_size, s DURING_GC_COULD_MALLOC_REGION_END(); } - st_data_t srcid = (st_data_t)src, id; - - /* If the source object's object_id has been seen, we need to update - * the object to object id mapping. */ - if (st_lookup(objspace->obj_to_id_tbl, srcid, &id)) { - gc_report(4, objspace, "Moving object with seen id: %p -> %p\n", (void *)src, (void *)dest); - /* Resizing the st table could cause a malloc */ - DURING_GC_COULD_MALLOC_REGION_START(); - { - st_delete(objspace->obj_to_id_tbl, &srcid, 0); - st_insert(objspace->obj_to_id_tbl, (st_data_t)dest, id); + if (FL_TEST((VALUE)src, FL_SEEN_OBJ_ID)) { + st_data_t srcid = (st_data_t)src, id; + /* If the source object's object_id has been seen, we need to update + * the object to object id mapping. */ + if (st_lookup(objspace->obj_to_id_tbl, srcid, &id)) { + gc_report(4, objspace, "Moving object with seen id: %p -> %p\n", (void *)src, (void *)dest); + /* Resizing the st table could cause a malloc */ + DURING_GC_COULD_MALLOC_REGION_START(); + { + st_delete(objspace->obj_to_id_tbl, &srcid, 0); + st_insert(objspace->obj_to_id_tbl, (st_data_t)dest, id); + } + DURING_GC_COULD_MALLOC_REGION_END(); } - DURING_GC_COULD_MALLOC_REGION_END(); + } + else { + GC_ASSERT(!st_lookup(objspace->obj_to_id_tbl, (st_data_t)src, NULL)); } /* Move the object */ From c50b6425b4b5b42f1177b2c4eb6fd61d9d95b4d4 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Tue, 26 Mar 2024 13:59:21 -0400 Subject: [PATCH 105/211] Remove st_lookup when updating object ID --- gc.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/gc.c b/gc.c index 84e9b20d4f2b8c..099a1f382b23fc 100644 --- a/gc.c +++ b/gc.c @@ -9560,19 +9560,21 @@ gc_move(rb_objspace_t *objspace, VALUE scan, VALUE free, size_t src_slot_size, s } if (FL_TEST((VALUE)src, FL_SEEN_OBJ_ID)) { - st_data_t srcid = (st_data_t)src, id; /* If the source object's object_id has been seen, we need to update * the object to object id mapping. */ - if (st_lookup(objspace->obj_to_id_tbl, srcid, &id)) { - gc_report(4, objspace, "Moving object with seen id: %p -> %p\n", (void *)src, (void *)dest); - /* Resizing the st table could cause a malloc */ - DURING_GC_COULD_MALLOC_REGION_START(); - { - st_delete(objspace->obj_to_id_tbl, &srcid, 0); - st_insert(objspace->obj_to_id_tbl, (st_data_t)dest, id); + st_data_t srcid = (st_data_t)src, id; + + gc_report(4, objspace, "Moving object with seen id: %p -> %p\n", (void *)src, (void *)dest); + /* Resizing the st table could cause a malloc */ + DURING_GC_COULD_MALLOC_REGION_START(); + { + if (!st_delete(objspace->obj_to_id_tbl, &srcid, &id)) { + rb_bug("gc_move: object ID seen, but not in mapping table: %s", obj_info((VALUE)src)); } - DURING_GC_COULD_MALLOC_REGION_END(); + + st_insert(objspace->obj_to_id_tbl, (st_data_t)dest, id); } + DURING_GC_COULD_MALLOC_REGION_END(); } else { GC_ASSERT(!st_lookup(objspace->obj_to_id_tbl, (st_data_t)src, NULL)); From 9b816e674a4ecadde047212827e3bbe87cd61345 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 27 Mar 2024 12:04:15 -0400 Subject: [PATCH 106/211] [ruby/prism] Add option for inlining messages for error formatting https://github.com/ruby/prism/commit/af0204a8ab --- prism/extension.c | 2 +- prism/prism.c | 11 ++++++----- prism/prism.h | 4 +++- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/prism/extension.c b/prism/extension.c index 5b5c0593c91719..921e197783edc1 100644 --- a/prism/extension.c +++ b/prism/extension.c @@ -1184,7 +1184,7 @@ format_errors(VALUE self, VALUE source, VALUE colorize) { pm_node_t *node = pm_parse(&parser); pm_buffer_t buffer = { 0 }; - pm_parser_errors_format(&parser, &buffer, RTEST(colorize)); + pm_parser_errors_format(&parser, &parser.error_list, &buffer, RTEST(colorize), true); rb_encoding *encoding = rb_enc_find(parser.encoding->name); VALUE result = rb_enc_str_new(pm_buffer_value(&buffer), pm_buffer_length(&buffer), encoding); diff --git a/prism/prism.c b/prism/prism.c index c6353f451b586c..b8787bb9a9e408 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -19671,8 +19671,7 @@ pm_parser_errors_format_line(const pm_parser_t *parser, const pm_newline_list_t * Format the errors on the parser into the given buffer. */ PRISM_EXPORTED_FUNCTION void -pm_parser_errors_format(const pm_parser_t *parser, pm_buffer_t *buffer, bool colorize) { - const pm_list_t *error_list = &parser->error_list; +pm_parser_errors_format(const pm_parser_t *parser, const pm_list_t *error_list, pm_buffer_t *buffer, bool colorize, bool inline_messages) { assert(error_list->size != 0); // First, we're going to sort all of the errors by line number using an @@ -19831,10 +19830,12 @@ pm_parser_errors_format(const pm_parser_t *parser, pm_buffer_t *buffer, bool col column += (char_width == 0 ? 1 : char_width); } - pm_buffer_append_byte(buffer, ' '); + if (inline_messages) { + pm_buffer_append_byte(buffer, ' '); + const char *message = error->error->message; + pm_buffer_append_string(buffer, message, strlen(message)); + } - const char *message = error->error->message; - pm_buffer_append_string(buffer, message, strlen(message)); pm_buffer_append_byte(buffer, '\n'); // Here we determine how many lines of padding to display after the diff --git a/prism/prism.h b/prism/prism.h index 34540b9441954a..70e1e9df4960a7 100644 --- a/prism/prism.h +++ b/prism/prism.h @@ -222,10 +222,12 @@ const char * pm_token_type_human(pm_token_type_t token_type); * Format the errors on the parser into the given buffer. * * @param parser The parser to format the errors for. + * @param error_list The list of errors to format. * @param buffer The buffer to format the errors into. * @param colorize Whether or not to colorize the errors with ANSI escape sequences. + * @param inline_messages Whether or not to inline the messages with the source. */ -PRISM_EXPORTED_FUNCTION void pm_parser_errors_format(const pm_parser_t *parser, pm_buffer_t *buffer, bool colorize); +PRISM_EXPORTED_FUNCTION void pm_parser_errors_format(const pm_parser_t *parser, const pm_list_t *error_list, pm_buffer_t *buffer, bool colorize, bool inline_messages); // We optionally support dumping to JSON. For systems that don't want or need // this functionality, it can be turned off with the PRISM_EXCLUDE_JSON define. From a69f0047cb489c136001937442c1d2ffd8ea1dd7 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 27 Mar 2024 12:23:10 -0400 Subject: [PATCH 107/211] [PRISM] Use new error formatting API --- prism_compile.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prism_compile.c b/prism_compile.c index aec2f80f4102dc..0ac931b59bf8e9 100644 --- a/prism_compile.c +++ b/prism_compile.c @@ -8398,7 +8398,7 @@ pm_parse_process_error(const pm_parse_result_t *result) pm_buffer_append_string(&buffer, "syntax errors found\n", 20); if (valid_utf8) { - pm_parser_errors_format(&result->parser, &buffer, rb_stderr_tty_p()); + pm_parser_errors_format(&result->parser, &result->parser.error_list, &buffer, rb_stderr_tty_p(), true); } else { const pm_string_t *filepath = &result->parser.filepath; From db5686a8ba7f6157deb2b49f3e16196f1506fa83 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Thu, 28 Mar 2024 02:27:08 +0900 Subject: [PATCH 108/211] Read as binary regardless locale --- tool/sync_default_gems.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tool/sync_default_gems.rb b/tool/sync_default_gems.rb index df7e59c9563a4b..0307028eb7569b 100755 --- a/tool/sync_default_gems.rb +++ b/tool/sync_default_gems.rb @@ -461,7 +461,7 @@ def commits_in_ranges(gem, repo, default_branch, ranges) # \r? needed in the regex in case the commit has windows-style line endings (because e.g. we're running # tests on Windows) pattern = "https://github\.com/#{Regexp.quote(repo)}/commit/([0-9a-f]+)\r?$" - log = IO.popen(%W"git log -E --grep=#{pattern} -n1 --format=%B", &:read) + log = IO.popen(%W"git log -E --grep=#{pattern} -n1 --format=%B", "rb", &:read) ranges = ["#{log[%r[#{pattern}\n\s*(?i:co-authored-by:.*)*\s*\Z], 1]}..#{gem}/#{default_branch}"] end @@ -471,7 +471,7 @@ def commits_in_ranges(gem, repo, default_branch, ranges) range = "#{range}~1..#{range}" end - IO.popen(%W"git log --format=%H,%s #{range} --") do |f| + IO.popen(%W"git log --format=%H,%s #{range} --", "rb") do |f| f.read.split("\n").reverse.map{|commit| commit.split(',', 2)} end end @@ -581,7 +581,7 @@ def pickup_files(gem, changed, picked) def pickup_commit(gem, sha, edit) # Attempt to cherry-pick a commit - result = IO.popen(%W"git cherry-pick #{sha}", &:read) + result = IO.popen(%W"git cherry-pick #{sha}", "rb", &:read) picked = $?.success? if result =~ /nothing\ to\ commit/ `git reset` From 0f5ab4ad5289d6385b74e800a73de005a48737b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20Barri=C3=A9?= Date: Wed, 27 Mar 2024 17:57:51 +0100 Subject: [PATCH 109/211] [ruby/stringio] Eagerly defrost chilled strings [Feature #20390] https://github.com/ruby/stringio/commit/17ee957f34 Co-authored-by: Jean Boussier --- ext/stringio/stringio.c | 24 ++++++++++++++++++++---- test/stringio/test_stringio.rb | 21 +++++++++++++++++++++ 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/ext/stringio/stringio.c b/ext/stringio/stringio.c index 1ce90030a82126..e2caf02a6464a0 100644 --- a/ext/stringio/stringio.c +++ b/ext/stringio/stringio.c @@ -15,6 +15,8 @@ static const char *const STRINGIO_VERSION = "3.1.1"; +#include + #include "ruby.h" #include "ruby/io.h" #include "ruby/encoding.h" @@ -49,6 +51,13 @@ static long strio_write(VALUE self, VALUE str); #define IS_STRIO(obj) (rb_typeddata_is_kind_of((obj), &strio_data_type)) #define error_inval(msg) (rb_syserr_fail(EINVAL, msg)) #define get_enc(ptr) ((ptr)->enc ? (ptr)->enc : !NIL_P((ptr)->string) ? rb_enc_get((ptr)->string) : NULL) +#ifndef HAVE_RB_STR_CHILLED_P +static bool +rb_str_chilled_p(VALUE str) +{ + return false; +} +#endif static struct StringIO * strio_alloc(void) @@ -166,8 +175,14 @@ writable(VALUE strio) static void check_modifiable(struct StringIO *ptr) { - if (OBJ_FROZEN(ptr->string)) { - rb_raise(rb_eIOError, "not modifiable string"); + if (NIL_P(ptr->string)) { + /* Null device StringIO */ + } + else if (rb_str_chilled_p(ptr->string)) { + rb_str_modify(ptr->string); + } + else if (OBJ_FROZEN_RAW(ptr->string)) { + rb_raise(rb_eIOError, "not modifiable string"); } } @@ -287,7 +302,8 @@ strio_init(int argc, VALUE *argv, struct StringIO *ptr, VALUE self) else if (!argc) { string = rb_enc_str_new("", 0, rb_default_external_encoding()); } - if (!NIL_P(string) && OBJ_FROZEN_RAW(string)) { + + if (!NIL_P(string) && OBJ_FROZEN_RAW(string) && !rb_str_chilled_p(string)) { if (ptr->flags & FMODE_WRITABLE) { rb_syserr_fail(EACCES, 0); } @@ -481,7 +497,7 @@ strio_set_string(VALUE self, VALUE string) rb_io_taint_check(self); ptr->flags &= ~FMODE_READWRITE; StringValue(string); - ptr->flags = OBJ_FROZEN(string) ? FMODE_READABLE : FMODE_READWRITE; + ptr->flags = OBJ_FROZEN(string) && !rb_str_chilled_p(string) ? FMODE_READABLE : FMODE_READWRITE; ptr->pos = 0; ptr->lineno = 0; RB_OBJ_WRITE(self, &ptr->string, string); diff --git a/test/stringio/test_stringio.rb b/test/stringio/test_stringio.rb index 2af69235744e09..e17cd0abb11a01 100644 --- a/test/stringio/test_stringio.rb +++ b/test/stringio/test_stringio.rb @@ -978,6 +978,27 @@ def test_coderange_after_overwrite assert_predicate(s.string, :ascii_only?) end + if eval(%{ "test".frozen? && !"test".equal?("test") }) # Ruby 3.4+ chilled strings + def test_chilled_string + chilled_string = eval(%{""}) + io = StringIO.new(chilled_string) + assert_warning(/literal string will be frozen/) { io << "test" } + assert_equal("test", io.string) + assert_same(chilled_string, io.string) + end + + def test_chilled_string_string_set + io = StringIO.new + chilled_string = eval(%{""}) + io.string = chilled_string + assert_warning(/literal string will be frozen/) { io << "test" } + assert_equal("test", io.string) + assert_same(chilled_string, io.string) + end + end + + private + def assert_string(content, encoding, str, mesg = nil) assert_equal([content, encoding], [str, str.encoding], mesg) end From 06563d78a1e28ad97593c3caaf5137f0c64884bf Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Thu, 28 Mar 2024 02:06:48 +0900 Subject: [PATCH 110/211] [ruby/stringio] Adjust styles [ci skip] https://github.com/ruby/stringio/commit/4e8e82fc30 --- ext/stringio/stringio.c | 108 ++++++++++++++++++++-------------------- 1 file changed, 54 insertions(+), 54 deletions(-) diff --git a/ext/stringio/stringio.c b/ext/stringio/stringio.c index e2caf02a6464a0..c9f6b38ae6a2db 100644 --- a/ext/stringio/stringio.c +++ b/ext/stringio/stringio.c @@ -176,13 +176,13 @@ static void check_modifiable(struct StringIO *ptr) { if (NIL_P(ptr->string)) { - /* Null device StringIO */ + /* Null device StringIO */ } else if (rb_str_chilled_p(ptr->string)) { - rb_str_modify(ptr->string); + rb_str_modify(ptr->string); } else if (OBJ_FROZEN_RAW(ptr->string)) { - rb_raise(rb_eIOError, "not modifiable string"); + rb_raise(rb_eIOError, "not modifiable string"); } } @@ -989,10 +989,10 @@ strio_ungetbyte(VALUE self, VALUE c) if (NIL_P(ptr->string)) return Qnil; if (NIL_P(c)) return Qnil; if (RB_INTEGER_TYPE_P(c)) { - /* rb_int_and() not visible from exts */ - VALUE v = rb_funcall(c, '&', 1, INT2FIX(0xff)); - const char cc = NUM2INT(v) & 0xFF; - strio_unget_bytes(ptr, &cc, 1); + /* rb_int_and() not visible from exts */ + VALUE v = rb_funcall(c, '&', 1, INT2FIX(0xff)); + const char cc = NUM2INT(v) & 0xFF; + strio_unget_bytes(ptr, &cc, 1); } else { long cl; @@ -1173,41 +1173,41 @@ prepare_getline_args(struct StringIO *ptr, struct getline_arg *arg, int argc, VA break; case 1: - if (!NIL_P(rs) && !RB_TYPE_P(rs, T_STRING)) { - VALUE tmp = rb_check_string_type(rs); + if (!NIL_P(rs) && !RB_TYPE_P(rs, T_STRING)) { + VALUE tmp = rb_check_string_type(rs); if (NIL_P(tmp)) { - limit = NUM2LONG(rs); - rs = rb_rs; + limit = NUM2LONG(rs); + rs = rb_rs; } else { - rs = tmp; + rs = tmp; } } break; case 2: - if (!NIL_P(rs)) StringValue(rs); + if (!NIL_P(rs)) StringValue(rs); if (!NIL_P(lim)) limit = NUM2LONG(lim); break; } if (!NIL_P(ptr->string) && !NIL_P(rs)) { - rb_encoding *enc_rs, *enc_io; - enc_rs = rb_enc_get(rs); - enc_io = get_enc(ptr); - if (enc_rs != enc_io && - (rb_enc_str_coderange(rs) != ENC_CODERANGE_7BIT || - (RSTRING_LEN(rs) > 0 && !rb_enc_asciicompat(enc_io)))) { - if (rs == rb_rs) { - rs = rb_enc_str_new(0, 0, enc_io); - rb_str_buf_cat_ascii(rs, "\n"); - rs = rs; - } - else { - rb_raise(rb_eArgError, "encoding mismatch: %s IO with %s RS", - rb_enc_name(enc_io), - rb_enc_name(enc_rs)); - } - } + rb_encoding *enc_rs, *enc_io; + enc_rs = rb_enc_get(rs); + enc_io = get_enc(ptr); + if (enc_rs != enc_io && + (rb_enc_str_coderange(rs) != ENC_CODERANGE_7BIT || + (RSTRING_LEN(rs) > 0 && !rb_enc_asciicompat(enc_io)))) { + if (rs == rb_rs) { + rs = rb_enc_str_new(0, 0, enc_io); + rb_str_buf_cat_ascii(rs, "\n"); + rs = rs; + } + else { + rb_raise(rb_eArgError, "encoding mismatch: %s IO with %s RS", + rb_enc_name(enc_io), + rb_enc_name(enc_rs)); + } + } } arg->rs = rs; arg->limit = limit; @@ -1219,9 +1219,9 @@ prepare_getline_args(struct StringIO *ptr, struct getline_arg *arg, int argc, VA keywords[0] = rb_intern_const("chomp"); } rb_get_kwargs(opts, keywords, 0, 1, &vchomp); - if (respect_chomp) { + if (respect_chomp) { arg->chomp = (vchomp != Qundef) && RTEST(vchomp); - } + } } return arg; } @@ -1261,7 +1261,7 @@ strio_getline(struct getline_arg *arg, struct StringIO *ptr) str = strio_substr(ptr, ptr->pos, e - s - w, enc); } else if ((n = RSTRING_LEN(str)) == 0) { - const char *paragraph_end = NULL; + const char *paragraph_end = NULL; p = s; while (p[(p + 1 < e) && (*p == '\r') && 0] == '\n') { p += *p == '\r'; @@ -1271,18 +1271,18 @@ strio_getline(struct getline_arg *arg, struct StringIO *ptr) } s = p; while ((p = memchr(p, '\n', e - p)) && (p != e)) { - p++; - if (!((p < e && *p == '\n') || - (p + 1 < e && *p == '\r' && *(p+1) == '\n'))) { - continue; - } - paragraph_end = p - ((*(p-2) == '\r') ? 2 : 1); - while ((p < e && *p == '\n') || - (p + 1 < e && *p == '\r' && *(p+1) == '\n')) { - p += (*p == '\r') ? 2 : 1; - } - e = p; - break; + p++; + if (!((p < e && *p == '\n') || + (p + 1 < e && *p == '\r' && *(p+1) == '\n'))) { + continue; + } + paragraph_end = p - ((*(p-2) == '\r') ? 2 : 1); + while ((p < e && *p == '\n') || + (p + 1 < e && *p == '\r' && *(p+1) == '\n')) { + p += (*p == '\r') ? 2 : 1; + } + e = p; + break; } if (arg->chomp && paragraph_end) { w = e - paragraph_end; @@ -1606,7 +1606,7 @@ strio_read(int argc, VALUE *argv, VALUE self) } break; default: - rb_error_arity(argc, 0, 2); + rb_error_arity(argc, 0, 2); } if (NIL_P(str)) { rb_encoding *enc = binary ? rb_ascii8bit_encoding() : get_enc(ptr); @@ -1642,28 +1642,28 @@ strio_pread(int argc, VALUE *argv, VALUE self) long offset = NUM2LONG(rb_offset); if (len < 0) { - rb_raise(rb_eArgError, "negative string size (or size too big): %" PRIsVALUE, rb_len); + rb_raise(rb_eArgError, "negative string size (or size too big): %" PRIsVALUE, rb_len); } if (len == 0) { - if (NIL_P(rb_buf)) { - return rb_str_new("", 0); - } - return rb_buf; + if (NIL_P(rb_buf)) { + return rb_str_new("", 0); + } + return rb_buf; } if (offset < 0) { - rb_syserr_fail_str(EINVAL, rb_sprintf("pread: Invalid offset argument: %" PRIsVALUE, rb_offset)); + rb_syserr_fail_str(EINVAL, rb_sprintf("pread: Invalid offset argument: %" PRIsVALUE, rb_offset)); } struct StringIO *ptr = readable(self); if (offset >= RSTRING_LEN(ptr->string)) { - rb_eof_error(); + rb_eof_error(); } if (NIL_P(rb_buf)) { - return strio_substr(ptr, offset, len, rb_ascii8bit_encoding()); + return strio_substr(ptr, offset, len, rb_ascii8bit_encoding()); } long rest = RSTRING_LEN(ptr->string) - offset; From 51e6becd391eac03fd3842e1db9b6907999d64ba Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Thu, 28 Mar 2024 02:30:46 +0900 Subject: [PATCH 111/211] [ruby/stringio] Extract `readonly_string_p` https://github.com/ruby/stringio/commit/0da5b725c8 --- ext/stringio/stringio.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ext/stringio/stringio.c b/ext/stringio/stringio.c index c9f6b38ae6a2db..f0e5ee4e85ff09 100644 --- a/ext/stringio/stringio.c +++ b/ext/stringio/stringio.c @@ -59,6 +59,12 @@ rb_str_chilled_p(VALUE str) } #endif +static bool +readonly_string_p(VALUE string) +{ + return OBJ_FROZEN_RAW(string) && !rb_str_chilled_p(string); +} + static struct StringIO * strio_alloc(void) { @@ -303,7 +309,7 @@ strio_init(int argc, VALUE *argv, struct StringIO *ptr, VALUE self) string = rb_enc_str_new("", 0, rb_default_external_encoding()); } - if (!NIL_P(string) && OBJ_FROZEN_RAW(string) && !rb_str_chilled_p(string)) { + if (!NIL_P(string) && readonly_string_p(string)) { if (ptr->flags & FMODE_WRITABLE) { rb_syserr_fail(EACCES, 0); } @@ -497,7 +503,7 @@ strio_set_string(VALUE self, VALUE string) rb_io_taint_check(self); ptr->flags &= ~FMODE_READWRITE; StringValue(string); - ptr->flags = OBJ_FROZEN(string) && !rb_str_chilled_p(string) ? FMODE_READABLE : FMODE_READWRITE; + ptr->flags = readonly_string_p(string) ? FMODE_READABLE : FMODE_READWRITE; ptr->pos = 0; ptr->lineno = 0; RB_OBJ_WRITE(self, &ptr->string, string); From eb995a64108d18fb809f02fa90e085f843ae4309 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 27 Mar 2024 13:37:01 -0400 Subject: [PATCH 112/211] [PRISM] Include file and line in error message --- bootstraptest/test_syntax.rb | 2 +- prism/prism.c | 12 ++++++++++-- prism_compile.c | 20 ++++++++++++++------ spec/prism.mspec | 4 ---- 4 files changed, 25 insertions(+), 13 deletions(-) diff --git a/bootstraptest/test_syntax.rb b/bootstraptest/test_syntax.rb index 18528db7a53396..44bd697d4f9848 100644 --- a/bootstraptest/test_syntax.rb +++ b/bootstraptest/test_syntax.rb @@ -529,7 +529,7 @@ def lines } def assert_syntax_error expected, code, message = '' assert_match /^#{Regexp.escape(expected)}/, - "begin eval(%q{#{code}}, nil, '', 0)"'; rescue SyntaxError => e; e.message[/(?:\A:(?:\d+:)?(?: syntax error,)?|\^) (.*)/, 1] end', message + "begin eval(%q{#{code}}, nil, '', 0)"'; rescue SyntaxError => e; e.message[/(?:\^|\A:(?:\d+:)?(?! syntax errors? found)(?: syntax error,)?) (.*)/, 1] end', message end assert_syntax_error "unterminated string meets end of file", '().."', '[ruby-dev:29732]' assert_equal %q{[]}, %q{$&;[]}, '[ruby-dev:31068]' diff --git a/prism/prism.c b/prism/prism.c index b8787bb9a9e408..3809ce394b3703 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -19686,7 +19686,15 @@ pm_parser_errors_format(const pm_parser_t *parser, const pm_list_t *error_list, // blank lines based on the maximum number of digits in the line numbers // that are going to be displaid. pm_error_format_t error_format; - int32_t max_line_number = errors[error_list->size - 1].line - start_line; + int32_t first_line_number = errors[0].line; + int32_t last_line_number = errors[error_list->size - 1].line; + + // If we have a maximum line number that is negative, then we're going to + // use the absolute value for comparison but multiple by 10 to additionally + // have a column for the negative sign. + if (first_line_number < 0) first_line_number = (-first_line_number) * 10; + if (last_line_number < 0) last_line_number = (-last_line_number) * 10; + int32_t max_line_number = first_line_number > last_line_number ? first_line_number : last_line_number; if (max_line_number < 10) { if (colorize) { @@ -19841,7 +19849,7 @@ pm_parser_errors_format(const pm_parser_t *parser, const pm_list_t *error_list, // Here we determine how many lines of padding to display after the // error, depending on where the next error is in source. last_line = error->line; - int32_t next_line = (index == error_list->size - 1) ? ((int32_t) newline_list->size) : errors[index + 1].line; + int32_t next_line = (index == error_list->size - 1) ? (((int32_t) newline_list->size) + parser->start_line) : errors[index + 1].line; if (next_line - last_line > 1) { pm_buffer_append_string(buffer, " ", 2); diff --git a/prism_compile.c b/prism_compile.c index 0ac931b59bf8e9..1027a2c06202b0 100644 --- a/prism_compile.c +++ b/prism_compile.c @@ -8373,9 +8373,13 @@ pm_parse_process_error_utf8_p(const pm_parser_t *parser, const pm_location_t *lo static VALUE pm_parse_process_error(const pm_parse_result_t *result) { - const pm_diagnostic_t *head = (const pm_diagnostic_t *) result->parser.error_list.head; + const pm_parser_t *parser = &result->parser; + const pm_diagnostic_t *head = (const pm_diagnostic_t *) parser->error_list.head; bool valid_utf8 = true; + pm_buffer_t buffer = { 0 }; + const pm_string_t *filepath = &parser->filepath; + for (const pm_diagnostic_t *error = head; error != NULL; error = (const pm_diagnostic_t *) error->node.next) { // Any errors with the level PM_ERROR_LEVEL_ARGUMENT effectively take // over as the only argument that gets raised. This is to allow priority @@ -8389,20 +8393,24 @@ pm_parse_process_error(const pm_parse_result_t *result) // contain invalid byte sequences. So if any source examples include // invalid UTF-8 byte sequences, we will skip showing source examples // entirely. - if (valid_utf8 && !pm_parse_process_error_utf8_p(&result->parser, &error->location)) { + if (valid_utf8 && !pm_parse_process_error_utf8_p(parser, &error->location)) { valid_utf8 = false; } } - pm_buffer_t buffer = { 0 }; - pm_buffer_append_string(&buffer, "syntax errors found\n", 20); + pm_buffer_append_format( + &buffer, + "%.*s:%" PRIi32 ": syntax error%s found\n", + (int) pm_string_length(filepath), + pm_string_source(filepath), + (int32_t) pm_location_line_number(parser, &head->location), + (parser->error_list.size > 1) ? "s" : "" + ); if (valid_utf8) { pm_parser_errors_format(&result->parser, &result->parser.error_list, &buffer, rb_stderr_tty_p(), true); } else { - const pm_string_t *filepath = &result->parser.filepath; - for (const pm_diagnostic_t *error = head; error != NULL; error = (pm_diagnostic_t *) error->node.next) { if (error != head) pm_buffer_append_byte(&buffer, '\n'); pm_buffer_append_format(&buffer, "%.*s:%" PRIi32 ": %s", (int) pm_string_length(filepath), pm_string_source(filepath), (int32_t) pm_location_line_number(&result->parser, &error->location), error->message); diff --git a/spec/prism.mspec b/spec/prism.mspec index bd427392e2f4e3..252432c94354b9 100644 --- a/spec/prism.mspec +++ b/spec/prism.mspec @@ -33,8 +33,6 @@ MSpec.register(:exclude, "A Symbol literal raises an SyntaxError at parse time w ## Core MSpec.register(:exclude, "IO.popen with a leading Array argument accepts a trailing Hash of Process.exec options") MSpec.register(:exclude, "IO.popen with a leading Array argument accepts an IO mode argument following the Array") -MSpec.register(:exclude, "Kernel#eval includes file and line information in syntax error") -MSpec.register(:exclude, "Kernel#eval evaluates string with given filename and negative linenumber") MSpec.register(:exclude, "Kernel#eval with a magic encoding comment allows spaces before the magic encoding comment") MSpec.register(:exclude, "Kernel#eval with a magic encoding comment allows a shebang line and some spaces before the magic encoding comment") MSpec.register(:exclude, "TracePoint#eval_script is the evald source code") @@ -60,6 +58,4 @@ MSpec.register(:exclude, "Coverage.result does not clear counters when stop: fal MSpec.register(:exclude, "Coverage.result clears counters (sets 0 values) when stop: false and clear: true specified") MSpec.register(:exclude, "Coverage.result does not clear counters when stop: false and clear: false specified") MSpec.register(:exclude, "Coverage.start measures coverage within eval") -MSpec.register(:exclude, "ERB#filename raises an exception if there are errors processing content") -MSpec.register(:exclude, "ERB#filename uses '(erb)' as filename when filename is not set") MSpec.register(:exclude, "Socket.gethostbyaddr using an IPv6 address with an explicit address family raises SocketError when the address is not supported by the family") From ab2ee308aa5f26bd28b6d9de4dc3a24483ff333a Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 27 Mar 2024 13:40:38 -0400 Subject: [PATCH 113/211] [PRISM] Match style for invalid encoding error --- prism_compile.c | 25 ++++++++++++++++++++++++- test/.excludes-prism/TestParse.rb | 1 - test/ruby/test_parse.rb | 10 ++++++++-- 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/prism_compile.c b/prism_compile.c index 1027a2c06202b0..848a5d4ecd2dbf 100644 --- a/prism_compile.c +++ b/prism_compile.c @@ -8385,7 +8385,30 @@ pm_parse_process_error(const pm_parse_result_t *result) // over as the only argument that gets raised. This is to allow priority // messages that should be handled before anything else. if (error->level == PM_ERROR_LEVEL_ARGUMENT) { - return rb_exc_new(rb_eArgError, error->message, strlen(error->message)); + int32_t line_number = (int32_t) pm_location_line_number(parser, &error->location); + + pm_buffer_append_format( + &buffer, + "%.*s:%" PRIi32 ": %s", + (int) pm_string_length(filepath), + pm_string_source(filepath), + line_number, + error->message + ); + + if (pm_parse_process_error_utf8_p(parser, &error->location)) { + pm_buffer_append_byte(&buffer, '\n'); + + pm_list_node_t *list_node = (pm_list_node_t *) error; + pm_list_t error_list = { .size = 1, .head = list_node, .tail = list_node }; + + pm_parser_errors_format(parser, &error_list, &buffer, rb_stderr_tty_p(), false); + } + + VALUE arg_error = rb_exc_new(rb_eArgError, pm_buffer_value(&buffer), pm_buffer_length(&buffer)); + pm_buffer_free(&buffer); + + return arg_error; } // It is implicitly assumed that the error messages will be encodeable diff --git a/test/.excludes-prism/TestParse.rb b/test/.excludes-prism/TestParse.rb index 87694ac15a9851..d832970d766f1a 100644 --- a/test/.excludes-prism/TestParse.rb +++ b/test/.excludes-prism/TestParse.rb @@ -1,5 +1,4 @@ exclude(:test_assign_in_conditional, "missing warning") -exclude(:test_magic_comment, "incorrect encoding") exclude(:test_shareable_constant_value_nested, "ractor support") exclude(:test_shareable_constant_value_nonliteral, "ractor support") exclude(:test_shareable_constant_value_simple, "ractor support") diff --git a/test/ruby/test_parse.rb b/test/ruby/test_parse.rb index 62498170fd5242..e365c39def583b 100644 --- a/test/ruby/test_parse.rb +++ b/test/ruby/test_parse.rb @@ -803,14 +803,20 @@ def test_magic_comment # coding: foo END end - assert_include(e.message, "# coding: foo\n ^~~") + + message = e.message.gsub(/\033\[.*?m/, "") + assert_include(message, "# coding: foo\n") + assert_include(message, " ^") e = assert_raise(ArgumentError) do eval <<-END, nil, __FILE__, __LINE__+1 # coding = foo END end - assert_include(e.message, "# coding = foo\n ^~~") + + message = e.message.gsub(/\033\[.*?m/, "") + assert_include(message, "# coding = foo\n") + assert_include(message, " ^") end def test_utf8_bom From 39606f36e3c82c38cc17c9179d4f6e8f6c51adee Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 27 Mar 2024 13:46:00 -0400 Subject: [PATCH 114/211] [PRISM] Implicitly change encoding when a UTF-8 BOM is found --- prism/prism.c | 5 +++++ prism_compile.c | 4 ++-- test/.excludes-prism/TestParse.rb | 1 - 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/prism/prism.c b/prism/prism.c index 3809ce394b3703..3be4bdce2df55e 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -19251,6 +19251,11 @@ pm_parser_init(pm_parser_t *parser, const uint8_t *source, size_t size, const pm if (size >= 3 && source[0] == 0xef && source[1] == 0xbb && source[2] == 0xbf) { parser->current.end += 3; parser->encoding_comment_start += 3; + + if (parser->encoding != PM_ENCODING_UTF_8_ENTRY) { + parser->encoding = PM_ENCODING_UTF_8_ENTRY; + if (parser->encoding_changed_callback != NULL) parser->encoding_changed_callback(parser); + } } // If the first two bytes of the source are a shebang, then we'll indicate diff --git a/prism_compile.c b/prism_compile.c index 848a5d4ecd2dbf..84a29e59dd7b7a 100644 --- a/prism_compile.c +++ b/prism_compile.c @@ -8431,12 +8431,12 @@ pm_parse_process_error(const pm_parse_result_t *result) ); if (valid_utf8) { - pm_parser_errors_format(&result->parser, &result->parser.error_list, &buffer, rb_stderr_tty_p(), true); + pm_parser_errors_format(parser, &parser->error_list, &buffer, rb_stderr_tty_p(), true); } else { for (const pm_diagnostic_t *error = head; error != NULL; error = (pm_diagnostic_t *) error->node.next) { if (error != head) pm_buffer_append_byte(&buffer, '\n'); - pm_buffer_append_format(&buffer, "%.*s:%" PRIi32 ": %s", (int) pm_string_length(filepath), pm_string_source(filepath), (int32_t) pm_location_line_number(&result->parser, &error->location), error->message); + pm_buffer_append_format(&buffer, "%.*s:%" PRIi32 ": %s", (int) pm_string_length(filepath), pm_string_source(filepath), (int32_t) pm_location_line_number(parser, &error->location), error->message); } } diff --git a/test/.excludes-prism/TestParse.rb b/test/.excludes-prism/TestParse.rb index d832970d766f1a..3451aadacb7a8f 100644 --- a/test/.excludes-prism/TestParse.rb +++ b/test/.excludes-prism/TestParse.rb @@ -5,5 +5,4 @@ exclude(:test_shareable_constant_value_unfrozen, "ractor support") exclude(:test_shareable_constant_value_unshareable_literal, "ractor support") exclude(:test_unused_variable, "missing warning") -exclude(:test_utf8_bom, "incorrect error") exclude(:test_void_expr_stmts_value, "missing warning") From 4361727d28da2e3e58cc1e52c20cf86dd9cd28c9 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 27 Mar 2024 14:20:48 -0400 Subject: [PATCH 115/211] [ruby/prism] Warn on static literal arrays in predicate writes https://github.com/ruby/prism/commit/faadd05693 --- prism/prism.c | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/prism/prism.c b/prism/prism.c index 3be4bdce2df55e..5389cac9f60839 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -812,12 +812,21 @@ pm_parser_warn_conditional_predicate_literal(pm_parser_t *parser, pm_node_t *nod } /** - * Add a warning to the parser if the value that is being written inside of a - * predicate to a conditional is a literal. + * Return true if the value being written within the predicate of a conditional + * is a literal value. */ -static void -pm_conditional_predicate_warn_write_literal(pm_parser_t *parser, pm_node_t *node) { +static bool +pm_conditional_predicate_warn_write_literal_p(const pm_node_t *node) { switch (PM_NODE_TYPE(node)) { + case PM_ARRAY_NODE: { + const pm_array_node_t *cast = (const pm_array_node_t *) node; + for (size_t index = 0; index < cast->elements.size; index++) { + if (!pm_conditional_predicate_warn_write_literal_p(cast->elements.nodes[index])) { + return false; + } + } + return true; + } case PM_FALSE_NODE: case PM_FLOAT_NODE: case PM_IMAGINARY_NODE: @@ -831,10 +840,20 @@ pm_conditional_predicate_warn_write_literal(pm_parser_t *parser, pm_node_t *node case PM_STRING_NODE: case PM_SYMBOL_NODE: case PM_TRUE_NODE: - pm_parser_warn_node(parser, node, parser->version == PM_OPTIONS_VERSION_CRUBY_3_3_0 ? PM_WARN_EQUAL_IN_CONDITIONAL_3_3_0 : PM_WARN_EQUAL_IN_CONDITIONAL); - break; + return true; default: - break; + return false; + } +} + +/** + * Add a warning to the parser if the value that is being written inside of a + * predicate to a conditional is a literal. + */ +static inline void +pm_conditional_predicate_warn_write_literal(pm_parser_t *parser, const pm_node_t *node) { + if (pm_conditional_predicate_warn_write_literal_p(node)) { + pm_parser_warn_node(parser, node, parser->version == PM_OPTIONS_VERSION_CRUBY_3_3_0 ? PM_WARN_EQUAL_IN_CONDITIONAL_3_3_0 : PM_WARN_EQUAL_IN_CONDITIONAL); } } From 9f9c0425c314ea814c60035ddbcd41d3404a0366 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 27 Mar 2024 14:31:11 -0400 Subject: [PATCH 116/211] [PRISM] Turn on passing test --- test/.excludes-prism/TestParse.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/test/.excludes-prism/TestParse.rb b/test/.excludes-prism/TestParse.rb index 3451aadacb7a8f..f2ef81cd119693 100644 --- a/test/.excludes-prism/TestParse.rb +++ b/test/.excludes-prism/TestParse.rb @@ -1,4 +1,3 @@ -exclude(:test_assign_in_conditional, "missing warning") exclude(:test_shareable_constant_value_nested, "ractor support") exclude(:test_shareable_constant_value_nonliteral, "ractor support") exclude(:test_shareable_constant_value_simple, "ractor support") From 9b97f1f3e8c50742c25de46469389bb06ca873a7 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 27 Mar 2024 14:50:40 -0400 Subject: [PATCH 117/211] [ruby/prism] Compare duplicates keys/whens for __FILE__ https://github.com/ruby/prism/commit/85263ade63 --- prism/static_literals.c | 12 ++++++------ test/prism/static_literals_test.rb | 13 ++++++++++--- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/prism/static_literals.c b/prism/static_literals.c index 1708e31a1674de..469bdfd5ea2bd4 100644 --- a/prism/static_literals.c +++ b/prism/static_literals.c @@ -95,13 +95,17 @@ node_hash(const pm_parser_t *parser, const pm_node_t *node) { // Strings hash their value and mix in their flags so that different // encodings are not considered equal. const pm_string_t *value = &((const pm_string_node_t *) node)->unescaped; - return murmur_hash(pm_string_source(value), pm_string_length(value) * sizeof(uint8_t)) ^ murmur_scramble((uint32_t) node->flags); + + pm_node_flags_t flags = node->flags; + flags &= (PM_STRING_FLAGS_FORCED_BINARY_ENCODING | PM_STRING_FLAGS_FORCED_UTF8_ENCODING); + + return murmur_hash(pm_string_source(value), pm_string_length(value) * sizeof(uint8_t)) ^ murmur_scramble((uint32_t) flags); } case PM_SOURCE_FILE_NODE: { // Source files hash their value and mix in their flags so that // different encodings are not considered equal. const pm_string_t *value = &((const pm_source_file_node_t *) node)->filepath; - return murmur_hash(pm_string_source(value), pm_string_length(value) * sizeof(uint8_t)) ^ murmur_scramble((uint32_t) node->flags); + return murmur_hash(pm_string_source(value), pm_string_length(value) * sizeof(uint8_t)); } case PM_REGULAR_EXPRESSION_NODE: { // Regular expressions hash their value and mix in their flags so @@ -316,8 +320,6 @@ pm_compare_regular_expression_nodes(PRISM_ATTRIBUTE_UNUSED const pm_parser_t *pa */ pm_node_t * pm_static_literals_add(const pm_parser_t *parser, pm_static_literals_t *literals, pm_node_t *node) { - if (!PM_NODE_FLAG_P(node, PM_NODE_FLAG_STATIC_LITERAL)) return NULL; - switch (PM_NODE_TYPE(node)) { case PM_INTEGER_NODE: case PM_SOURCE_LINE_NODE: @@ -435,8 +437,6 @@ pm_rational_inspect(pm_buffer_t *buffer, pm_rational_node_t *node) { */ PRISM_EXPORTED_FUNCTION void pm_static_literal_inspect(pm_buffer_t *buffer, const pm_parser_t *parser, const pm_node_t *node) { - assert(PM_NODE_FLAG_P(node, PM_NODE_FLAG_STATIC_LITERAL)); - switch (PM_NODE_TYPE(node)) { case PM_FALSE_NODE: pm_buffer_append_string(buffer, "false", 5); diff --git a/test/prism/static_literals_test.rb b/test/prism/static_literals_test.rb index 26699319741999..31c802bf90d188 100644 --- a/test/prism/static_literals_test.rb +++ b/test/prism/static_literals_test.rb @@ -32,6 +32,7 @@ def test_static_literals assert_warning("1ri", "1ri", "(0+(1/1)*i)") assert_warning("1.0ri", "1.0ri", "(0+(1/1)*i)") + assert_warning("__FILE__", "\"#{__FILE__}\"", __FILE__) assert_warning("\"#{__FILE__}\"") assert_warning("\"foo\"") @@ -50,17 +51,23 @@ def test_static_literals private + class NullWarning + def message + "" + end + end + def parse_warnings(left, right) warnings = [] - warnings << Prism.parse(<<~RUBY, filepath: __FILE__).warnings.first + warnings << (Prism.parse(<<~RUBY, filepath: __FILE__).warnings.first || NullWarning.new) { #{left} => 1, #{right} => 2 } RUBY - warnings << Prism.parse(<<~RUBY, filepath: __FILE__).warnings.first + warnings << (Prism.parse(<<~RUBY, filepath: __FILE__).warnings.first || NullWarning.new) case foo when #{left} when #{right} @@ -79,7 +86,7 @@ def assert_warning(left, right = left, message = left) end def refute_warning(left, right) - assert_empty parse_warnings(left, right).compact + assert_empty parse_warnings(left, right).grep_v(NullWarning) end end end From 010286c767df393740872c8331408b42875ba64a Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 27 Mar 2024 14:59:03 -0400 Subject: [PATCH 118/211] [PRISM] Enable passing test for hash duplicated keys --- test/.excludes-prism/TestRubyLiteral.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/test/.excludes-prism/TestRubyLiteral.rb b/test/.excludes-prism/TestRubyLiteral.rb index 4cfe25215e1190..a5ad6d6d9d0bf7 100644 --- a/test/.excludes-prism/TestRubyLiteral.rb +++ b/test/.excludes-prism/TestRubyLiteral.rb @@ -1,3 +1,2 @@ exclude(:test_dregexp, "unknown") -exclude(:test_hash_duplicated_key, "parser implementation dependent") exclude(:test_string, "https://github.com/ruby/prism/issues/2331") From 7e12b03c5a179c1c738fec5ac1ad06dfdc879b1b Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 27 Mar 2024 15:04:27 -0400 Subject: [PATCH 119/211] [PRISM] Set path on syntax error --- prism_compile.c | 36 ++++++++++++++++++++------- prism_compile.h | 16 ++++++++++++ test/.excludes-prism/TestEval.rb | 1 - test/.excludes-prism/TestException.rb | 1 - 4 files changed, 43 insertions(+), 11 deletions(-) delete mode 100644 test/.excludes-prism/TestEval.rb delete mode 100644 test/.excludes-prism/TestException.rb diff --git a/prism_compile.c b/prism_compile.c index 84a29e59dd7b7a..6e27d017295d30 100644 --- a/prism_compile.c +++ b/prism_compile.c @@ -528,6 +528,21 @@ pm_compile_regexp_dynamic(rb_iseq_t *iseq, const pm_node_t *node, const pm_node_ PUSH_INSN2(ret, *node_location, toregexp, INT2FIX(parse_regexp_flags(node) & 0xFF), INT2FIX(length)); } +static VALUE +pm_source_file_value(const pm_source_file_node_t *node, const pm_scope_node_t *scope_node) +{ + const pm_string_t *filepath = &node->filepath; + size_t length = pm_string_length(filepath); + + if (length > 0) { + rb_encoding *filepath_encoding = scope_node->filepath_encoding != NULL ? scope_node->filepath_encoding : rb_utf8_encoding(); + return rb_fstring(rb_enc_str_new((const char *) pm_string_source(filepath), length, filepath_encoding)); + } + else { + return rb_fstring_lit(""); + } +} + /** * Certain nodes can be compiled literally. This function returns the literal * value described by the given node. For example, an array node with all static @@ -612,14 +627,7 @@ pm_static_literal_value(rb_iseq_t *iseq, const pm_node_t *node, const pm_scope_n return rb_enc_from_encoding(scope_node->encoding); case PM_SOURCE_FILE_NODE: { const pm_source_file_node_t *cast = (const pm_source_file_node_t *) node; - size_t length = pm_string_length(&cast->filepath); - - if (length > 0) { - return rb_enc_str_new((const char *) pm_string_source(&cast->filepath), length, scope_node->encoding); - } - else { - return rb_fstring_lit(""); - } + return pm_source_file_value(cast, scope_node); } case PM_SOURCE_LINE_NODE: return INT2FIX(pm_node_line_number(scope_node->parser, node)); @@ -2669,6 +2677,7 @@ pm_scope_node_init(const pm_node_t *node, pm_scope_node_t *scope, pm_scope_node_ if (previous) { scope->parser = previous->parser; scope->encoding = previous->encoding; + scope->filepath_encoding = previous->filepath_encoding; scope->constants = previous->constants; } @@ -8074,7 +8083,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, // ^^^^^^^^ if (!popped) { const pm_source_file_node_t *cast = (const pm_source_file_node_t *) node; - VALUE string = rb_fstring(parse_string(scope_node, &cast->filepath)); + VALUE string = pm_source_file_value(cast, scope_node); if (PM_NODE_FLAG_P(cast, PM_STRING_FLAGS_FROZEN)) { PUSH_INSN1(ret, location, putobject, string); @@ -8441,6 +8450,11 @@ pm_parse_process_error(const pm_parse_result_t *result) } VALUE error = rb_exc_new(rb_eSyntaxError, pm_buffer_value(&buffer), pm_buffer_length(&buffer)); + + rb_encoding *filepath_encoding = result->node.filepath_encoding != NULL ? result->node.filepath_encoding : rb_utf8_encoding(); + VALUE path = rb_enc_str_new((const char *) pm_string_source(filepath), pm_string_length(filepath), filepath_encoding); + + rb_ivar_set(error, rb_intern_const("@path"), path); pm_buffer_free(&buffer); return error; @@ -8459,7 +8473,10 @@ pm_parse_process(pm_parse_result_t *result, pm_node_t *node) // First, set up the scope node so that the AST node is attached and can be // freed regardless of whether or we return an error. pm_scope_node_t *scope_node = &result->node; + rb_encoding *filepath_encoding = scope_node->filepath_encoding; + pm_scope_node_init(node, scope_node, NULL); + scope_node->filepath_encoding = filepath_encoding; // If there are errors, raise an appropriate error and free the result. if (parser->error_list.size > 0) { @@ -8658,6 +8675,7 @@ pm_parse_string(pm_parse_result_t *result, VALUE source, VALUE filepath) pm_string_constant_init(&result->input, RSTRING_PTR(source), RSTRING_LEN(source)); pm_options_encoding_set(&result->options, rb_enc_name(encoding)); + result->node.filepath_encoding = rb_enc_get(filepath); pm_options_filepath_set(&result->options, RSTRING_PTR(filepath)); RB_GC_GUARD(filepath); diff --git a/prism_compile.h b/prism_compile.h index 427fa54b516013..32f8c128442820 100644 --- a/prism_compile.h +++ b/prism_compile.h @@ -26,6 +26,13 @@ typedef struct pm_scope_node { const pm_parser_t *parser; rb_encoding *encoding; + /** + * This is the encoding of the actual filepath object that will be used when + * a __FILE__ node is compiled or when the path has to be set on a syntax + * error. + */ + rb_encoding *filepath_encoding; + // The size of the local table // on the iseq which includes // locals and hidden variables @@ -40,10 +47,19 @@ void pm_scope_node_destroy(pm_scope_node_t *scope_node); bool *rb_ruby_prism_ptr(void); typedef struct { + /** The parser that will do the actual parsing. */ pm_parser_t parser; + + /** The options that will be passed to the parser. */ pm_options_t options; + + /** The input that represents the source to be parsed. */ pm_string_t input; + + /** The resulting scope node that will hold the generated AST. */ pm_scope_node_t node; + + /** Whether or not this parse result has performed its parsing yet. */ bool parsed; } pm_parse_result_t; diff --git a/test/.excludes-prism/TestEval.rb b/test/.excludes-prism/TestEval.rb deleted file mode 100644 index 6cc6bdfb1d65e6..00000000000000 --- a/test/.excludes-prism/TestEval.rb +++ /dev/null @@ -1 +0,0 @@ -exclude(:test_file_encoding, "incorrect encoding") diff --git a/test/.excludes-prism/TestException.rb b/test/.excludes-prism/TestException.rb deleted file mode 100644 index 3f6d0e3b812f7e..00000000000000 --- a/test/.excludes-prism/TestException.rb +++ /dev/null @@ -1 +0,0 @@ -exclude(:test_syntax_error_path, "unknown") From 8c7b9bd0eb15356ac47407f7692869ad147c7954 Mon Sep 17 00:00:00 2001 From: KJ Tsanaktsidis Date: Sat, 17 Feb 2024 17:50:16 +1100 Subject: [PATCH 120/211] Disable ASAN handle_segv in test_rubyoptions.rb ASAN registers a sigsegv handler and causes extra output to be emitted that these tests are not expecting. --- test/ruby/test_rubyoptions.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/ruby/test_rubyoptions.rb b/test/ruby/test_rubyoptions.rb index 446548d6d9a395..8396066dc1ba19 100644 --- a/test/ruby/test_rubyoptions.rb +++ b/test/ruby/test_rubyoptions.rb @@ -865,6 +865,9 @@ def assert_segv(args, message=nil, list: SEGVTest::ExpectedStderrList, **opt, &b env = Hash === args.first ? args.shift : {} args.unshift("--yjit") if self.class.yjit_enabled? env.update({'RUBY_ON_BUG' => nil}) + # ASAN registers a segv handler which prints out "AddressSanitizer: DEADLYSIGNAL" when + # catching sigsegv; we don't expect that output, so suppress it. + env.update({'ASAN_OPTIONS' => 'handle_segv=0'}) args.unshift(env) test_stdin = "" From 7bdd742c0241435001efe641332b2088536bd074 Mon Sep 17 00:00:00 2001 From: KJ Tsanaktsidis Date: Tue, 12 Mar 2024 18:24:56 +1100 Subject: [PATCH 121/211] Set ASAN_OPTIONS=disable_coredump=0 for test_execopts_rlimit test By default, ASAN sets RLIMIT_CORE to zero, "to avoid dumping a 16T+ core file" on 64 bit systems. These tests are just asserting on the expected value of RLIMIT_CORE, not actually dumping core files, so it's fine to disable that behaviour of ASAN for this test. --- test/ruby/test_process.rb | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/test/ruby/test_process.rb b/test/ruby/test_process.rb index 59140ba664541c..7ef184d63964ab 100644 --- a/test/ruby/test_process.rb +++ b/test/ruby/test_process.rb @@ -199,58 +199,67 @@ def test_execopts_rlimit max = Process.getrlimit(:CORE).last + # When running under ASAN, we need to set disable_coredump=0 for this test; by default + # the ASAN runtime library sets RLIMIT_CORE to 0, "to avoid dumping a 16T+ core file", and + # that inteferes with this test. + asan_options = ENV['ASAN_OPTIONS'] || '' + asan_options << ':' unless asan_options.empty? + env = { + 'ASAN_OPTIONS' => "#{asan_options}disable_coredump=0" + } + n = max - IO.popen([RUBY, "-e", + IO.popen([env, RUBY, "-e", "puts Process.getrlimit(:CORE)", :rlimit_core=>n]) {|io| assert_equal("#{n}\n#{n}\n", io.read) } n = 0 - IO.popen([RUBY, "-e", + IO.popen([env, RUBY, "-e", "puts Process.getrlimit(:CORE)", :rlimit_core=>n]) {|io| assert_equal("#{n}\n#{n}\n", io.read) } n = max - IO.popen([RUBY, "-e", + IO.popen([env, RUBY, "-e", "puts Process.getrlimit(:CORE)", :rlimit_core=>[n]]) {|io| assert_equal("#{n}\n#{n}\n", io.read) } m, n = 0, max - IO.popen([RUBY, "-e", + IO.popen([env, RUBY, "-e", "puts Process.getrlimit(:CORE)", :rlimit_core=>[m,n]]) {|io| assert_equal("#{m}\n#{n}\n", io.read) } m, n = 0, 0 - IO.popen([RUBY, "-e", + IO.popen([env, RUBY, "-e", "puts Process.getrlimit(:CORE)", :rlimit_core=>[m,n]]) {|io| assert_equal("#{m}\n#{n}\n", io.read) } n = max - IO.popen([RUBY, "-e", + IO.popen([env, RUBY, "-e", "puts Process.getrlimit(:CORE), Process.getrlimit(:CPU)", :rlimit_core=>n, :rlimit_cpu=>3600]) {|io| assert_equal("#{n}\n#{n}\n""3600\n3600\n", io.read) } assert_raise(ArgumentError) do - system(RUBY, '-e', 'exit', 'rlimit_bogus'.to_sym => 123) + system(env, RUBY, '-e', 'exit', 'rlimit_bogus'.to_sym => 123) end - assert_separately([],"#{<<~"begin;"}\n#{<<~'end;'}", 'rlimit_cpu'.to_sym => 3600) + assert_separately([env],"#{<<~"begin;"}\n#{<<~'end;'}", 'rlimit_cpu'.to_sym => 3600) BUG = "[ruby-core:82033] [Bug #13744]" begin; assert_equal([3600,3600], Process.getrlimit(:CPU), BUG) end; assert_raise_with_message(ArgumentError, /bogus/) do - system(RUBY, '-e', 'exit', :rlimit_bogus => 123) + system(env, RUBY, '-e', 'exit', :rlimit_bogus => 123) end assert_raise_with_message(ArgumentError, /rlimit_cpu/) { - system(RUBY, '-e', 'exit', "rlimit_cpu\0".to_sym => 3600) + system(env, RUBY, '-e', 'exit', "rlimit_cpu\0".to_sym => 3600) } end From 75234beb2456ce0a1f059e06bc5125cd18b683ab Mon Sep 17 00:00:00 2001 From: KJ Tsanaktsidis Date: Tue, 12 Mar 2024 20:55:44 +1100 Subject: [PATCH 122/211] Make TestParallel#test_retry_workers consider RUBY_TEST_TIMEOUT_SCALE This test currently fails if RUBY_TEST_TIMEOUT_SCALE is set, because the worker timeout is scaled out but the duration of the sleep does not; thus, the test-test-case does not timeout when it should. --- tool/test/testunit/tests_for_parallel/slow_helper.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tool/test/testunit/tests_for_parallel/slow_helper.rb b/tool/test/testunit/tests_for_parallel/slow_helper.rb index d8372730a8f50b..38067c1f470fb5 100644 --- a/tool/test/testunit/tests_for_parallel/slow_helper.rb +++ b/tool/test/testunit/tests_for_parallel/slow_helper.rb @@ -2,6 +2,7 @@ module TestSlowTimeout def test_slow - sleep (ENV['sec'] || 3).to_i if on_parallel_worker? + sleep_for = EnvUtil.apply_timeout_scale((ENV['sec'] || 3).to_i) + sleep sleep_for if on_parallel_worker? end end From dc9d2455b6bddb2bea21fe983de7be6b78924c1b Mon Sep 17 00:00:00 2001 From: KJ Tsanaktsidis Date: Tue, 12 Mar 2024 18:24:07 +1100 Subject: [PATCH 123/211] Add a missing asan_unpoisoning_p in gc_set_candidate_object_i It walks the heap, and checks for T_NONE and T_ZOMBIE objects, so it needs to unpoison these slots before accessing them when ASAN is enabled. --- gc.c | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/gc.c b/gc.c index 099a1f382b23fc..a5023d62d82d5b 100644 --- a/gc.c +++ b/gc.c @@ -9362,18 +9362,20 @@ gc_set_candidate_object_i(void *vstart, void *vend, size_t stride, void *data) rb_objspace_t *objspace = &rb_objspace; VALUE v = (VALUE)vstart; for (; v != (VALUE)vend; v += stride) { - switch (BUILTIN_TYPE(v)) { - case T_NONE: - case T_ZOMBIE: - break; - case T_STRING: - // precompute the string coderange. This both save time for when it will be - // eventually needed, and avoid mutating heap pages after a potential fork. - rb_enc_str_coderange(v); - // fall through - default: - if (!RVALUE_OLD_P(v) && !RVALUE_WB_UNPROTECTED(v)) { - RVALUE_AGE_SET_CANDIDATE(objspace, v); + asan_unpoisoning_object(v) { + switch (BUILTIN_TYPE(v)) { + case T_NONE: + case T_ZOMBIE: + break; + case T_STRING: + // precompute the string coderange. This both save time for when it will be + // eventually needed, and avoid mutating heap pages after a potential fork. + rb_enc_str_coderange(v); + // fall through + default: + if (!RVALUE_OLD_P(v) && !RVALUE_WB_UNPROTECTED(v)) { + RVALUE_AGE_SET_CANDIDATE(objspace, v); + } } } } From 84236132760fbc09c511b17fd1a49c6320f95b74 Mon Sep 17 00:00:00 2001 From: Naoto Ono Date: Tue, 26 Mar 2024 12:47:29 +0900 Subject: [PATCH 124/211] Launchable: Configure OS correctly in macos.yaml --- .github/workflows/macos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 4bb41de22e799f..c89635ea822430 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -88,7 +88,7 @@ jobs: - name: Set up Launchable uses: ./.github/actions/launchable/setup with: - os: ${{ matrix.os }} + os: ${{ matrix.os || (github.repository == 'ruby/ruby' && 'macos-arm-oss' || 'macos-14')}} test-opts: ${{ matrix.test_opts }} launchable-token: ${{ secrets.LAUNCHABLE_TOKEN }} builddir: build From 7293cef0a84026ff53c37926839b5bc0a154bf57 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Thu, 28 Mar 2024 10:55:19 +0900 Subject: [PATCH 125/211] [DOC] molinillo has been moved --- LEGAL | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LEGAL b/LEGAL index e352c55ee50005..e8b16319739e74 100644 --- a/LEGAL +++ b/LEGAL @@ -952,7 +952,7 @@ mentioned below. {MIT License}[rdoc-label:label-MIT+License] -[lib/rubygems/resolver/molinillo] +[lib/rubygems/vendor/molinillo] molinillo is under the following license. From 67bdb7aabaef106ee377dd7757365db1ce46d531 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Thu, 28 Mar 2024 11:18:31 +0900 Subject: [PATCH 126/211] [DOC] Use `rdoc-ref:@` shorthands for `rdoc-label:` tags --- LEGAL | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/LEGAL b/LEGAL index e8b16319739e74..c931291c8ad9b6 100644 --- a/LEGAL +++ b/LEGAL @@ -58,12 +58,12 @@ mentioned below. [ccan/list/list.h] - This file is licensed under the {MIT License}[rdoc-label:label-MIT+License]. + This file is licensed under the {MIT License}[rdoc-ref:@MIT+License]. [coroutine] Unless otherwise specified, these files are licensed under the - {MIT License}[rdoc-label:label-MIT+License]. + {MIT License}[rdoc-ref:@MIT+License]. [include/ruby/onigmo.h] [include/ruby/oniguruma.h] @@ -546,7 +546,7 @@ mentioned below. [vsnprintf.c] - This file is under the {old-style BSD license}[rdoc-label:label-Old-style+BSD+license]. + This file is under the {old-style BSD license}[rdoc-ref:@Old-style+BSD+license]. >>> Copyright (c) 1990, 1993:: @@ -577,7 +577,7 @@ mentioned below. [missing/crypt.c] - This file is under the {old-style BSD license}[rdoc-label:label-Old-style+BSD+license]. + This file is under the {old-style BSD license}[rdoc-ref:@Old-style+BSD+license]. >>> Copyright (c) 1989, 1993:: @@ -588,7 +588,7 @@ mentioned below. [missing/setproctitle.c] - This file is under the {old-style BSD license}[rdoc-label:label-Old-style+BSD+license]. + This file is under the {old-style BSD license}[rdoc-ref:@Old-style+BSD+license]. >>> Copyright 2003:: Damien Miller @@ -879,7 +879,7 @@ mentioned below. >>> RubyGems is copyrighted free software by Chad Fowler, Rich Kilmer, Jim Weirich and others. You can redistribute it and/or modify it under - either the terms of the {MIT license}[rdoc-label:label-MIT+License], or the conditions + either the terms of the {MIT license}[rdoc-ref:@MIT+License], or the conditions below: 1. You may make and give away verbatim copies of the source form of the @@ -941,7 +941,7 @@ mentioned below. Portions copyright (c) 2010:: Andre Arko Portions copyright (c) 2009:: Engine Yard - {MIT License}[rdoc-label:label-MIT+License] + {MIT License}[rdoc-ref:@MIT+License] [lib/bundler/vendor/thor] @@ -950,7 +950,7 @@ mentioned below. >>> Copyright (c) 2008 Yehuda Katz, Eric Hodel, et al. - {MIT License}[rdoc-label:label-MIT+License] + {MIT License}[rdoc-ref:@MIT+License] [lib/rubygems/vendor/molinillo] @@ -959,7 +959,7 @@ mentioned below. >>> Copyright (c) 2014 Samuel E. Giddins segiddins@segiddins.me - {MIT License}[rdoc-label:label-MIT+License] + {MIT License}[rdoc-ref:@MIT+License] [lib/bundler/vendor/pub_grub] @@ -968,7 +968,7 @@ mentioned below. >>> Copyright (c) 2018 John Hawthorn - {MIT License}[rdoc-label:label-MIT+License] + {MIT License}[rdoc-ref:@MIT+License] [lib/bundler/vendor/connection_pool] @@ -977,7 +977,7 @@ mentioned below. >>> Copyright (c) 2011 Mike Perham - {MIT License}[rdoc-label:label-MIT+License] + {MIT License}[rdoc-ref:@MIT+License] [lib/bundler/vendor/net-http-persistent] @@ -986,7 +986,7 @@ mentioned below. >>> Copyright (c) Eric Hodel, Aaron Patterson - {MIT License}[rdoc-label:label-MIT+License] + {MIT License}[rdoc-ref:@MIT+License] [lib/did_you_mean] [lib/did_you_mean.rb] @@ -997,7 +997,7 @@ mentioned below. >>> Copyright (c) 2014-2016 Yuki Nishijima - {MIT License}[rdoc-label:label-MIT+License] + {MIT License}[rdoc-ref:@MIT+License] [lib/error_highlight] [lib/error_highlight.rb] @@ -1008,7 +1008,7 @@ mentioned below. >>> Copyright (c) 2021 Yusuke Endoh - {MIT License}[rdoc-label:label-MIT+License] + {MIT License}[rdoc-ref:@MIT+License] [benchmark/so_ackermann.rb] [benchmark/so_array.rb] From 7630a89a4bf352e1310b5323e3e2ee976eecddca Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Thu, 28 Mar 2024 09:45:01 +0900 Subject: [PATCH 127/211] Use www.rfc-editor.org for RFC text. We use the following site for that now: * https://tools.ietf.org/ or http * https://datatracker.ietf.org or http Today, IETF said the official site of RFC is www.rfc-editor.org. FYI: https://authors.ietf.org/en/references-in-rfcxml I replaced them to www.rfc-editor.org. --- doc/ChangeLog/ChangeLog-2.1.0 | 2 +- doc/ChangeLog/ChangeLog-2.2.0 | 2 +- doc/ChangeLog/ChangeLog-2.4.0 | 2 +- doc/csv/recipes/generating.rdoc | 2 +- doc/csv/recipes/parsing.rdoc | 2 +- doc/strftime_formatting.rdoc | 6 +++--- enc/ebcdic.h | 2 +- ext/openssl/ossl_kdf.c | 10 +++++----- ext/openssl/ossl_ns_spki.c | 4 ++-- lib/random/formatter.rb | 2 +- lib/uri.rb | 18 +++++++++--------- lib/uri/generic.rb | 2 +- lib/uri/http.rb | 4 ++-- spec/bundler/bundler/digest_spec.rb | 2 +- test/openssl/test_ssl.rb | 2 +- tool/lib/webrick/httprequest.rb | 2 +- 16 files changed, 32 insertions(+), 32 deletions(-) diff --git a/doc/ChangeLog/ChangeLog-2.1.0 b/doc/ChangeLog/ChangeLog-2.1.0 index 5b670b31c97c8c..7964a682ebd9c8 100644 --- a/doc/ChangeLog/ChangeLog-2.1.0 +++ b/doc/ChangeLog/ChangeLog-2.1.0 @@ -659,7 +659,7 @@ Mon Dec 9 02:10:32 2013 NARUSE, Yui * lib/net/http/responses.rb: Add `HTTPIMUsed`, as it is also supported by rack/rails. - RFC - http://tools.ietf.org/html/rfc3229 + RFC - https://www.rfc-editor.org/rfc/rfc3229 by Vipul A M https://github.com/ruby/ruby/pull/447 fix GH-447 diff --git a/doc/ChangeLog/ChangeLog-2.2.0 b/doc/ChangeLog/ChangeLog-2.2.0 index 5a7dbf826d3135..0edcf0122b190d 100644 --- a/doc/ChangeLog/ChangeLog-2.2.0 +++ b/doc/ChangeLog/ChangeLog-2.2.0 @@ -5648,7 +5648,7 @@ Wed Aug 6 04:16:05 2014 NARUSE, Yui * lib/net/http/requests.rb (Net::HTTP::Options::RESPONSE_HAS_BODY): OPTIONS requests may have response bodies. [Feature #8429] - http://tools.ietf.org/html/rfc7231#section-4.3.7 + https://www.rfc-editor.org/rfc/rfc7231#section-4.3.7 Wed Aug 6 03:18:04 2014 NARUSE, Yui diff --git a/doc/ChangeLog/ChangeLog-2.4.0 b/doc/ChangeLog/ChangeLog-2.4.0 index a297a579d1d4ba..30e9ccab3d9aac 100644 --- a/doc/ChangeLog/ChangeLog-2.4.0 +++ b/doc/ChangeLog/ChangeLog-2.4.0 @@ -7356,7 +7356,7 @@ Thu Mar 17 11:51:48 2016 NARUSE, Yui Note: CryptGenRandom function is PRNG and doesn't check its entropy, so it won't block. [Bug #12139] https://msdn.microsoft.com/ja-jp/library/windows/desktop/aa379942.aspx - https://tools.ietf.org/html/rfc4086#section-7.1.3 + https://www.rfc-editor.org/rfc/rfc4086#section-7.1.3 https://eprint.iacr.org/2007/419.pdf http://www.cs.huji.ac.il/~dolev/pubs/thesis/msc-thesis-leo.pdf diff --git a/doc/csv/recipes/generating.rdoc b/doc/csv/recipes/generating.rdoc index a6bd88a7147a1e..e61838d31a5b3c 100644 --- a/doc/csv/recipes/generating.rdoc +++ b/doc/csv/recipes/generating.rdoc @@ -163,7 +163,7 @@ This example defines and uses two custom write converters to strip and upcase ge === RFC 4180 Compliance By default, \CSV generates data that is compliant with -{RFC 4180}[https://tools.ietf.org/html/rfc4180] +{RFC 4180}[https://www.rfc-editor.org/rfc/rfc4180] with respect to: - Column separator. - Quote character. diff --git a/doc/csv/recipes/parsing.rdoc b/doc/csv/recipes/parsing.rdoc index f3528fbdf1bc8b..1b7071e33f6543 100644 --- a/doc/csv/recipes/parsing.rdoc +++ b/doc/csv/recipes/parsing.rdoc @@ -191,7 +191,7 @@ Output: === RFC 4180 Compliance By default, \CSV parses data that is compliant with -{RFC 4180}[https://tools.ietf.org/html/rfc4180] +{RFC 4180}[https://www.rfc-editor.org/rfc/rfc4180] with respect to: - Row separator. - Column separator. diff --git a/doc/strftime_formatting.rdoc b/doc/strftime_formatting.rdoc index 7694752a21c1ec..5c7b33155df9ec 100644 --- a/doc/strftime_formatting.rdoc +++ b/doc/strftime_formatting.rdoc @@ -356,7 +356,7 @@ each based on an external standard. == HTTP Format The HTTP date format is based on -{RFC 2616}[https://datatracker.ietf.org/doc/html/rfc2616], +{RFC 2616}[https://www.rfc-editor.org/rfc/rfc2616], and treats dates in the format '%a, %d %b %Y %T GMT': d = Date.new(2001, 2, 3) # => # @@ -371,7 +371,7 @@ and treats dates in the format '%a, %d %b %Y %T GMT': == RFC 3339 Format The RFC 3339 date format is based on -{RFC 3339}[https://datatracker.ietf.org/doc/html/rfc3339]: +{RFC 3339}[https://www.rfc-editor.org/rfc/rfc3339]: d = Date.new(2001, 2, 3) # => # # Return 3339-formatted string. @@ -385,7 +385,7 @@ The RFC 3339 date format is based on == RFC 2822 Format The RFC 2822 date format is based on -{RFC 2822}[https://datatracker.ietf.org/doc/html/rfc2822], +{RFC 2822}[https://www.rfc-editor.org/rfc/rfc2822], and treats dates in the format '%a, %-d %b %Y %T %z']: d = Date.new(2001, 2, 3) # => # diff --git a/enc/ebcdic.h b/enc/ebcdic.h index a3b380a32760ad..5109bf7065abb1 100644 --- a/enc/ebcdic.h +++ b/enc/ebcdic.h @@ -7,5 +7,5 @@ ENC_ALIAS("ebcdic-cp-us", "IBM037"); * hopefully the most widely used one. * * See http://www.iana.org/assignments/character-sets/character-sets.xhtml - * http://tools.ietf.org/html/rfc1345 + * https://www.rfc-editor.org/rfc/rfc1345 */ diff --git a/ext/openssl/ossl_kdf.c b/ext/openssl/ossl_kdf.c index 48b161d4f47749..ba197a659ec777 100644 --- a/ext/openssl/ossl_kdf.c +++ b/ext/openssl/ossl_kdf.c @@ -18,7 +18,7 @@ static VALUE mKDF, eKDF; * of _length_ bytes. * * For more information about PBKDF2, see RFC 2898 Section 5.2 - * (https://tools.ietf.org/html/rfc2898#section-5.2). + * (https://www.rfc-editor.org/rfc/rfc2898#section-5.2). * * === Parameters * pass :: The password. @@ -81,10 +81,10 @@ kdf_pbkdf2_hmac(int argc, VALUE *argv, VALUE self) * bcrypt. * * The keyword arguments _N_, _r_ and _p_ can be used to tune scrypt. RFC 7914 - * (published on 2016-08, https://tools.ietf.org/html/rfc7914#section-2) states + * (published on 2016-08, https://www.rfc-editor.org/rfc/rfc7914#section-2) states * that using values r=8 and p=1 appears to yield good results. * - * See RFC 7914 (https://tools.ietf.org/html/rfc7914) for more information. + * See RFC 7914 (https://www.rfc-editor.org/rfc/rfc7914) for more information. * * === Parameters * pass :: Passphrase. @@ -147,7 +147,7 @@ kdf_scrypt(int argc, VALUE *argv, VALUE self) * KDF.hkdf(ikm, salt:, info:, length:, hash:) -> String * * HMAC-based Extract-and-Expand Key Derivation Function (HKDF) as specified in - * {RFC 5869}[https://tools.ietf.org/html/rfc5869]. + * {RFC 5869}[https://www.rfc-editor.org/rfc/rfc5869]. * * New in OpenSSL 1.1.0. * @@ -165,7 +165,7 @@ kdf_scrypt(int argc, VALUE *argv, VALUE self) * The hash function. * * === Example - * # The values from https://datatracker.ietf.org/doc/html/rfc5869#appendix-A.1 + * # The values from https://www.rfc-editor.org/rfc/rfc5869#appendix-A.1 * ikm = ["0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b"].pack("H*") * salt = ["000102030405060708090a0b0c"].pack("H*") * info = ["f0f1f2f3f4f5f6f7f8f9"].pack("H*") diff --git a/ext/openssl/ossl_ns_spki.c b/ext/openssl/ossl_ns_spki.c index 9bed1f330ec5e2..9d70b5d87a5d59 100644 --- a/ext/openssl/ossl_ns_spki.c +++ b/ext/openssl/ossl_ns_spki.c @@ -365,8 +365,8 @@ ossl_spki_verify(VALUE self, VALUE key) * * OpenSSL::Netscape is a namespace for SPKI (Simple Public Key * Infrastructure) which implements Signed Public Key and Challenge. - * See {RFC 2692}[http://tools.ietf.org/html/rfc2692] and {RFC - * 2693}[http://tools.ietf.org/html/rfc2692] for details. + * See {RFC 2692}[https://www.rfc-editor.org/rfc/rfc2692] and {RFC + * 2693}[https://www.rfc-editor.org/rfc/rfc2692] for details. */ /* Document-class: OpenSSL::Netscape::SPKIError diff --git a/lib/random/formatter.rb b/lib/random/formatter.rb index 0548e86ccaafc6..037f9d8748f27b 100644 --- a/lib/random/formatter.rb +++ b/lib/random/formatter.rb @@ -165,7 +165,7 @@ def urlsafe_base64(n=nil, padding=false) # # The result contains 122 random bits (15.25 random bytes). # - # See RFC4122[https://datatracker.ietf.org/doc/html/rfc4122] for details of UUID. + # See RFC4122[https://www.rfc-editor.org/rfc/rfc4122] for details of UUID. # def uuid ary = random_bytes(16) diff --git a/lib/uri.rb b/lib/uri.rb index cd8083b65aa071..dfdb052a79057c 100644 --- a/lib/uri.rb +++ b/lib/uri.rb @@ -1,6 +1,6 @@ # frozen_string_literal: false # URI is a module providing classes to handle Uniform Resource Identifiers -# (RFC2396[https://datatracker.ietf.org/doc/html/rfc2396]). +# (RFC2396[https://www.rfc-editor.org/rfc/rfc2396]). # # == Features # @@ -47,14 +47,14 @@ # A good place to view an RFC spec is http://www.ietf.org/rfc.html. # # Here is a list of all related RFC's: -# - RFC822[https://datatracker.ietf.org/doc/html/rfc822] -# - RFC1738[https://datatracker.ietf.org/doc/html/rfc1738] -# - RFC2255[https://datatracker.ietf.org/doc/html/rfc2255] -# - RFC2368[https://datatracker.ietf.org/doc/html/rfc2368] -# - RFC2373[https://datatracker.ietf.org/doc/html/rfc2373] -# - RFC2396[https://datatracker.ietf.org/doc/html/rfc2396] -# - RFC2732[https://datatracker.ietf.org/doc/html/rfc2732] -# - RFC3986[https://datatracker.ietf.org/doc/html/rfc3986] +# - RFC822[https://www.rfc-editor.org/rfc/rfc822] +# - RFC1738[https://www.rfc-editor.org/rfc/rfc1738] +# - RFC2255[https://www.rfc-editor.org/rfc/rfc2255] +# - RFC2368[https://www.rfc-editor.org/rfc/rfc2368] +# - RFC2373[https://www.rfc-editor.org/rfc/rfc2373] +# - RFC2396[https://www.rfc-editor.org/rfc/rfc2396] +# - RFC2732[https://www.rfc-editor.org/rfc/rfc2732] +# - RFC3986[https://www.rfc-editor.org/rfc/rfc3986] # # == Class tree # diff --git a/lib/uri/generic.rb b/lib/uri/generic.rb index baa6a4c34c3de3..bdd366661ecfd4 100644 --- a/lib/uri/generic.rb +++ b/lib/uri/generic.rb @@ -945,7 +945,7 @@ def fragment=(v) # == Description # # URI has components listed in order of decreasing significance from left to right, - # see RFC3986 https://tools.ietf.org/html/rfc3986 1.2.3. + # see RFC3986 https://www.rfc-editor.org/rfc/rfc3986 1.2.3. # # == Usage # diff --git a/lib/uri/http.rb b/lib/uri/http.rb index 306daf19656d93..900b132c8c1cbf 100644 --- a/lib/uri/http.rb +++ b/lib/uri/http.rb @@ -85,7 +85,7 @@ def request_uri # == Description # # Returns the authority for an HTTP uri, as defined in - # https://datatracker.ietf.org/doc/html/rfc3986/#section-3.2. + # https://www.rfc-editor.org/rfc/rfc3986#section-3.2. # # # Example: @@ -106,7 +106,7 @@ def authority # == Description # # Returns the origin for an HTTP uri, as defined in - # https://datatracker.ietf.org/doc/html/rfc6454. + # https://www.rfc-editor.org/rfc/rfc6454. # # # Example: diff --git a/spec/bundler/bundler/digest_spec.rb b/spec/bundler/bundler/digest_spec.rb index fd7b0c968e007c..f876827964a3a0 100644 --- a/spec/bundler/bundler/digest_spec.rb +++ b/spec/bundler/bundler/digest_spec.rb @@ -11,7 +11,7 @@ it "is compatible with stdlib" do random_strings = ["foo", "skfjsdlkfjsdf", "3924m", "ldskfj"] - # https://datatracker.ietf.org/doc/html/rfc3174#section-7.3 + # https://www.rfc-editor.org/rfc/rfc3174#section-7.3 rfc3174_test_cases = ["abc", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", "a", "01234567" * 8] (random_strings + rfc3174_test_cases).each do |payload| diff --git a/test/openssl/test_ssl.rb b/test/openssl/test_ssl.rb index dcb7757add02be..66d63a981d08c2 100644 --- a/test/openssl/test_ssl.rb +++ b/test/openssl/test_ssl.rb @@ -691,7 +691,7 @@ def test_verify_wildcard assert_equal(true, OpenSSL::SSL.verify_wildcard("xn--qdk4b9b", "xn--qdk4b9b")) end - # Comments in this test is excerpted from http://tools.ietf.org/html/rfc6125#page-27 + # Comments in this test is excerpted from https://www.rfc-editor.org/rfc/rfc6125#page-27 def test_post_connection_check_wildcard_san # case-insensitive ASCII comparison # RFC 6125, section 6.4.1 diff --git a/tool/lib/webrick/httprequest.rb b/tool/lib/webrick/httprequest.rb index d34eac7ecf6ed0..258ee37a38efbe 100644 --- a/tool/lib/webrick/httprequest.rb +++ b/tool/lib/webrick/httprequest.rb @@ -402,7 +402,7 @@ def fixup() # :nodoc: # This method provides the metavariables defined by the revision 3 # of "The WWW Common Gateway Interface Version 1.1" # To browse the current document of CGI Version 1.1, see below: - # http://tools.ietf.org/html/rfc3875 + # https://www.rfc-editor.org/rfc/rfc3875 def meta_vars meta = Hash.new From 2ab9fb1c2e659f1f819ed63796171b2129255185 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Thu, 28 Mar 2024 22:28:37 +0900 Subject: [PATCH 128/211] [Bug #20398] Terminate token buffer at invalid octal number --- parse.y | 1 + 1 file changed, 1 insertion(+) diff --git a/parse.y b/parse.y index 585130c3465ed6..55619273b8e317 100644 --- a/parse.y +++ b/parse.y @@ -10164,6 +10164,7 @@ parse_numeric(struct parser_params *p, int c) /* prefixed octal */ c = nextc(p); if (c == -1 || c == '_' || !ISDIGIT(c)) { + tokfix(p); return no_digits(p); } } From 4fa8fefd9ca298096d4d3a52a29542d5e6d045d6 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Thu, 28 Mar 2024 23:19:39 +0900 Subject: [PATCH 129/211] Suppress warning at literal string --- spec/ruby/library/stringio/binmode_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/ruby/library/stringio/binmode_spec.rb b/spec/ruby/library/stringio/binmode_spec.rb index 853d9c9bd66542..9e92c63814e84a 100644 --- a/spec/ruby/library/stringio/binmode_spec.rb +++ b/spec/ruby/library/stringio/binmode_spec.rb @@ -3,7 +3,7 @@ describe "StringIO#binmode" do it "returns self" do - io = StringIO.new("example") + io = StringIO.new(+"example") io.binmode.should equal(io) end From 7055dcf9156e5951b0539577379356f02356ea05 Mon Sep 17 00:00:00 2001 From: Andrew Konchin Date: Thu, 28 Mar 2024 16:12:45 +0200 Subject: [PATCH 130/211] [ruby/prism] Improve description for InterpolatedStringNodeFlags https://github.com/ruby/prism/commit/caa576d63f --- prism/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/prism/config.yml b/prism/config.yml index bb1dd6e6b02ec0..1d2c627ce4dcff 100644 --- a/prism/config.yml +++ b/prism/config.yml @@ -629,9 +629,9 @@ flags: - name: InterpolatedStringNodeFlags values: - name: FROZEN - comment: "frozen by virtue of a `frozen_string_literal: true` comment or `--enable-frozen-string-literal`" + comment: "frozen by virtue of a `frozen_string_literal: true` comment or `--enable-frozen-string-literal`; only for adjacent string literals like `'a' 'b'`" - name: MUTABLE - comment: "mutable by virtue of a `frozen_string_literal: false` comment or `--disable-frozen-string-literal`" + comment: "mutable by virtue of a `frozen_string_literal: false` comment or `--disable-frozen-string-literal`; only for adjacent string literals like `'a' 'b'`" comment: Flags for interpolated string nodes that indicated mutability if they are also marked as literals. - name: KeywordHashNodeFlags values: From 03ab4a56d239a9816431fe65729243a223db6e2c Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Thu, 28 Mar 2024 23:58:07 +0900 Subject: [PATCH 131/211] Clean symlinks to be runnable [ci skip] --- common.mk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common.mk b/common.mk index 9dbca5b731da3f..173b4fb2a725bd 100644 --- a/common.mk +++ b/common.mk @@ -730,10 +730,10 @@ clean-local:: clean-runnable bin/clean-runnable:: PHONY $(Q)$(CHDIR) bin 2>$(NULL) && $(RM) $(PROGRAM) $(WPROGRAM) $(GORUBY)$(EXEEXT) bin/*.$(DLEXT) 2>$(NULL) || $(NULLCMD) lib/clean-runnable:: PHONY - $(Q)$(CHDIR) lib 2>$(NULL) && $(RM) $(LIBRUBY_A) $(LIBRUBY) $(LIBRUBY_ALIASES) $(RUBY_BASE_NAME)/$(RUBY_PROGRAM_VERSION) $(RUBY_BASE_NAME)/vendor_ruby 2>$(NULL) || $(NULLCMD) + $(Q)$(CHDIR) lib 2>$(NULL) && $(RM) $(LIBRUBY_A) $(LIBRUBY) $(LIBRUBY_ALIASES) $(RUBY_BASE_NAME)/$(ruby_version) $(RUBY_BASE_NAME)/vendor_ruby 2>$(NULL) || $(NULLCMD) clean-runnable:: bin/clean-runnable lib/clean-runnable PHONY $(Q)$(RMDIR) lib/$(RUBY_BASE_NAME) lib bin 2>$(NULL) || $(NULLCMD) - -$(Q)$(RM) $(EXTOUT)/$(arch)/rbconfig.rb + -$(Q)$(RM) $(EXTOUT)/$(arch)/rbconfig.rb $(EXTOUT)/common/$(arch) -$(Q)$(RMALL) exe/ clean-ext:: PHONY clean-golf: PHONY From fa0a62413ab9bdf72855a6614835174f50f29474 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Wed, 27 Mar 2024 11:16:24 -0400 Subject: [PATCH 132/211] Don't check for dynamic symbol when reference updating All symbols in the GC are dynamic symbols, so we don't need to check it. --- gc.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/gc.c b/gc.c index a5023d62d82d5b..8d65fa31fc032f 100644 --- a/gc.c +++ b/gc.c @@ -10202,9 +10202,7 @@ gc_update_object_references(rb_objspace_t *objspace, VALUE obj) break; case T_SYMBOL: - if (DYNAMIC_SYM_P((VALUE)any)) { - UPDATE_IF_MOVED(objspace, RSYMBOL(any)->fstr); - } + UPDATE_IF_MOVED(objspace, RSYMBOL(any)->fstr); break; case T_FLOAT: From 97b2cc34359968459a6eba2ac166f3650adf47be Mon Sep 17 00:00:00 2001 From: Jake Zimmerman Date: Mon, 25 Mar 2024 15:53:53 -0700 Subject: [PATCH 133/211] Allow FormatError to take either String or Gem for source MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Most of the calls to `FormatError.new` pass `@gem` for the second argument, which has a `path` method. But in one case—on package.rb:691 in `verify_gz`, the `source` argument is a `String`. So if there's ever a GZip decode error when attempting to read the contents of the `data.tar.gz` file, instead of reporting the underlying GZip error (which might be something like "unexpected end of file"), we would report instead a NoMethodError coming from package.rb ``` Exception while verifying sorbet-0.5.11301.gem ERROR: While executing gem ... (NoMethodError) undefined method `path' for "data.tar.gz":String @path = source.path ^^^^^ ``` There are two ways to fix this: 1. Make `FormatError#initialize` aware of the fact that `source` might sometimes be a `String` 2. Make the call to `FormatError.new` in `verify_gz` pass `@gem` instead of `entry.full_name`. I've chosen 1 because I think it's more useful to see "unexpected end of file in data.tar.gz" instead of "unexpected end of file in sorbet-0.5.11301.gem." The end of file **is actually** in data.tar.gz, not in the gem file itself, which was decoded successfully. --- lib/rubygems/package.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/rubygems/package.rb b/lib/rubygems/package.rb index 72a179da376d87..1d5d7642376a55 100644 --- a/lib/rubygems/package.rb +++ b/lib/rubygems/package.rb @@ -59,7 +59,7 @@ class FormatError < Error def initialize(message, source = nil) if source - @path = source.path + @path = source.is_a?(String) ? source : source.path message += " in #{path}" if path end From fcc06fa82ab6f3916d8d9202c47db4fae48137dd Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 28 Mar 2024 09:51:16 -0400 Subject: [PATCH 134/211] [ruby/prism] CLI -x flag https://github.com/ruby/prism/commit/2068e3c30a --- lib/prism/ffi.rb | 3 - prism/config.yml | 2 + prism/extension.c | 8 +- prism/options.c | 11 - prism/options.h | 15 - prism/prism.c | 110 +++- .../templates/include/prism/diagnostic.h.erb | 9 +- prism/templates/src/diagnostic.c.erb | 469 +++++++++--------- prism/util/pm_newline_list.c | 8 + prism/util/pm_newline_list.h | 6 + test/prism/command_line_test.rb | 23 + test/prism/ruby_api_test.rb | 15 - 12 files changed, 375 insertions(+), 304 deletions(-) diff --git a/lib/prism/ffi.rb b/lib/prism/ffi.rb index 1ca99db6810034..0a064a5c941850 100644 --- a/lib/prism/ffi.rb +++ b/lib/prism/ffi.rb @@ -382,9 +382,6 @@ def dump_options(options) template << "l" values << options.fetch(:line, 1) - template << "L" - values << options.fetch(:offset, 0) - template << "L" if (encoding = options[:encoding]) name = encoding.name diff --git a/prism/config.yml b/prism/config.yml index 1d2c627ce4dcff..5ed4cf1b2128ea 100644 --- a/prism/config.yml +++ b/prism/config.yml @@ -206,6 +206,7 @@ errors: - RESCUE_TERM - RESCUE_VARIABLE - RETURN_INVALID + - SCRIPT_NOT_FOUND - SINGLETON_FOR_LITERALS - STATEMENT_ALIAS - STATEMENT_POSTEXE_END @@ -255,6 +256,7 @@ warnings: - KEYWORD_EOL - LITERAL_IN_CONDITION_DEFAULT - LITERAL_IN_CONDITION_VERBOSE + - SHEBANG_CARRIAGE_RETURN - UNEXPECTED_CARRIAGE_RETURN tokens: - name: EOF diff --git a/prism/extension.c b/prism/extension.c index 921e197783edc1..d19a004beb5ab3 100644 --- a/prism/extension.c +++ b/prism/extension.c @@ -28,7 +28,6 @@ ID rb_option_id_encoding; ID rb_option_id_filepath; ID rb_option_id_frozen_string_literal; ID rb_option_id_line; -ID rb_option_id_offset; ID rb_option_id_scopes; ID rb_option_id_version; @@ -139,8 +138,6 @@ build_options_i(VALUE key, VALUE value, VALUE argument) { if (!NIL_P(value)) pm_options_encoding_set(options, rb_enc_name(rb_to_encoding(value))); } else if (key_id == rb_option_id_line) { if (!NIL_P(value)) pm_options_line_set(options, NUM2INT(value)); - } else if (key_id == rb_option_id_offset) { - if (!NIL_P(value)) pm_options_offset_set(options, NUM2UINT(value)); } else if (key_id == rb_option_id_frozen_string_literal) { if (!NIL_P(value)) pm_options_frozen_string_literal_set(options, RTEST(value)); } else if (key_id == rb_option_id_version) { @@ -448,8 +445,8 @@ parser_errors(pm_parser_t *parser, rb_encoding *encoding, VALUE source) { VALUE level = Qnil; switch (error->level) { - case PM_ERROR_LEVEL_FATAL: - level = ID2SYM(rb_intern("fatal")); + case PM_ERROR_LEVEL_SYNTAX: + level = ID2SYM(rb_intern("syntax")); break; case PM_ERROR_LEVEL_ARGUMENT: level = ID2SYM(rb_intern("argument")); @@ -1347,7 +1344,6 @@ Init_prism(void) { rb_option_id_filepath = rb_intern_const("filepath"); rb_option_id_frozen_string_literal = rb_intern_const("frozen_string_literal"); rb_option_id_line = rb_intern_const("line"); - rb_option_id_offset = rb_intern_const("offset"); rb_option_id_scopes = rb_intern_const("scopes"); rb_option_id_version = rb_intern_const("version"); diff --git a/prism/options.c b/prism/options.c index cd575357aae58f..2854b765b9323b 100644 --- a/prism/options.c +++ b/prism/options.c @@ -24,14 +24,6 @@ pm_options_line_set(pm_options_t *options, int32_t line) { options->line = line; } -/** - * Set the offset option on the given options struct. - */ -PRISM_EXPORTED_FUNCTION void -pm_options_offset_set(pm_options_t *options, uint32_t offset) { - options->offset = offset; -} - /** * Set the frozen string literal option on the given options struct. */ @@ -201,9 +193,6 @@ pm_options_read(pm_options_t *options, const char *data) { options->line = pm_options_read_s32(data); data += 4; - options->offset = pm_options_read_u32(data); - data += 4; - uint32_t encoding_length = pm_options_read_u32(data); data += 4; diff --git a/prism/options.h b/prism/options.h index 9a4d6969c3c67b..d0b46a086491c7 100644 --- a/prism/options.h +++ b/prism/options.h @@ -66,12 +66,6 @@ typedef struct { */ int32_t line; - /** - * The offset within the file that the parse starts on. This value is - * 0-indexed. - */ - uint32_t offset; - /** * The name of the encoding that the source file is in. Note that this must * correspond to a name that can be found with Encoding.find in Ruby. @@ -164,14 +158,6 @@ PRISM_EXPORTED_FUNCTION void pm_options_filepath_set(pm_options_t *options, cons */ PRISM_EXPORTED_FUNCTION void pm_options_line_set(pm_options_t *options, int32_t line); -/** - * Set the offset option on the given options struct. - * - * @param options The options struct to set the offset on. - * @param offset The offset to set. - */ -PRISM_EXPORTED_FUNCTION void pm_options_offset_set(pm_options_t *options, uint32_t offset); - /** * Set the encoding option on the given options struct. * @@ -267,7 +253,6 @@ PRISM_EXPORTED_FUNCTION void pm_options_free(pm_options_t *options); * | `4` | the length of the filepath | * | ... | the filepath bytes | * | `4` | the line number | - * | `4` | the offset | * | `4` | the length the encoding | * | ... | the encoding bytes | * | `1` | frozen string literal | diff --git a/prism/prism.c b/prism/prism.c index 5389cac9f60839..e33d3e1d3c8760 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -500,6 +500,9 @@ debug_lex_state_set(pm_parser_t *parser, pm_lex_state_t state, char const * call /** True if the -p command line option was given. */ #define PM_PARSER_COMMAND_LINE_OPTION_P(parser) PM_PARSER_COMMAND_LINE_OPTION(parser, PM_OPTIONS_COMMAND_LINE_P) +/** True if the -x command line option was given. */ +#define PM_PARSER_COMMAND_LINE_OPTION_X(parser) PM_PARSER_COMMAND_LINE_OPTION(parser, PM_OPTIONS_COMMAND_LINE_X) + /******************************************************************************/ /* Diagnostic-related functions */ /******************************************************************************/ @@ -19122,6 +19125,38 @@ parse_program(pm_parser_t *parser) { /* External functions */ /******************************************************************************/ +/** + * A vendored version of strnstr that is used to find a substring within a + * string with a given length. This function is used to search for the Ruby + * engine name within a shebang when the -x option is passed to Ruby. + * + * The only modification that we made here is that we don't do NULL byte checks + * because we know the little parameter will not have a NULL byte and we allow + * the big parameter to have them. + */ +static const char * +pm_strnstr(const char *big, const char *little, size_t big_length) { + size_t little_length = strlen(little); + + for (const char *big_end = big + big_length; big < big_end; big++) { + if (*big == *little && memcmp(big, little, little_length) == 0) return big; + } + + return NULL; +} + +/** + * Potentially warn the user if the shebang that has been found to include + * "ruby" has a carriage return at the end, as that can cause problems on some + * platforms. + */ +static void +pm_parser_warn_shebang_carriage_return(pm_parser_t *parser, const uint8_t *start, size_t length) { + if (length > 2 && start[length - 1] == '\n' && start[length - 2] == '\r') { + pm_parser_warn(parser, start, start + length, PM_WARN_SHEBANG_CARRIAGE_RETURN); + } +} + /** * Initialize a parser with the given start and end pointers. */ @@ -19208,22 +19243,6 @@ pm_parser_init(pm_parser_t *parser, const uint8_t *source, size_t size, const pm // line option parser->start_line = options->line; - // offset option - if (options->offset != 0) { - const uint8_t *cursor = parser->start; - const uint8_t *offset = cursor + options->offset; - - const uint8_t *newline = NULL; - while ((newline = next_newline(cursor, parser->end - cursor)) != NULL) { - if (newline > offset) break; - pm_newline_list_append(&parser->newline_list, newline); - cursor = newline + 1; - } - - parser->previous = (pm_token_t) { .type = PM_TOKEN_EOF, .start = offset, .end = offset }; - parser->current = (pm_token_t) { .type = PM_TOKEN_EOF, .start = offset, .end = offset }; - } - // encoding option size_t encoding_length = pm_string_length(&options->encoding); if (encoding_length > 0) { @@ -19277,12 +19296,65 @@ pm_parser_init(pm_parser_t *parser, const uint8_t *source, size_t size, const pm } } + // If the -x command line flag is set, or the first shebang of the file does + // not include "ruby", then we'll search for a shebang that does include + // "ruby" and start parsing from there. + bool search_shebang = PM_PARSER_COMMAND_LINE_OPTION_X(parser); + // If the first two bytes of the source are a shebang, then we'll indicate // that the encoding comment is at the end of the shebang. if (peek(parser) == '#' && peek_offset(parser, 1) == '!') { - const uint8_t *encoding_comment_start = next_newline(source, (ptrdiff_t) size); - if (encoding_comment_start) { - parser->encoding_comment_start = encoding_comment_start + 1; + const uint8_t *newline = next_newline(parser->start, parser->end - parser->start); + size_t length = (size_t) ((newline != NULL ? newline : parser->end) - parser->start); + + if (pm_strnstr((const char *) parser->start, "ruby", length) != NULL) { + pm_parser_warn_shebang_carriage_return(parser, parser->start, length); + if (newline != NULL) parser->encoding_comment_start = newline + 1; + search_shebang = false; + } else { + search_shebang = true; + } + } + + // Here we're going to find the first shebang that includes "ruby" and start + // parsing from there. + if (search_shebang) { + bool found = false; + + // This is going to point to the start of each line as we check it. + // We'll maintain a moving window looking at each line at they come. + const uint8_t *cursor = parser->start; + + // The newline pointer points to the end of the current line that we're + // considering. If it is NULL, then we're at the end of the file. + const uint8_t *newline = next_newline(cursor, parser->end - cursor); + + while (newline != NULL) { + pm_newline_list_append(&parser->newline_list, newline); + + cursor = newline + 1; + newline = next_newline(cursor, parser->end - cursor); + + size_t length = (size_t) ((newline != NULL ? newline : parser->end) - cursor); + if (length > 2 && cursor[0] == '#' && cursor[1] == '!') { + if (parser->newline_list.size == 1) { + pm_parser_warn_shebang_carriage_return(parser, cursor, length); + } + + if (pm_strnstr((const char *) cursor, "ruby", length) != NULL) { + found = true; + parser->encoding_comment_start = newline + 1; + break; + } + } + } + + if (found) { + parser->previous = (pm_token_t) { .type = PM_TOKEN_EOF, .start = cursor, .end = cursor }; + parser->current = (pm_token_t) { .type = PM_TOKEN_EOF, .start = cursor, .end = cursor }; + } else { + pm_parser_err(parser, parser->start, parser->start, PM_ERR_SCRIPT_NOT_FOUND); + pm_newline_list_clear(&parser->newline_list); } } } diff --git a/prism/templates/include/prism/diagnostic.h.erb b/prism/templates/include/prism/diagnostic.h.erb index 53a700d409397d..07bbc8fae79264 100644 --- a/prism/templates/include/prism/diagnostic.h.erb +++ b/prism/templates/include/prism/diagnostic.h.erb @@ -66,11 +66,14 @@ typedef struct { * The levels of errors generated during parsing. */ typedef enum { - /** For errors that cannot be recovered from. */ - PM_ERROR_LEVEL_FATAL = 0, + /** For errors that should raise a syntax error. */ + PM_ERROR_LEVEL_SYNTAX = 0, /** For errors that should raise an argument error. */ - PM_ERROR_LEVEL_ARGUMENT = 1 + PM_ERROR_LEVEL_ARGUMENT = 1, + + /** For errors that should raise a load error. */ + PM_ERROR_LEVEL_LOAD = 2 } pm_error_level_t; /** diff --git a/prism/templates/src/diagnostic.c.erb b/prism/templates/src/diagnostic.c.erb index 818b12d98be8c1..4e6b6ec4eefb18 100644 --- a/prism/templates/src/diagnostic.c.erb +++ b/prism/templates/src/diagnostic.c.erb @@ -65,8 +65,9 @@ typedef struct { * * For errors, they are: * - * * `PM_ERROR_LEVEL_FATAL` - The default level for errors. + * * `PM_ERROR_LEVEL_SYNTAX` - Errors that should raise SyntaxError. * * `PM_ERROR_LEVEL_ARGUMENT` - Errors that should raise ArgumentError. + * * `PM_ERROR_LEVEL_LOAD` - Errors that should raise LoadError. * * For warnings, they are: * @@ -75,242 +76,245 @@ typedef struct { */ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { // Special error that can be replaced - [PM_ERR_CANNOT_PARSE_EXPRESSION] = { "cannot parse the expression", PM_ERROR_LEVEL_FATAL }, + [PM_ERR_CANNOT_PARSE_EXPRESSION] = { "cannot parse the expression", PM_ERROR_LEVEL_SYNTAX }, // Errors that should raise argument errors [PM_ERR_INVALID_ENCODING_MAGIC_COMMENT] = { "unknown or invalid encoding in the magic comment", PM_ERROR_LEVEL_ARGUMENT }, + // Errors that should raise load errors + [PM_ERR_SCRIPT_NOT_FOUND] = { "no Ruby script found in input", PM_ERROR_LEVEL_LOAD }, + // Errors that should raise syntax errors - [PM_ERR_ALIAS_ARGUMENT] = { "invalid argument being passed to `alias`; expected a bare word, symbol, constant, or global variable", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_AMPAMPEQ_MULTI_ASSIGN] = { "unexpected `&&=` in a multiple assignment", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ARGUMENT_AFTER_BLOCK] = { "unexpected argument after a block argument", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ARGUMENT_AFTER_FORWARDING_ELLIPSES] = { "unexpected argument after `...`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ARGUMENT_BARE_HASH] = { "unexpected bare hash argument", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ARGUMENT_BLOCK_FORWARDING] = { "both a block argument and a forwarding argument; only one block is allowed", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ARGUMENT_BLOCK_MULTI] = { "multiple block arguments; only one block is allowed", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ARGUMENT_FORMAL_CLASS] = { "invalid formal argument; formal argument cannot be a class variable", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ARGUMENT_FORMAL_CONSTANT] = { "invalid formal argument; formal argument cannot be a constant", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ARGUMENT_FORMAL_GLOBAL] = { "invalid formal argument; formal argument cannot be a global variable", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ARGUMENT_FORMAL_IVAR] = { "invalid formal argument; formal argument cannot be an instance variable", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ARGUMENT_FORWARDING_UNBOUND] = { "unexpected `...` in an non-parenthesized call", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ARGUMENT_IN] = { "unexpected `in` keyword in arguments", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ARGUMENT_NO_FORWARDING_AMP] = { "unexpected `&` when the parent method is not forwarding", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ARGUMENT_NO_FORWARDING_ELLIPSES] = { "unexpected `...` when the parent method is not forwarding", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ARGUMENT_NO_FORWARDING_STAR] = { "unexpected `*` when the parent method is not forwarding", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ARGUMENT_NO_FORWARDING_STAR_STAR] = { "unexpected `**` when the parent method is not forwarding", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ARGUMENT_SPLAT_AFTER_ASSOC_SPLAT] = { "unexpected `*` splat argument after a `**` keyword splat argument", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ARGUMENT_SPLAT_AFTER_SPLAT] = { "unexpected `*` splat argument after a `*` splat argument", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ARGUMENT_TERM_PAREN] = { "expected a `)` to close the arguments", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ARGUMENT_UNEXPECTED_BLOCK] = { "unexpected `{` after a method call without parenthesis", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ARRAY_ELEMENT] = { "expected an element for the array", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ARRAY_EXPRESSION] = { "expected an expression for the array element", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ARRAY_EXPRESSION_AFTER_STAR] = { "expected an expression after `*` in the array", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ARRAY_SEPARATOR] = { "expected a `,` separator for the array elements", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ARRAY_TERM] = { "expected a `]` to close the array", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_BEGIN_LONELY_ELSE] = { "unexpected `else` in `begin` block; else without rescue is useless", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_BEGIN_TERM] = { "expected an `end` to close the `begin` statement", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_BEGIN_UPCASE_BRACE] = { "expected a `{` after `BEGIN`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_BEGIN_UPCASE_TERM] = { "expected a `}` to close the `BEGIN` statement", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_BEGIN_UPCASE_TOPLEVEL] = { "BEGIN is permitted only at toplevel", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_BLOCK_PARAM_LOCAL_VARIABLE] = { "expected a local variable name in the block parameters", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_BLOCK_PARAM_PIPE_TERM] = { "expected the block parameters to end with `|`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_BLOCK_TERM_BRACE] = { "expected a block beginning with `{` to end with `}`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_BLOCK_TERM_END] = { "expected a block beginning with `do` to end with `end`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_CANNOT_PARSE_STRING_PART] = { "cannot parse the string part", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_CASE_EXPRESSION_AFTER_CASE] = { "expected an expression after `case`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_CASE_EXPRESSION_AFTER_WHEN] = { "expected an expression after `when`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_CASE_MATCH_MISSING_PREDICATE] = { "expected a predicate for a case matching statement", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_CASE_MISSING_CONDITIONS] = { "expected a `when` or `in` clause after `case`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_CASE_TERM] = { "expected an `end` to close the `case` statement", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_CLASS_IN_METHOD] = { "unexpected class definition in a method definition", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_CLASS_NAME] = { "expected a constant name after `class`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_CLASS_SUPERCLASS] = { "expected a superclass after `<`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_CLASS_TERM] = { "expected an `end` to close the `class` statement", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_CLASS_UNEXPECTED_END] = { "unexpected `end`, expecting ';' or '\\n'", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_CONDITIONAL_ELSIF_PREDICATE] = { "expected a predicate expression for the `elsif` statement", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_CONDITIONAL_IF_PREDICATE] = { "expected a predicate expression for the `if` statement", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_CONDITIONAL_PREDICATE_TERM] = { "expected `then` or `;` or '\\n'", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_CONDITIONAL_TERM] = { "expected an `end` to close the conditional clause", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_CONDITIONAL_TERM_ELSE] = { "expected an `end` to close the `else` clause", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_CONDITIONAL_UNLESS_PREDICATE] = { "expected a predicate expression for the `unless` statement", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_CONDITIONAL_UNTIL_PREDICATE] = { "expected a predicate expression for the `until` statement", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_CONDITIONAL_WHILE_PREDICATE] = { "expected a predicate expression for the `while` statement", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_CONSTANT_PATH_COLON_COLON_CONSTANT] = { "expected a constant after the `::` operator", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_DEF_ENDLESS] = { "could not parse the endless method body", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_DEF_ENDLESS_SETTER] = { "invalid method name; a setter method cannot be defined in an endless method definition", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_DEF_NAME] = { "expected a method name", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_DEF_NAME_AFTER_RECEIVER] = { "expected a method name after the receiver", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_DEF_PARAMS_TERM] = { "expected a delimiter to close the parameters", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_DEF_PARAMS_TERM_PAREN] = { "expected a `)` to close the parameters", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_DEF_RECEIVER] = { "expected a receiver for the method definition", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_DEF_RECEIVER_TERM] = { "expected a `.` or `::` after the receiver in a method definition", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_DEF_TERM] = { "expected an `end` to close the `def` statement", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_DEFINED_EXPRESSION] = { "expected an expression after `defined?`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_EMBDOC_TERM] = { "could not find a terminator for the embedded document", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_EMBEXPR_END] = { "expected a `}` to close the embedded expression", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_EMBVAR_INVALID] = { "invalid embedded variable", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_END_UPCASE_BRACE] = { "expected a `{` after `END`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_END_UPCASE_TERM] = { "expected a `}` to close the `END` statement", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ESCAPE_INVALID_CONTROL] = { "invalid control escape sequence", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ESCAPE_INVALID_CONTROL_REPEAT] = { "invalid control escape sequence; control cannot be repeated", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ESCAPE_INVALID_HEXADECIMAL] = { "invalid hexadecimal escape sequence", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ESCAPE_INVALID_META] = { "invalid meta escape sequence", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ESCAPE_INVALID_META_REPEAT] = { "invalid meta escape sequence; meta cannot be repeated", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ESCAPE_INVALID_UNICODE] = { "invalid Unicode escape sequence", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ESCAPE_INVALID_UNICODE_CM_FLAGS] = { "invalid Unicode escape sequence; Unicode cannot be combined with control or meta flags", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ESCAPE_INVALID_UNICODE_LITERAL] = { "invalid Unicode escape sequence; multiple codepoints are not allowed in a character literal", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ESCAPE_INVALID_UNICODE_LONG] = { "invalid Unicode escape sequence; maximum length is 6 digits", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ESCAPE_INVALID_UNICODE_TERM] = { "invalid Unicode escape sequence; needs closing `}`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_EXPECT_ARGUMENT] = { "expected an argument", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_EXPECT_EOL_AFTER_STATEMENT] = { "unexpected %s, expecting end-of-input", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_EXPECT_EXPRESSION_AFTER_AMPAMPEQ] = { "expected an expression after `&&=`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_EXPECT_EXPRESSION_AFTER_PIPEPIPEEQ] = { "expected an expression after `||=`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_EXPECT_EXPRESSION_AFTER_COMMA] = { "expected an expression after `,`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_EXPECT_EXPRESSION_AFTER_EQUAL] = { "expected an expression after `=`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_EXPECT_EXPRESSION_AFTER_LESS_LESS] = { "expected an expression after `<<`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_EXPECT_EXPRESSION_AFTER_LPAREN] = { "expected an expression after `(`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_EXPECT_EXPRESSION_AFTER_OPERATOR] = { "expected an expression after the operator", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_EXPECT_EXPRESSION_AFTER_SPLAT] = { "expected an expression after `*` splat in an argument", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_EXPECT_EXPRESSION_AFTER_SPLAT_HASH] = { "expected an expression after `**` in a hash", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_EXPECT_EXPRESSION_AFTER_STAR] = { "expected an expression after `*`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_EXPECT_IDENT_REQ_PARAMETER] = { "expected an identifier for the required parameter", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_EXPECT_LPAREN_REQ_PARAMETER] = { "expected a `(` to start a required parameter", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_EXPECT_RBRACKET] = { "expected a matching `]`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_EXPECT_RPAREN] = { "expected a matching `)`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_EXPECT_RPAREN_AFTER_MULTI] = { "expected a `)` after multiple assignment", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_EXPECT_RPAREN_REQ_PARAMETER] = { "expected a `)` to end a required parameter", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_EXPECT_STRING_CONTENT] = { "expected string content after opening string delimiter", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_EXPECT_WHEN_DELIMITER] = { "expected a delimiter after the predicates of a `when` clause", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_EXPRESSION_BARE_HASH] = { "unexpected bare hash in expression", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_FLOAT_PARSE] = { "could not parse the float '%.*s'", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_FOR_COLLECTION] = { "expected a collection after the `in` in a `for` statement", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_FOR_INDEX] = { "expected an index after `for`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_FOR_IN] = { "expected an `in` after the index in a `for` statement", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_FOR_TERM] = { "expected an `end` to close the `for` loop", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_HASH_EXPRESSION_AFTER_LABEL] = { "expected an expression after the label in a hash", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_HASH_KEY] = { "unexpected %s, expecting '}' or a key in the hash literal", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_HASH_ROCKET] = { "expected a `=>` between the hash key and value", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_HASH_TERM] = { "expected a `}` to close the hash literal", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_HASH_VALUE] = { "expected a value in the hash literal", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_HEREDOC_TERM] = { "could not find a terminator for the heredoc", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_INCOMPLETE_QUESTION_MARK] = { "incomplete expression at `?`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_INCOMPLETE_VARIABLE_CLASS_3_3_0] = { "`%.*s' is not allowed as a class variable name", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_INCOMPLETE_VARIABLE_CLASS] = { "'%.*s' is not allowed as a class variable name", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_INCOMPLETE_VARIABLE_INSTANCE_3_3_0] = { "`%.*s' is not allowed as an instance variable name", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_INCOMPLETE_VARIABLE_INSTANCE] = { "'%.*s' is not allowed as an instance variable name", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_INVALID_FLOAT_EXPONENT] = { "invalid exponent", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_INVALID_NUMBER_BINARY] = { "invalid binary number", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_INVALID_NUMBER_DECIMAL] = { "invalid decimal number", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_INVALID_NUMBER_HEXADECIMAL] = { "invalid hexadecimal number", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_INVALID_NUMBER_OCTAL] = { "invalid octal number", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_INVALID_NUMBER_UNDERSCORE] = { "invalid underscore placement in number", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_INVALID_CHARACTER] = { "invalid character 0x%X", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_INVALID_MULTIBYTE_CHAR] = { "invalid multibyte char (%s)", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_INVALID_MULTIBYTE_CHARACTER] = { "invalid multibyte character 0x%X", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_INVALID_MULTIBYTE_ESCAPE] = { "invalid multibyte escape: /%.*s/", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_INVALID_PRINTABLE_CHARACTER] = { "invalid character `%c`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_INVALID_PERCENT] = { "invalid `%` token", PM_ERROR_LEVEL_FATAL }, // TODO WHAT? - [PM_ERR_INVALID_VARIABLE_GLOBAL_3_3_0] = { "`%.*s' is not allowed as a global variable name", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_INVALID_VARIABLE_GLOBAL] = { "'%.*s' is not allowed as a global variable name", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_IT_NOT_ALLOWED_NUMBERED] = { "`it` is not allowed when an numbered parameter is defined", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_IT_NOT_ALLOWED_ORDINARY] = { "`it` is not allowed when an ordinary parameter is defined", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_LAMBDA_OPEN] = { "expected a `do` keyword or a `{` to open the lambda block", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_LAMBDA_TERM_BRACE] = { "expected a lambda block beginning with `{` to end with `}`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_LAMBDA_TERM_END] = { "expected a lambda block beginning with `do` to end with `end`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_LIST_I_LOWER_ELEMENT] = { "expected a symbol in a `%i` list", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_LIST_I_LOWER_TERM] = { "expected a closing delimiter for the `%i` list", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_LIST_I_UPPER_ELEMENT] = { "expected a symbol in a `%I` list", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_LIST_I_UPPER_TERM] = { "expected a closing delimiter for the `%I` list", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_LIST_W_LOWER_ELEMENT] = { "expected a string in a `%w` list", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_LIST_W_LOWER_TERM] = { "expected a closing delimiter for the `%w` list", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_LIST_W_UPPER_ELEMENT] = { "expected a string in a `%W` list", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_LIST_W_UPPER_TERM] = { "expected a closing delimiter for the `%W` list", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_MALLOC_FAILED] = { "failed to allocate memory", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_MIXED_ENCODING] = { "UTF-8 mixed within %s source", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_MODULE_IN_METHOD] = { "unexpected module definition in a method definition", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_MODULE_NAME] = { "expected a constant name after `module`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_MODULE_TERM] = { "expected an `end` to close the `module` statement", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_MULTI_ASSIGN_MULTI_SPLATS] = { "multiple splats in multiple assignment", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_MULTI_ASSIGN_UNEXPECTED_REST] = { "unexpected '%.*s' resulting in multiple splats in multiple assignment", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_NOT_EXPRESSION] = { "expected an expression after `not`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_NO_LOCAL_VARIABLE] = { "%.*s: no such local variable", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_NUMBER_LITERAL_UNDERSCORE] = { "number literal ending with a `_`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_NUMBERED_PARAMETER_IT] = { "numbered parameters are not allowed when an 'it' parameter is defined", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_NUMBERED_PARAMETER_ORDINARY] = { "numbered parameters are not allowed when an ordinary parameter is defined", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_NUMBERED_PARAMETER_OUTER_SCOPE] = { "numbered parameter is already used in outer scope", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_OPERATOR_MULTI_ASSIGN] = { "unexpected operator for a multiple assignment", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_OPERATOR_WRITE_ARGUMENTS] = { "unexpected operator after a call with arguments", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_OPERATOR_WRITE_BLOCK] = { "unexpected operator after a call with a block", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PARAMETER_ASSOC_SPLAT_MULTI] = { "unexpected multiple `**` splat parameters", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PARAMETER_BLOCK_MULTI] = { "multiple block parameters; only one block is allowed", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PARAMETER_CIRCULAR] = { "parameter default value references itself", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PARAMETER_METHOD_NAME] = { "unexpected name for a parameter", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PARAMETER_NAME_REPEAT] = { "repeated parameter name", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PARAMETER_NO_DEFAULT] = { "expected a default value for the parameter", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PARAMETER_NO_DEFAULT_KW] = { "expected a default value for the keyword parameter", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PARAMETER_NUMBERED_RESERVED] = { "%.2s is reserved for numbered parameters", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PARAMETER_ORDER] = { "unexpected parameter order", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PARAMETER_SPLAT_MULTI] = { "unexpected multiple `*` splat parameters", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PARAMETER_STAR] = { "unexpected parameter `*`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PARAMETER_UNEXPECTED_FWD] = { "unexpected `...` in parameters", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PARAMETER_WILD_LOOSE_COMMA] = { "unexpected `,` in parameters", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PATTERN_EXPRESSION_AFTER_BRACKET] = { "expected a pattern expression after the `[` operator", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PATTERN_EXPRESSION_AFTER_COMMA] = { "expected a pattern expression after `,`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PATTERN_EXPRESSION_AFTER_HROCKET] = { "expected a pattern expression after `=>`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PATTERN_EXPRESSION_AFTER_IN] = { "expected a pattern expression after the `in` keyword", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PATTERN_EXPRESSION_AFTER_KEY] = { "expected a pattern expression after the key", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PATTERN_EXPRESSION_AFTER_PAREN] = { "expected a pattern expression after the `(` operator", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PATTERN_EXPRESSION_AFTER_PIN] = { "expected a pattern expression after the `^` pin operator", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PATTERN_EXPRESSION_AFTER_PIPE] = { "expected a pattern expression after the `|` operator", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PATTERN_EXPRESSION_AFTER_RANGE] = { "expected a pattern expression after the range operator", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PATTERN_EXPRESSION_AFTER_REST] = { "unexpected pattern expression after the `**` expression", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PATTERN_HASH_KEY] = { "expected a key in the hash pattern", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PATTERN_HASH_KEY_LABEL] = { "expected a label as the key in the hash pattern", PM_ERROR_LEVEL_FATAL }, // TODO // THIS // AND // ABOVE // IS WEIRD - [PM_ERR_PATTERN_IDENT_AFTER_HROCKET] = { "expected an identifier after the `=>` operator", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PATTERN_LABEL_AFTER_COMMA] = { "expected a label after the `,` in the hash pattern", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PATTERN_REST] = { "unexpected rest pattern", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PATTERN_TERM_BRACE] = { "expected a `}` to close the pattern expression", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PATTERN_TERM_BRACKET] = { "expected a `]` to close the pattern expression", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PATTERN_TERM_PAREN] = { "expected a `)` to close the pattern expression", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PIPEPIPEEQ_MULTI_ASSIGN] = { "unexpected `||=` in a multiple assignment", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_REGEXP_ENCODING_OPTION_MISMATCH] = { "regexp encoding option '%c' differs from source encoding '%s'", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_REGEXP_INCOMPAT_CHAR_ENCODING] = { "incompatible character encoding: /%.*s/", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_REGEXP_NON_ESCAPED_MBC] = { "/.../n has a non escaped non ASCII character in non ASCII-8BIT script: /%.*s/", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_REGEXP_INVALID_UNICODE_RANGE] = { "invalid Unicode range: /%.*s/", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_REGEXP_UNKNOWN_OPTIONS] = { "unknown regexp %s: %.*s", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_REGEXP_TERM] = { "expected a closing delimiter for the regular expression", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_REGEXP_UTF8_CHAR_NON_UTF8_REGEXP] = { "UTF-8 character in non UTF-8 regexp: /%s/", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_RESCUE_EXPRESSION] = { "expected a rescued expression", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_RESCUE_MODIFIER_VALUE] = { "expected a value after the `rescue` modifier", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_RESCUE_TERM] = { "expected a closing delimiter for the `rescue` clause", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_RESCUE_VARIABLE] = { "expected an exception variable after `=>` in a rescue statement", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_RETURN_INVALID] = { "invalid `return` in a class or module body", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_SINGLETON_FOR_LITERALS] = { "cannot define singleton method for literals", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_STATEMENT_ALIAS] = { "unexpected an `alias` at a non-statement position", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_STATEMENT_POSTEXE_END] = { "unexpected an `END` at a non-statement position", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_STATEMENT_PREEXE_BEGIN] = { "unexpected a `BEGIN` at a non-statement position", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_STATEMENT_UNDEF] = { "unexpected an `undef` at a non-statement position", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_STRING_CONCATENATION] = { "expected a string for concatenation", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_STRING_INTERPOLATED_TERM] = { "expected a closing delimiter for the interpolated string", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_STRING_LITERAL_EOF] = { "unterminated string meets end of file", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_STRING_LITERAL_TERM] = { "unexpected %s, expected a string literal terminator", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_SYMBOL_INVALID] = { "invalid symbol", PM_ERROR_LEVEL_FATAL }, // TODO expected symbol? prism.c ~9719 - [PM_ERR_SYMBOL_TERM_DYNAMIC] = { "expected a closing delimiter for the dynamic symbol", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_SYMBOL_TERM_INTERPOLATED] = { "expected a closing delimiter for the interpolated symbol", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_TERNARY_COLON] = { "expected a `:` after the true expression of a ternary operator", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_TERNARY_EXPRESSION_FALSE] = { "expected an expression after `:` in the ternary operator", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_TERNARY_EXPRESSION_TRUE] = { "expected an expression after `?` in the ternary operator", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_UNDEF_ARGUMENT] = { "invalid argument being passed to `undef`; expected a bare word, constant, or symbol argument", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_UNARY_RECEIVER] = { "unexpected %s, expected a receiver for unary `%c`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_UNEXPECTED_TOKEN_CLOSE_CONTEXT] = { "unexpected %s, assuming it is closing the parent %s", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_UNEXPECTED_TOKEN_IGNORE] = { "unexpected %s, ignoring it", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_UNTIL_TERM] = { "expected an `end` to close the `until` statement", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_VOID_EXPRESSION] = { "unexpected void value expression", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_WHILE_TERM] = { "expected an `end` to close the `while` statement", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_WRITE_TARGET_IN_METHOD] = { "dynamic constant assignment", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_WRITE_TARGET_READONLY] = { "Can't set variable %.*s", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_WRITE_TARGET_UNEXPECTED] = { "unexpected write target", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_XSTRING_TERM] = { "expected a closing delimiter for the `%x` or backtick string", PM_ERROR_LEVEL_FATAL }, + [PM_ERR_ALIAS_ARGUMENT] = { "invalid argument being passed to `alias`; expected a bare word, symbol, constant, or global variable", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_AMPAMPEQ_MULTI_ASSIGN] = { "unexpected `&&=` in a multiple assignment", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARGUMENT_AFTER_BLOCK] = { "unexpected argument after a block argument", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARGUMENT_AFTER_FORWARDING_ELLIPSES] = { "unexpected argument after `...`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARGUMENT_BARE_HASH] = { "unexpected bare hash argument", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARGUMENT_BLOCK_FORWARDING] = { "both a block argument and a forwarding argument; only one block is allowed", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARGUMENT_BLOCK_MULTI] = { "multiple block arguments; only one block is allowed", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARGUMENT_FORMAL_CLASS] = { "invalid formal argument; formal argument cannot be a class variable", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARGUMENT_FORMAL_CONSTANT] = { "invalid formal argument; formal argument cannot be a constant", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARGUMENT_FORMAL_GLOBAL] = { "invalid formal argument; formal argument cannot be a global variable", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARGUMENT_FORMAL_IVAR] = { "invalid formal argument; formal argument cannot be an instance variable", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARGUMENT_FORWARDING_UNBOUND] = { "unexpected `...` in an non-parenthesized call", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARGUMENT_IN] = { "unexpected `in` keyword in arguments", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARGUMENT_NO_FORWARDING_AMP] = { "unexpected `&` when the parent method is not forwarding", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARGUMENT_NO_FORWARDING_ELLIPSES] = { "unexpected `...` when the parent method is not forwarding", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARGUMENT_NO_FORWARDING_STAR] = { "unexpected `*` when the parent method is not forwarding", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARGUMENT_NO_FORWARDING_STAR_STAR] = { "unexpected `**` when the parent method is not forwarding", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARGUMENT_SPLAT_AFTER_ASSOC_SPLAT] = { "unexpected `*` splat argument after a `**` keyword splat argument", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARGUMENT_SPLAT_AFTER_SPLAT] = { "unexpected `*` splat argument after a `*` splat argument", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARGUMENT_TERM_PAREN] = { "expected a `)` to close the arguments", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARGUMENT_UNEXPECTED_BLOCK] = { "unexpected `{` after a method call without parenthesis", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARRAY_ELEMENT] = { "expected an element for the array", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARRAY_EXPRESSION] = { "expected an expression for the array element", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARRAY_EXPRESSION_AFTER_STAR] = { "expected an expression after `*` in the array", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARRAY_SEPARATOR] = { "expected a `,` separator for the array elements", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARRAY_TERM] = { "expected a `]` to close the array", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_BEGIN_LONELY_ELSE] = { "unexpected `else` in `begin` block; else without rescue is useless", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_BEGIN_TERM] = { "expected an `end` to close the `begin` statement", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_BEGIN_UPCASE_BRACE] = { "expected a `{` after `BEGIN`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_BEGIN_UPCASE_TERM] = { "expected a `}` to close the `BEGIN` statement", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_BEGIN_UPCASE_TOPLEVEL] = { "BEGIN is permitted only at toplevel", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_BLOCK_PARAM_LOCAL_VARIABLE] = { "expected a local variable name in the block parameters", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_BLOCK_PARAM_PIPE_TERM] = { "expected the block parameters to end with `|`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_BLOCK_TERM_BRACE] = { "expected a block beginning with `{` to end with `}`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_BLOCK_TERM_END] = { "expected a block beginning with `do` to end with `end`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_CANNOT_PARSE_STRING_PART] = { "cannot parse the string part", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_CASE_EXPRESSION_AFTER_CASE] = { "expected an expression after `case`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_CASE_EXPRESSION_AFTER_WHEN] = { "expected an expression after `when`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_CASE_MATCH_MISSING_PREDICATE] = { "expected a predicate for a case matching statement", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_CASE_MISSING_CONDITIONS] = { "expected a `when` or `in` clause after `case`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_CASE_TERM] = { "expected an `end` to close the `case` statement", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_CLASS_IN_METHOD] = { "unexpected class definition in a method definition", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_CLASS_NAME] = { "expected a constant name after `class`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_CLASS_SUPERCLASS] = { "expected a superclass after `<`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_CLASS_TERM] = { "expected an `end` to close the `class` statement", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_CLASS_UNEXPECTED_END] = { "unexpected `end`, expecting ';' or '\\n'", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_CONDITIONAL_ELSIF_PREDICATE] = { "expected a predicate expression for the `elsif` statement", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_CONDITIONAL_IF_PREDICATE] = { "expected a predicate expression for the `if` statement", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_CONDITIONAL_PREDICATE_TERM] = { "expected `then` or `;` or '\\n'", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_CONDITIONAL_TERM] = { "expected an `end` to close the conditional clause", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_CONDITIONAL_TERM_ELSE] = { "expected an `end` to close the `else` clause", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_CONDITIONAL_UNLESS_PREDICATE] = { "expected a predicate expression for the `unless` statement", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_CONDITIONAL_UNTIL_PREDICATE] = { "expected a predicate expression for the `until` statement", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_CONDITIONAL_WHILE_PREDICATE] = { "expected a predicate expression for the `while` statement", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_CONSTANT_PATH_COLON_COLON_CONSTANT] = { "expected a constant after the `::` operator", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_DEF_ENDLESS] = { "could not parse the endless method body", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_DEF_ENDLESS_SETTER] = { "invalid method name; a setter method cannot be defined in an endless method definition", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_DEF_NAME] = { "expected a method name", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_DEF_NAME_AFTER_RECEIVER] = { "expected a method name after the receiver", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_DEF_PARAMS_TERM] = { "expected a delimiter to close the parameters", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_DEF_PARAMS_TERM_PAREN] = { "expected a `)` to close the parameters", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_DEF_RECEIVER] = { "expected a receiver for the method definition", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_DEF_RECEIVER_TERM] = { "expected a `.` or `::` after the receiver in a method definition", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_DEF_TERM] = { "expected an `end` to close the `def` statement", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_DEFINED_EXPRESSION] = { "expected an expression after `defined?`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EMBDOC_TERM] = { "could not find a terminator for the embedded document", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EMBEXPR_END] = { "expected a `}` to close the embedded expression", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EMBVAR_INVALID] = { "invalid embedded variable", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_END_UPCASE_BRACE] = { "expected a `{` after `END`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_END_UPCASE_TERM] = { "expected a `}` to close the `END` statement", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ESCAPE_INVALID_CONTROL] = { "invalid control escape sequence", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ESCAPE_INVALID_CONTROL_REPEAT] = { "invalid control escape sequence; control cannot be repeated", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ESCAPE_INVALID_HEXADECIMAL] = { "invalid hexadecimal escape sequence", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ESCAPE_INVALID_META] = { "invalid meta escape sequence", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ESCAPE_INVALID_META_REPEAT] = { "invalid meta escape sequence; meta cannot be repeated", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ESCAPE_INVALID_UNICODE] = { "invalid Unicode escape sequence", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ESCAPE_INVALID_UNICODE_CM_FLAGS] = { "invalid Unicode escape sequence; Unicode cannot be combined with control or meta flags", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ESCAPE_INVALID_UNICODE_LITERAL] = { "invalid Unicode escape sequence; multiple codepoints are not allowed in a character literal", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ESCAPE_INVALID_UNICODE_LONG] = { "invalid Unicode escape sequence; maximum length is 6 digits", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ESCAPE_INVALID_UNICODE_TERM] = { "invalid Unicode escape sequence; needs closing `}`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EXPECT_ARGUMENT] = { "expected an argument", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EXPECT_EOL_AFTER_STATEMENT] = { "unexpected %s, expecting end-of-input", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EXPECT_EXPRESSION_AFTER_AMPAMPEQ] = { "expected an expression after `&&=`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EXPECT_EXPRESSION_AFTER_PIPEPIPEEQ] = { "expected an expression after `||=`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EXPECT_EXPRESSION_AFTER_COMMA] = { "expected an expression after `,`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EXPECT_EXPRESSION_AFTER_EQUAL] = { "expected an expression after `=`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EXPECT_EXPRESSION_AFTER_LESS_LESS] = { "expected an expression after `<<`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EXPECT_EXPRESSION_AFTER_LPAREN] = { "expected an expression after `(`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EXPECT_EXPRESSION_AFTER_OPERATOR] = { "expected an expression after the operator", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EXPECT_EXPRESSION_AFTER_SPLAT] = { "expected an expression after `*` splat in an argument", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EXPECT_EXPRESSION_AFTER_SPLAT_HASH] = { "expected an expression after `**` in a hash", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EXPECT_EXPRESSION_AFTER_STAR] = { "expected an expression after `*`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EXPECT_IDENT_REQ_PARAMETER] = { "expected an identifier for the required parameter", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EXPECT_LPAREN_REQ_PARAMETER] = { "expected a `(` to start a required parameter", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EXPECT_RBRACKET] = { "expected a matching `]`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EXPECT_RPAREN] = { "expected a matching `)`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EXPECT_RPAREN_AFTER_MULTI] = { "expected a `)` after multiple assignment", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EXPECT_RPAREN_REQ_PARAMETER] = { "expected a `)` to end a required parameter", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EXPECT_STRING_CONTENT] = { "expected string content after opening string delimiter", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EXPECT_WHEN_DELIMITER] = { "expected a delimiter after the predicates of a `when` clause", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EXPRESSION_BARE_HASH] = { "unexpected bare hash in expression", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_FLOAT_PARSE] = { "could not parse the float '%.*s'", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_FOR_COLLECTION] = { "expected a collection after the `in` in a `for` statement", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_FOR_INDEX] = { "expected an index after `for`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_FOR_IN] = { "expected an `in` after the index in a `for` statement", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_FOR_TERM] = { "expected an `end` to close the `for` loop", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_HASH_EXPRESSION_AFTER_LABEL] = { "expected an expression after the label in a hash", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_HASH_KEY] = { "unexpected %s, expecting '}' or a key in the hash literal", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_HASH_ROCKET] = { "expected a `=>` between the hash key and value", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_HASH_TERM] = { "expected a `}` to close the hash literal", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_HASH_VALUE] = { "expected a value in the hash literal", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_HEREDOC_TERM] = { "could not find a terminator for the heredoc", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_INCOMPLETE_QUESTION_MARK] = { "incomplete expression at `?`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_INCOMPLETE_VARIABLE_CLASS_3_3_0] = { "`%.*s' is not allowed as a class variable name", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_INCOMPLETE_VARIABLE_CLASS] = { "'%.*s' is not allowed as a class variable name", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_INCOMPLETE_VARIABLE_INSTANCE_3_3_0] = { "`%.*s' is not allowed as an instance variable name", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_INCOMPLETE_VARIABLE_INSTANCE] = { "'%.*s' is not allowed as an instance variable name", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_INVALID_FLOAT_EXPONENT] = { "invalid exponent", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_INVALID_NUMBER_BINARY] = { "invalid binary number", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_INVALID_NUMBER_DECIMAL] = { "invalid decimal number", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_INVALID_NUMBER_HEXADECIMAL] = { "invalid hexadecimal number", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_INVALID_NUMBER_OCTAL] = { "invalid octal number", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_INVALID_NUMBER_UNDERSCORE] = { "invalid underscore placement in number", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_INVALID_CHARACTER] = { "invalid character 0x%X", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_INVALID_MULTIBYTE_CHAR] = { "invalid multibyte char (%s)", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_INVALID_MULTIBYTE_CHARACTER] = { "invalid multibyte character 0x%X", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_INVALID_MULTIBYTE_ESCAPE] = { "invalid multibyte escape: /%.*s/", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_INVALID_PRINTABLE_CHARACTER] = { "invalid character `%c`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_INVALID_PERCENT] = { "invalid `%` token", PM_ERROR_LEVEL_SYNTAX }, // TODO WHAT? + [PM_ERR_INVALID_VARIABLE_GLOBAL_3_3_0] = { "`%.*s' is not allowed as a global variable name", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_INVALID_VARIABLE_GLOBAL] = { "'%.*s' is not allowed as a global variable name", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_IT_NOT_ALLOWED_NUMBERED] = { "`it` is not allowed when an numbered parameter is defined", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_IT_NOT_ALLOWED_ORDINARY] = { "`it` is not allowed when an ordinary parameter is defined", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_LAMBDA_OPEN] = { "expected a `do` keyword or a `{` to open the lambda block", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_LAMBDA_TERM_BRACE] = { "expected a lambda block beginning with `{` to end with `}`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_LAMBDA_TERM_END] = { "expected a lambda block beginning with `do` to end with `end`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_LIST_I_LOWER_ELEMENT] = { "expected a symbol in a `%i` list", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_LIST_I_LOWER_TERM] = { "expected a closing delimiter for the `%i` list", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_LIST_I_UPPER_ELEMENT] = { "expected a symbol in a `%I` list", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_LIST_I_UPPER_TERM] = { "expected a closing delimiter for the `%I` list", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_LIST_W_LOWER_ELEMENT] = { "expected a string in a `%w` list", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_LIST_W_LOWER_TERM] = { "expected a closing delimiter for the `%w` list", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_LIST_W_UPPER_ELEMENT] = { "expected a string in a `%W` list", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_LIST_W_UPPER_TERM] = { "expected a closing delimiter for the `%W` list", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_MALLOC_FAILED] = { "failed to allocate memory", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_MIXED_ENCODING] = { "UTF-8 mixed within %s source", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_MODULE_IN_METHOD] = { "unexpected module definition in a method definition", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_MODULE_NAME] = { "expected a constant name after `module`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_MODULE_TERM] = { "expected an `end` to close the `module` statement", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_MULTI_ASSIGN_MULTI_SPLATS] = { "multiple splats in multiple assignment", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_MULTI_ASSIGN_UNEXPECTED_REST] = { "unexpected '%.*s' resulting in multiple splats in multiple assignment", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_NOT_EXPRESSION] = { "expected an expression after `not`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_NO_LOCAL_VARIABLE] = { "%.*s: no such local variable", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_NUMBER_LITERAL_UNDERSCORE] = { "number literal ending with a `_`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_NUMBERED_PARAMETER_IT] = { "numbered parameters are not allowed when an 'it' parameter is defined", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_NUMBERED_PARAMETER_ORDINARY] = { "numbered parameters are not allowed when an ordinary parameter is defined", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_NUMBERED_PARAMETER_OUTER_SCOPE] = { "numbered parameter is already used in outer scope", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_OPERATOR_MULTI_ASSIGN] = { "unexpected operator for a multiple assignment", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_OPERATOR_WRITE_ARGUMENTS] = { "unexpected operator after a call with arguments", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_OPERATOR_WRITE_BLOCK] = { "unexpected operator after a call with a block", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PARAMETER_ASSOC_SPLAT_MULTI] = { "unexpected multiple `**` splat parameters", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PARAMETER_BLOCK_MULTI] = { "multiple block parameters; only one block is allowed", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PARAMETER_CIRCULAR] = { "parameter default value references itself", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PARAMETER_METHOD_NAME] = { "unexpected name for a parameter", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PARAMETER_NAME_REPEAT] = { "repeated parameter name", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PARAMETER_NO_DEFAULT] = { "expected a default value for the parameter", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PARAMETER_NO_DEFAULT_KW] = { "expected a default value for the keyword parameter", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PARAMETER_NUMBERED_RESERVED] = { "%.2s is reserved for numbered parameters", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PARAMETER_ORDER] = { "unexpected parameter order", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PARAMETER_SPLAT_MULTI] = { "unexpected multiple `*` splat parameters", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PARAMETER_STAR] = { "unexpected parameter `*`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PARAMETER_UNEXPECTED_FWD] = { "unexpected `...` in parameters", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PARAMETER_WILD_LOOSE_COMMA] = { "unexpected `,` in parameters", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PATTERN_EXPRESSION_AFTER_BRACKET] = { "expected a pattern expression after the `[` operator", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PATTERN_EXPRESSION_AFTER_COMMA] = { "expected a pattern expression after `,`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PATTERN_EXPRESSION_AFTER_HROCKET] = { "expected a pattern expression after `=>`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PATTERN_EXPRESSION_AFTER_IN] = { "expected a pattern expression after the `in` keyword", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PATTERN_EXPRESSION_AFTER_KEY] = { "expected a pattern expression after the key", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PATTERN_EXPRESSION_AFTER_PAREN] = { "expected a pattern expression after the `(` operator", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PATTERN_EXPRESSION_AFTER_PIN] = { "expected a pattern expression after the `^` pin operator", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PATTERN_EXPRESSION_AFTER_PIPE] = { "expected a pattern expression after the `|` operator", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PATTERN_EXPRESSION_AFTER_RANGE] = { "expected a pattern expression after the range operator", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PATTERN_EXPRESSION_AFTER_REST] = { "unexpected pattern expression after the `**` expression", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PATTERN_HASH_KEY] = { "expected a key in the hash pattern", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PATTERN_HASH_KEY_LABEL] = { "expected a label as the key in the hash pattern", PM_ERROR_LEVEL_SYNTAX }, // TODO // THIS // AND // ABOVE // IS WEIRD + [PM_ERR_PATTERN_IDENT_AFTER_HROCKET] = { "expected an identifier after the `=>` operator", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PATTERN_LABEL_AFTER_COMMA] = { "expected a label after the `,` in the hash pattern", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PATTERN_REST] = { "unexpected rest pattern", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PATTERN_TERM_BRACE] = { "expected a `}` to close the pattern expression", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PATTERN_TERM_BRACKET] = { "expected a `]` to close the pattern expression", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PATTERN_TERM_PAREN] = { "expected a `)` to close the pattern expression", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PIPEPIPEEQ_MULTI_ASSIGN] = { "unexpected `||=` in a multiple assignment", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_REGEXP_ENCODING_OPTION_MISMATCH] = { "regexp encoding option '%c' differs from source encoding '%s'", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_REGEXP_INCOMPAT_CHAR_ENCODING] = { "incompatible character encoding: /%.*s/", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_REGEXP_NON_ESCAPED_MBC] = { "/.../n has a non escaped non ASCII character in non ASCII-8BIT script: /%.*s/", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_REGEXP_INVALID_UNICODE_RANGE] = { "invalid Unicode range: /%.*s/", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_REGEXP_UNKNOWN_OPTIONS] = { "unknown regexp %s: %.*s", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_REGEXP_TERM] = { "expected a closing delimiter for the regular expression", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_REGEXP_UTF8_CHAR_NON_UTF8_REGEXP] = { "UTF-8 character in non UTF-8 regexp: /%s/", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_RESCUE_EXPRESSION] = { "expected a rescued expression", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_RESCUE_MODIFIER_VALUE] = { "expected a value after the `rescue` modifier", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_RESCUE_TERM] = { "expected a closing delimiter for the `rescue` clause", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_RESCUE_VARIABLE] = { "expected an exception variable after `=>` in a rescue statement", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_RETURN_INVALID] = { "invalid `return` in a class or module body", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_SINGLETON_FOR_LITERALS] = { "cannot define singleton method for literals", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_STATEMENT_ALIAS] = { "unexpected an `alias` at a non-statement position", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_STATEMENT_POSTEXE_END] = { "unexpected an `END` at a non-statement position", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_STATEMENT_PREEXE_BEGIN] = { "unexpected a `BEGIN` at a non-statement position", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_STATEMENT_UNDEF] = { "unexpected an `undef` at a non-statement position", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_STRING_CONCATENATION] = { "expected a string for concatenation", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_STRING_INTERPOLATED_TERM] = { "expected a closing delimiter for the interpolated string", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_STRING_LITERAL_EOF] = { "unterminated string meets end of file", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_STRING_LITERAL_TERM] = { "unexpected %s, expected a string literal terminator", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_SYMBOL_INVALID] = { "invalid symbol", PM_ERROR_LEVEL_SYNTAX }, // TODO expected symbol? prism.c ~9719 + [PM_ERR_SYMBOL_TERM_DYNAMIC] = { "expected a closing delimiter for the dynamic symbol", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_SYMBOL_TERM_INTERPOLATED] = { "expected a closing delimiter for the interpolated symbol", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_TERNARY_COLON] = { "expected a `:` after the true expression of a ternary operator", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_TERNARY_EXPRESSION_FALSE] = { "expected an expression after `:` in the ternary operator", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_TERNARY_EXPRESSION_TRUE] = { "expected an expression after `?` in the ternary operator", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_UNDEF_ARGUMENT] = { "invalid argument being passed to `undef`; expected a bare word, constant, or symbol argument", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_UNARY_RECEIVER] = { "unexpected %s, expected a receiver for unary `%c`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_UNEXPECTED_TOKEN_CLOSE_CONTEXT] = { "unexpected %s, assuming it is closing the parent %s", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_UNEXPECTED_TOKEN_IGNORE] = { "unexpected %s, ignoring it", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_UNTIL_TERM] = { "expected an `end` to close the `until` statement", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_VOID_EXPRESSION] = { "unexpected void value expression", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_WHILE_TERM] = { "expected an `end` to close the `while` statement", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_WRITE_TARGET_IN_METHOD] = { "dynamic constant assignment", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_WRITE_TARGET_READONLY] = { "Can't set variable %.*s", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_WRITE_TARGET_UNEXPECTED] = { "unexpected write target", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_XSTRING_TERM] = { "expected a closing delimiter for the `%x` or backtick string", PM_ERROR_LEVEL_SYNTAX }, // Warnings [PM_WARN_AMBIGUOUS_FIRST_ARGUMENT_MINUS] = { "ambiguous first argument; put parentheses or a space even after `-` operator", PM_WARNING_LEVEL_VERBOSE }, @@ -335,6 +339,7 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_WARN_KEYWORD_EOL] = { "`%.*s` at the end of line without an expression", PM_WARNING_LEVEL_VERBOSE }, [PM_WARN_LITERAL_IN_CONDITION_DEFAULT] = { "%sliteral in %s", PM_WARNING_LEVEL_DEFAULT }, [PM_WARN_LITERAL_IN_CONDITION_VERBOSE] = { "%sliteral in %s", PM_WARNING_LEVEL_VERBOSE }, + [PM_WARN_SHEBANG_CARRIAGE_RETURN] = { "shebang line ending with \\r may cause problems", PM_WARNING_LEVEL_DEFAULT }, [PM_WARN_UNEXPECTED_CARRIAGE_RETURN] = { "encountered \\r in middle of line, treated as a mere space", PM_WARNING_LEVEL_DEFAULT } }; diff --git a/prism/util/pm_newline_list.c b/prism/util/pm_newline_list.c index f9dff4c1666187..ce07ce8c8e4644 100644 --- a/prism/util/pm_newline_list.c +++ b/prism/util/pm_newline_list.c @@ -19,6 +19,14 @@ pm_newline_list_init(pm_newline_list_t *list, const uint8_t *start, size_t capac return true; } +/** + * Clear out the newlines that have been appended to the list. + */ +void +pm_newline_list_clear(pm_newline_list_t *list) { + list->size = 1; +} + /** * Append a new offset to the newline list. Returns true if the reallocation of * the offsets succeeds (if one was necessary), otherwise returns false. diff --git a/prism/util/pm_newline_list.h b/prism/util/pm_newline_list.h index 612ee35d3f8fd6..11abe237a6e511 100644 --- a/prism/util/pm_newline_list.h +++ b/prism/util/pm_newline_list.h @@ -61,6 +61,12 @@ typedef struct { */ bool pm_newline_list_init(pm_newline_list_t *list, const uint8_t *start, size_t capacity); +/** + * Clear out the newlines that have been appended to the list. + */ +void +pm_newline_list_clear(pm_newline_list_t *list); + /** * Append a new offset to the newline list. Returns true if the reallocation of * the offsets succeeds (if one was necessary), otherwise returns false. diff --git a/test/prism/command_line_test.rb b/test/prism/command_line_test.rb index 57ab0dee45e944..96ceb2da38f19c 100644 --- a/test/prism/command_line_test.rb +++ b/test/prism/command_line_test.rb @@ -65,5 +65,28 @@ def test_command_line_e result = Prism.parse("1 if 2..3", command_line: "e") assert_equal 0, result.warnings.length end + + def test_command_line_x_implicit + result = Prism.parse(<<~RUBY) + #!/bin/bash + exit 1 + + #!/usr/bin/env ruby + 1 + RUBY + + assert_kind_of IntegerNode, result.value.statements.body.first + end + + def test_command_line_x_explicit + result = Prism.parse(<<~RUBY, command_line: "x") + exit 1 + + #!/usr/bin/env ruby + 1 + RUBY + + assert_kind_of IntegerNode, result.value.statements.body.first + end end end diff --git a/test/prism/ruby_api_test.rb b/test/prism/ruby_api_test.rb index 4153a69ad75644..49296117bf5488 100644 --- a/test/prism/ruby_api_test.rb +++ b/test/prism/ruby_api_test.rb @@ -233,21 +233,6 @@ def test_integer_base_flags assert_equal 16, base[parse_expression("0x1")] end - def test_offset - source = <<~RUBY - #!/bin/sh - - echo "foo" - exit 0 - - #!/usr/bin/env ruby - - puts "bar" - RUBY - - assert Prism.parse_success?(source, offset: 30) - end - private def parse_expression(source) From f7c5e11d894c6c6a965a9fa98ff9519635b6db2b Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 28 Mar 2024 10:07:04 -0400 Subject: [PATCH 135/211] [PRISM] Use new -x prism API --- prism_compile.c | 27 +++++++++++++++++++++------ spec/prism.mspec | 4 ---- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/prism_compile.c b/prism_compile.c index 6e27d017295d30..5a5b1e7d61ec93 100644 --- a/prism_compile.c +++ b/prism_compile.c @@ -8390,10 +8390,11 @@ pm_parse_process_error(const pm_parse_result_t *result) const pm_string_t *filepath = &parser->filepath; for (const pm_diagnostic_t *error = head; error != NULL; error = (const pm_diagnostic_t *) error->node.next) { - // Any errors with the level PM_ERROR_LEVEL_ARGUMENT effectively take - // over as the only argument that gets raised. This is to allow priority - // messages that should be handled before anything else. - if (error->level == PM_ERROR_LEVEL_ARGUMENT) { + // Any errors with the level that is not PM_ERROR_LEVEL_SYNTAX + // effectively take over as the only argument that gets raised. This is + // to allow priority messages that should be handled before anything + // else. + if (error->level != PM_ERROR_LEVEL_SYNTAX) { int32_t line_number = (int32_t) pm_location_line_number(parser, &error->location); pm_buffer_append_format( @@ -8414,10 +8415,24 @@ pm_parse_process_error(const pm_parse_result_t *result) pm_parser_errors_format(parser, &error_list, &buffer, rb_stderr_tty_p(), false); } - VALUE arg_error = rb_exc_new(rb_eArgError, pm_buffer_value(&buffer), pm_buffer_length(&buffer)); + VALUE class; + switch (error->level) { + case PM_ERROR_LEVEL_ARGUMENT: + class = rb_eArgError; + break; + case PM_ERROR_LEVEL_LOAD: + class = rb_eLoadError; + break; + default: + class = rb_eSyntaxError; + RUBY_ASSERT(false && "unexpected error level"); + break; + } + + VALUE value = rb_exc_new(class, pm_buffer_value(&buffer), pm_buffer_length(&buffer)); pm_buffer_free(&buffer); - return arg_error; + return value; } // It is implicitly assumed that the error messages will be encodeable diff --git a/spec/prism.mspec b/spec/prism.mspec index 252432c94354b9..dd499dbc294916 100644 --- a/spec/prism.mspec +++ b/spec/prism.mspec @@ -1,10 +1,6 @@ # frozen_string_literal: true ## Command line -MSpec.register(:exclude, "The -S command line option runs launcher found in PATH, but only code after the first /#!.*ruby.*/-ish line in target file") -MSpec.register(:exclude, "The -x command line option runs code after the first /#!.*ruby.*/-ish line in target file") -MSpec.register(:exclude, "The -x command line option fails when /#!.*ruby.*/-ish line in target file is not found") -MSpec.register(:exclude, "The -x command line option behaves as -x was set when non-ruby shebang is encountered on first line") MSpec.register(:exclude, "The --debug flag produces debugging info on attempted frozen string modification") ## Language From 35ff302893dfb1efd03ea17e76b9a09e2d3377a2 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 28 Mar 2024 10:30:52 -0400 Subject: [PATCH 136/211] [ruby/prism] Various cleanup with new -x option https://github.com/ruby/prism/commit/020756fb11 --- prism/util/pm_newline_list.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/prism/util/pm_newline_list.h b/prism/util/pm_newline_list.h index 11abe237a6e511..7ae9b6b3da0ac5 100644 --- a/prism/util/pm_newline_list.h +++ b/prism/util/pm_newline_list.h @@ -63,6 +63,8 @@ bool pm_newline_list_init(pm_newline_list_t *list, const uint8_t *start, size_t /** * Clear out the newlines that have been appended to the list. + * + * @param list The list to clear. */ void pm_newline_list_clear(pm_newline_list_t *list); From d583616f32795de61ecd4efd60346c278873da7f Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 28 Mar 2024 10:47:04 -0400 Subject: [PATCH 137/211] [ruby/prism] Ensure deserialization works with errors+warnings>256 https://github.com/ruby/prism/commit/f540e830b5 --- prism/extension.c | 3 +++ prism/templates/lib/prism/serialize.rb.erb | 8 +++++--- prism/templates/src/serialize.c.erb | 2 +- test/prism/command_line_test.rb | 19 +++++++++++++++++++ 4 files changed, 28 insertions(+), 4 deletions(-) diff --git a/prism/extension.c b/prism/extension.c index d19a004beb5ab3..27e799a8da645e 100644 --- a/prism/extension.c +++ b/prism/extension.c @@ -451,6 +451,9 @@ parser_errors(pm_parser_t *parser, rb_encoding *encoding, VALUE source) { case PM_ERROR_LEVEL_ARGUMENT: level = ID2SYM(rb_intern("argument")); break; + case PM_ERROR_LEVEL_LOAD: + level = ID2SYM(rb_intern("load")); + break; default: rb_raise(rb_eRuntimeError, "Unknown level: %" PRIu8, error->level); } diff --git a/prism/templates/lib/prism/serialize.rb.erb b/prism/templates/lib/prism/serialize.rb.erb index 1a723f64afcd3f..ac7ab4fff3c65d 100644 --- a/prism/templates/lib/prism/serialize.rb.erb +++ b/prism/templates/lib/prism/serialize.rb.erb @@ -131,8 +131,8 @@ module Prism comments = load_comments magic_comments = Array.new(load_varuint) { MagicComment.new(load_location_object, load_location_object) } data_loc = load_optional_location_object - errors = Array.new(load_varuint) { ParseError.new(DIAGNOSTIC_TYPES[io.getbyte], load_embedded_string, load_location_object, load_error_level) } - warnings = Array.new(load_varuint) { ParseWarning.new(DIAGNOSTIC_TYPES[io.getbyte], load_embedded_string, load_location_object, load_warning_level) } + errors = Array.new(load_varuint) { ParseError.new(DIAGNOSTIC_TYPES[load_varuint], load_embedded_string, load_location_object, load_error_level) } + warnings = Array.new(load_varuint) { ParseWarning.new(DIAGNOSTIC_TYPES[load_varuint], load_embedded_string, load_location_object, load_warning_level) } [comments, magic_comments, data_loc, errors, warnings] end @@ -296,9 +296,11 @@ module Prism case level when 0 - :fatal + :syntax when 1 :argument + when 2 + :load else raise "Unknown level: #{level}" end diff --git a/prism/templates/src/serialize.c.erb b/prism/templates/src/serialize.c.erb index 94b976645dd1d1..97101e36d52d91 100644 --- a/prism/templates/src/serialize.c.erb +++ b/prism/templates/src/serialize.c.erb @@ -220,7 +220,7 @@ pm_serialize_data_loc(const pm_parser_t *parser, pm_buffer_t *buffer) { static void pm_serialize_diagnostic(pm_parser_t *parser, pm_diagnostic_t *diagnostic, pm_buffer_t *buffer) { // serialize the type - pm_buffer_append_byte(buffer, (uint8_t) diagnostic->diag_id); + pm_buffer_append_varuint(buffer, (uint32_t) diagnostic->diag_id); // serialize message size_t message_length = strlen(diagnostic->message); diff --git a/test/prism/command_line_test.rb b/test/prism/command_line_test.rb index 96ceb2da38f19c..4b04c36f3aa5e2 100644 --- a/test/prism/command_line_test.rb +++ b/test/prism/command_line_test.rb @@ -88,5 +88,24 @@ def test_command_line_x_explicit assert_kind_of IntegerNode, result.value.statements.body.first end + + def test_command_line_x_implicit_fail + result = Prism.parse(<<~RUBY) + #!/bin/bash + exit 1 + RUBY + + assert_equal 1, result.errors.length + assert_equal :load, result.errors.first.level + end + + def test_command_line_x_explicit_fail + result = Prism.parse(<<~RUBY, command_line: "x") + exit 1 + RUBY + + assert_equal 1, result.errors.length + assert_equal :load, result.errors.first.level + end end end From 86e0d83a32508f2aa42ec301238e0892caf980c0 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 28 Mar 2024 11:05:47 -0400 Subject: [PATCH 138/211] [PRISM] Simplify raising load errors --- prism/prism.c | 8 ++++--- prism_compile.c | 55 +++++++++++++++++++++++-------------------------- 2 files changed, 31 insertions(+), 32 deletions(-) diff --git a/prism/prism.c b/prism/prism.c index e33d3e1d3c8760..929255ef44a571 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -19319,7 +19319,9 @@ pm_parser_init(pm_parser_t *parser, const uint8_t *source, size_t size, const pm // Here we're going to find the first shebang that includes "ruby" and start // parsing from there. if (search_shebang) { - bool found = false; + // If a shebang that includes "ruby" is not found, then we're going to a + // a load error to the list of errors on the parser. + bool found_shebang = false; // This is going to point to the start of each line as we check it. // We'll maintain a moving window looking at each line at they come. @@ -19342,14 +19344,14 @@ pm_parser_init(pm_parser_t *parser, const uint8_t *source, size_t size, const pm } if (pm_strnstr((const char *) cursor, "ruby", length) != NULL) { - found = true; + found_shebang = true; parser->encoding_comment_start = newline + 1; break; } } } - if (found) { + if (found_shebang) { parser->previous = (pm_token_t) { .type = PM_TOKEN_EOF, .start = cursor, .end = cursor }; parser->current = (pm_token_t) { .type = PM_TOKEN_EOF, .start = cursor, .end = cursor }; } else { diff --git a/prism_compile.c b/prism_compile.c index 5a5b1e7d61ec93..e63933926b324b 100644 --- a/prism_compile.c +++ b/prism_compile.c @@ -8390,11 +8390,21 @@ pm_parse_process_error(const pm_parse_result_t *result) const pm_string_t *filepath = &parser->filepath; for (const pm_diagnostic_t *error = head; error != NULL; error = (const pm_diagnostic_t *) error->node.next) { - // Any errors with the level that is not PM_ERROR_LEVEL_SYNTAX - // effectively take over as the only argument that gets raised. This is - // to allow priority messages that should be handled before anything - // else. - if (error->level != PM_ERROR_LEVEL_SYNTAX) { + switch (error->level) { + case PM_ERROR_LEVEL_SYNTAX: + // It is implicitly assumed that the error messages will be + // encodeable as UTF-8. Because of this, we can't include source + // examples that contain invalid byte sequences. So if any source + // examples include invalid UTF-8 byte sequences, we will skip + // showing source examples entirely. + if (valid_utf8 && !pm_parse_process_error_utf8_p(parser, &error->location)) { + valid_utf8 = false; + } + break; + case PM_ERROR_LEVEL_ARGUMENT: { + // Any errors with the level PM_ERROR_LEVEL_ARGUMENT take over as + // the only argument that gets raised. This is to allow priority + // messages that should be handled before anything else. int32_t line_number = (int32_t) pm_location_line_number(parser, &error->location); pm_buffer_append_format( @@ -8415,33 +8425,20 @@ pm_parse_process_error(const pm_parse_result_t *result) pm_parser_errors_format(parser, &error_list, &buffer, rb_stderr_tty_p(), false); } - VALUE class; - switch (error->level) { - case PM_ERROR_LEVEL_ARGUMENT: - class = rb_eArgError; - break; - case PM_ERROR_LEVEL_LOAD: - class = rb_eLoadError; - break; - default: - class = rb_eSyntaxError; - RUBY_ASSERT(false && "unexpected error level"); - break; - } - - VALUE value = rb_exc_new(class, pm_buffer_value(&buffer), pm_buffer_length(&buffer)); + VALUE value = rb_exc_new(rb_eArgError, pm_buffer_value(&buffer), pm_buffer_length(&buffer)); pm_buffer_free(&buffer); return value; - } - - // It is implicitly assumed that the error messages will be encodeable - // as UTF-8. Because of this, we can't include source examples that - // contain invalid byte sequences. So if any source examples include - // invalid UTF-8 byte sequences, we will skip showing source examples - // entirely. - if (valid_utf8 && !pm_parse_process_error_utf8_p(parser, &error->location)) { - valid_utf8 = false; + } + case PM_ERROR_LEVEL_LOAD: { + // Load errors are much simpler, because they don't include any of + // the source in them. We create the error directly from the + // message. + VALUE message = rb_enc_str_new_cstr(error->message, rb_locale_encoding()); + VALUE value = rb_exc_new3(rb_eLoadError, message); + rb_ivar_set(value, rb_intern_const("@path"), Qnil); + return value; + } } } From 3e9c6842363303d01770413a3f8c28adc1d43848 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 28 Mar 2024 12:28:06 -0400 Subject: [PATCH 139/211] [PRISM] Allow space before encoding comment --- prism/prism.c | 5 +++++ spec/prism.mspec | 2 -- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/prism/prism.c b/prism/prism.c index 929255ef44a571..9d9aec00d53697 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -19359,6 +19359,11 @@ pm_parser_init(pm_parser_t *parser, const uint8_t *source, size_t size, const pm pm_newline_list_clear(&parser->newline_list); } } + + // The encoding comment can start after any amount of inline whitespace, so + // here we'll advance it to the first non-inline-whitespace character so + // that it is ready for future comparisons. + parser->encoding_comment_start += pm_strspn_inline_whitespace(parser->encoding_comment_start, parser->end - parser->encoding_comment_start); } /** diff --git a/spec/prism.mspec b/spec/prism.mspec index dd499dbc294916..9f20beff0a27f3 100644 --- a/spec/prism.mspec +++ b/spec/prism.mspec @@ -29,8 +29,6 @@ MSpec.register(:exclude, "A Symbol literal raises an SyntaxError at parse time w ## Core MSpec.register(:exclude, "IO.popen with a leading Array argument accepts a trailing Hash of Process.exec options") MSpec.register(:exclude, "IO.popen with a leading Array argument accepts an IO mode argument following the Array") -MSpec.register(:exclude, "Kernel#eval with a magic encoding comment allows spaces before the magic encoding comment") -MSpec.register(:exclude, "Kernel#eval with a magic encoding comment allows a shebang line and some spaces before the magic encoding comment") MSpec.register(:exclude, "TracePoint#eval_script is the evald source code") MSpec.register(:exclude, "TracePoint#event returns the type of event") MSpec.register(:exclude, "TracePoint#inspect returns a String showing the event, method, path and line for a :return event") From a8f902ea8ef4051e0dd761bbb220fd721550a4ff Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 28 Mar 2024 14:43:25 -0400 Subject: [PATCH 140/211] [PRISM] Add debug info for frozen strings --- prism_compile.c | 39 +++++++++++++++++++++++++++++++++------ spec/prism.mspec | 3 --- 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/prism_compile.c b/prism_compile.c index e63933926b324b..81b1e7dd11d42c 100644 --- a/prism_compile.c +++ b/prism_compile.c @@ -543,6 +543,23 @@ pm_source_file_value(const pm_source_file_node_t *node, const pm_scope_node_t *s } } +/** + * Return a static literal string, optionally with attached debugging + * information. + */ +static VALUE +pm_static_literal_string(rb_iseq_t *iseq, VALUE string, int line_number) +{ + if (ISEQ_COMPILE_DATA(iseq)->option->debug_frozen_string_literal || RTEST(ruby_debug)) { + VALUE debug_info = rb_ary_new_from_args(2, rb_iseq_path(iseq), INT2FIX(line_number)); + rb_ivar_set(string, id_debug_created_info, rb_obj_freeze(debug_info)); + return rb_str_freeze(string); + } + else { + return rb_fstring(string); + } +} + /** * Certain nodes can be compiled literally. This function returns the literal * value described by the given node. For example, an array node with all static @@ -603,8 +620,11 @@ pm_static_literal_value(rb_iseq_t *iseq, const pm_node_t *node, const pm_scope_n const pm_interpolated_regular_expression_node_t *cast = (const pm_interpolated_regular_expression_node_t *) node; return parse_regexp_concat(iseq, scope_node, (const pm_node_t *) cast, &cast->parts); } - case PM_INTERPOLATED_STRING_NODE: - return pm_static_literal_concat(&((const pm_interpolated_string_node_t *) node)->parts, scope_node, true); + case PM_INTERPOLATED_STRING_NODE: { + VALUE string = pm_static_literal_concat(&((const pm_interpolated_string_node_t *) node)->parts, scope_node, false); + int line_number = pm_node_line_number(scope_node->parser, node); + return pm_static_literal_string(iseq, string, line_number); + } case PM_INTERPOLATED_SYMBOL_NODE: { const pm_interpolated_symbol_node_t *cast = (const pm_interpolated_symbol_node_t *) node; VALUE string = pm_static_literal_concat(&cast->parts, scope_node, true); @@ -631,8 +651,11 @@ pm_static_literal_value(rb_iseq_t *iseq, const pm_node_t *node, const pm_scope_n } case PM_SOURCE_LINE_NODE: return INT2FIX(pm_node_line_number(scope_node->parser, node)); - case PM_STRING_NODE: - return rb_fstring(parse_string_encoded(scope_node, node, &((pm_string_node_t *)node)->unescaped)); + case PM_STRING_NODE: { + VALUE string = parse_string_encoded(scope_node, node, &((const pm_string_node_t *) node)->unescaped); + int line_number = pm_node_line_number(scope_node->parser, node); + return pm_static_literal_string(iseq, string, line_number); + } case PM_SYMBOL_NODE: return ID2SYM(parse_string_symbol(scope_node, (const pm_symbol_node_t *) node)); case PM_TRUE_NODE: @@ -4308,7 +4331,10 @@ pm_compile_case_node_dispatch(rb_iseq_t *iseq, VALUE dispatch, const pm_node_t * break; case PM_STRING_NODE: { const pm_string_node_t *cast = (const pm_string_node_t *) node; - key = rb_fstring(parse_string_encoded(scope_node, node, &cast->unescaped)); + VALUE string = parse_string_encoded(scope_node, node, &cast->unescaped); + + int line_number = pm_node_line_number(scope_node->parser, node); + key = pm_static_literal_string(iseq, string, line_number); break; } default: @@ -8140,7 +8166,8 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, // ^^^^^ if (!popped) { const pm_string_node_t *cast = (const pm_string_node_t *) node; - VALUE value = rb_fstring(parse_string_encoded(scope_node, node, &cast->unescaped)); + VALUE value = parse_string_encoded(scope_node, node, &cast->unescaped); + value = pm_static_literal_string(iseq, value, location.line); if (PM_NODE_FLAG_P(node, PM_STRING_FLAGS_FROZEN)) { PUSH_INSN1(ret, location, putobject, value); diff --git a/spec/prism.mspec b/spec/prism.mspec index 9f20beff0a27f3..b1b0fa8c6bd958 100644 --- a/spec/prism.mspec +++ b/spec/prism.mspec @@ -1,8 +1,5 @@ # frozen_string_literal: true -## Command line -MSpec.register(:exclude, "The --debug flag produces debugging info on attempted frozen string modification") - ## Language MSpec.register(:exclude, "The BEGIN keyword runs multiple begins in FIFO order") MSpec.register(:exclude, "Executing break from within a block works when passing through a super call") From bb3cbdfe2fcbfc2b6c7ee8699d35fde838615c26 Mon Sep 17 00:00:00 2001 From: Maxime Chevalier-Boisvert Date: Thu, 28 Mar 2024 15:21:09 -0400 Subject: [PATCH 141/211] YJIT: add iseq_alloc_count to stats (#10398) * YJIT: add iseq_alloc_count to stats * Remove an empty line --------- Co-authored-by: Takashi Kokubun --- compile.c | 1 + yjit.h | 1 + yjit.rb | 1 + yjit/src/stats.rs | 7 ++++++- 4 files changed, 9 insertions(+), 1 deletion(-) diff --git a/compile.c b/compile.c index 5516f64f54621f..ab041e6e89d6f6 100644 --- a/compile.c +++ b/compile.c @@ -999,6 +999,7 @@ rb_iseq_translate_threaded_code(rb_iseq_t *iseq) #if USE_YJIT rb_yjit_live_iseq_count++; + rb_yjit_iseq_alloc_count++; #endif return COMPILE_OK; diff --git a/yjit.h b/yjit.h index 2f5317ad977b6e..dde9f750aaf72b 100644 --- a/yjit.h +++ b/yjit.h @@ -28,6 +28,7 @@ extern uint64_t rb_yjit_call_threshold; extern uint64_t rb_yjit_cold_threshold; extern uint64_t rb_yjit_live_iseq_count; +extern uint64_t rb_yjit_iseq_alloc_count; extern bool rb_yjit_enabled_p; void rb_yjit_incr_counter(const char *counter_name); void rb_yjit_invalidate_all_method_lookup_assumptions(void); diff --git a/yjit.rb b/yjit.rb index 50cb2483980fec..eedd00c358150b 100644 --- a/yjit.rb +++ b/yjit.rb @@ -346,6 +346,7 @@ def _print_stats(out: $stderr) # :nodoc: out.puts "bindings_set: " + format_number(13, stats[:binding_set]) out.puts "compilation_failure: " + format_number(13, compilation_failure) if compilation_failure != 0 out.puts "live_iseq_count: " + format_number(13, stats[:live_iseq_count]) + out.puts "iseq_alloc_count: " + format_number(13, stats[:iseq_alloc_count]) out.puts "compiled_iseq_entry: " + format_number(13, stats[:compiled_iseq_entry]) out.puts "cold_iseq_entry: " + format_number_pct(13, stats[:cold_iseq_entry], stats[:compiled_iseq_entry] + stats[:cold_iseq_entry]) out.puts "compiled_iseq_count: " + format_number(13, stats[:compiled_iseq_count]) diff --git a/yjit/src/stats.rs b/yjit/src/stats.rs index 621854d48793b2..fb4ed03bd5c4af 100644 --- a/yjit/src/stats.rs +++ b/yjit/src/stats.rs @@ -15,10 +15,14 @@ use crate::cruby::*; use crate::options::*; use crate::yjit::yjit_enabled_p; -/// A running total of how many ISeqs are in the system. +/// Running total of how many ISeqs are in the system. #[no_mangle] pub static mut rb_yjit_live_iseq_count: u64 = 0; +/// Monotonically increasing total of how many ISEQs were allocated +#[no_mangle] +pub static mut rb_yjit_iseq_alloc_count: u64 = 0; + /// A middleware to count Rust-allocated bytes as yjit_alloc_size. #[global_allocator] static GLOBAL_ALLOCATOR: StatsAlloc = StatsAlloc { alloc_size: AtomicUsize::new(0) }; @@ -748,6 +752,7 @@ fn rb_yjit_gen_stats_dict(context: bool) -> VALUE { hash_aset_usize!(hash, "vm_insns_count", rb_vm_insns_count as usize); hash_aset_usize!(hash, "live_iseq_count", rb_yjit_live_iseq_count as usize); + hash_aset_usize!(hash, "iseq_alloc_count", rb_yjit_iseq_alloc_count as usize); } // If we're not generating stats, put only default counters From 8780059c38319aa91452e726ca428ca1610e2d88 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 28 Mar 2024 15:16:26 -0400 Subject: [PATCH 142/211] [ruby/prism] Reject invalid capture groups (keywords) https://github.com/ruby/prism/commit/bb78d83e88 --- prism/prism.c | 110 +++++++++++++++++-- test/prism/fixtures/regex.txt | 4 + test/prism/snapshots/regex.txt | 187 +++++++++++++++++++++++---------- 3 files changed, 237 insertions(+), 64 deletions(-) diff --git a/prism/prism.c b/prism/prism.c index 9d9aec00d53697..1cfcf704bb87ef 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -1184,6 +1184,77 @@ token_is_setter_name(pm_token_t *token) { ); } +/** + * Returns true if the given local variable is a keyword. + */ +static bool +pm_local_is_keyword(const char *source, size_t length) { +#define KEYWORD(name) if (memcmp(source, name, length) == 0) return true + + switch (length) { + case 2: + switch (source[0]) { + case 'd': KEYWORD("do"); return false; + case 'i': KEYWORD("if"); KEYWORD("in"); return false; + case 'o': KEYWORD("or"); return false; + default: return false; + } + case 3: + switch (source[0]) { + case 'a': KEYWORD("and"); return false; + case 'd': KEYWORD("def"); return false; + case 'e': KEYWORD("end"); return false; + case 'f': KEYWORD("for"); return false; + case 'n': KEYWORD("nil"); KEYWORD("not"); return false; + default: return false; + } + case 4: + switch (source[0]) { + case 'c': KEYWORD("case"); return false; + case 'e': KEYWORD("else"); return false; + case 'n': KEYWORD("next"); return false; + case 'r': KEYWORD("redo"); return false; + case 's': KEYWORD("self"); return false; + case 't': KEYWORD("then"); KEYWORD("true"); return false; + case 'w': KEYWORD("when"); return false; + default: return false; + } + case 5: + switch (source[0]) { + case 'a': KEYWORD("alias"); return false; + case 'b': KEYWORD("begin"); KEYWORD("break"); return false; + case 'c': KEYWORD("class"); return false; + case 'e': KEYWORD("elsif"); return false; + case 'f': KEYWORD("false"); return false; + case 'r': KEYWORD("retry"); return false; + case 's': KEYWORD("super"); return false; + case 'u': KEYWORD("undef"); KEYWORD("until"); return false; + case 'w': KEYWORD("while"); return false; + case 'y': KEYWORD("yield"); return false; + default: return false; + } + case 6: + switch (source[0]) { + case 'e': KEYWORD("ensure"); return false; + case 'm': KEYWORD("module"); return false; + case 'r': KEYWORD("rescue"); KEYWORD("return"); return false; + case 'u': KEYWORD("unless"); return false; + default: return false; + } + case 8: + KEYWORD("__LINE__"); + KEYWORD("__FILE__"); + return false; + case 12: + KEYWORD("__ENCODING__"); + return false; + default: + return false; + } + +#undef KEYWORD +} + /******************************************************************************/ /* Node flag handling functions */ /******************************************************************************/ @@ -10576,19 +10647,19 @@ parser_lex(pm_parser_t *parser) { pm_token_type_t type = lex_identifier(parser, previous_command_start); - // If we've hit a __END__ and it was at the start of the line or the - // start of the file and it is followed by either a \n or a \r\n, then - // this is the last token of the file. + // If we've hit a __END__ and it was at the start of the + // line or the start of the file and it is followed by + // either a \n or a \r\n, then this is the last token of the + // file. if ( ((parser->current.end - parser->current.start) == 7) && current_token_starts_line(parser) && (memcmp(parser->current.start, "__END__", 7) == 0) && (parser->current.end == parser->end || match_eol(parser)) - ) - { - // Since we know we're about to add an __END__ comment, we know we - // need to add all of the newlines to get the correct column - // information for it. + ) { + // Since we know we're about to add an __END__ comment, + // we know we need to add all of the newlines to get the + // correct column information for it. const uint8_t *cursor = parser->current.end; while ((cursor = next_newline(cursor, parser->end - cursor)) != NULL) { pm_newline_list_append(&parser->newline_list, cursor++); @@ -18006,22 +18077,39 @@ parse_call_operator_write(pm_parser_t *parser, pm_call_node_t *call_node, const } } +/** + * Returns true if the name of the capture group is a valid local variable that + * can be written to. + */ static bool -name_is_identifier(pm_parser_t *parser, const uint8_t *source, size_t length) { +parse_regular_expression_named_capture(pm_parser_t *parser, const uint8_t *source, size_t length) { if (length == 0) { return false; } + // First ensure that it starts with a valid identifier starting character. size_t width = char_is_identifier_start(parser, source); if (!width) { return false; } - uint8_t *cursor = ((uint8_t *)source) + width; + // Next, ensure that it's not an uppercase character. + if (parser->encoding_changed) { + if (parser->encoding->isupper_char(source, (ptrdiff_t) length)) return false; + } else { + if (pm_encoding_utf_8_isupper_char(source, (ptrdiff_t) length)) return false; + } + + // Next, iterate through all of the bytes of the string to ensure that they + // are all valid identifier characters. + const uint8_t *cursor = source + width; while (cursor < source + length && (width = char_is_identifier(parser, cursor))) { cursor += width; } + // Finally, validate that the identifier is not a keywor. + if (pm_local_is_keyword((const char *) source, length)) return false; + return cursor == source + length; } @@ -18051,7 +18139,7 @@ parse_regular_expression_named_captures(pm_parser_t *parser, const pm_string_t * // If the name of the capture group isn't a valid identifier, we do // not add it to the local table. - if (!name_is_identifier(parser, source, length)) continue; + if (!parse_regular_expression_named_capture(parser, source, length)) continue; if (content->type == PM_STRING_SHARED) { // If the unescaped string is a slice of the source, then we can diff --git a/test/prism/fixtures/regex.txt b/test/prism/fixtures/regex.txt index ef2f6d45a32d8d..1010ffedc38fb7 100644 --- a/test/prism/fixtures/regex.txt +++ b/test/prism/fixtures/regex.txt @@ -38,3 +38,7 @@ b>)/ =~ ""; ab a = 1 tap { /(?)/ =~ to_s } + +/(?)/ =~ "" +/(?)/ =~ "" +/(?)/ =~ "" diff --git a/test/prism/snapshots/regex.txt b/test/prism/snapshots/regex.txt index d07ab8c5e71900..44657260c5e62e 100644 --- a/test/prism/snapshots/regex.txt +++ b/test/prism/snapshots/regex.txt @@ -1,8 +1,8 @@ -@ ProgramNode (location: (1,0)-(40,24)) +@ ProgramNode (location: (1,0)-(44,16)) ├── locals: [:foo, :ab, :abc, :a] └── statements: - @ StatementsNode (location: (1,0)-(40,24)) - └── body: (length: 21) + @ StatementsNode (location: (1,0)-(44,16)) + └── body: (length: 24) ├── @ CallNode (location: (1,0)-(1,9)) │ ├── flags: ignore_visibility │ ├── receiver: ∅ @@ -316,56 +316,137 @@ │ │ ├── flags: decimal │ │ └── value: 1 │ └── operator_loc: (39,2)-(39,3) = "=" - └── @ CallNode (location: (40,0)-(40,24)) - ├── flags: ignore_visibility - ├── receiver: ∅ + ├── @ CallNode (location: (40,0)-(40,24)) + │ ├── flags: ignore_visibility + │ ├── receiver: ∅ + │ ├── call_operator_loc: ∅ + │ ├── name: :tap + │ ├── message_loc: (40,0)-(40,3) = "tap" + │ ├── opening_loc: ∅ + │ ├── arguments: ∅ + │ ├── closing_loc: ∅ + │ └── block: + │ @ BlockNode (location: (40,4)-(40,24)) + │ ├── locals: [] + │ ├── parameters: ∅ + │ ├── body: + │ │ @ StatementsNode (location: (40,6)-(40,22)) + │ │ └── body: (length: 1) + │ │ └── @ MatchWriteNode (location: (40,6)-(40,22)) + │ │ ├── call: + │ │ │ @ CallNode (location: (40,6)-(40,22)) + │ │ │ ├── flags: ∅ + │ │ │ ├── receiver: + │ │ │ │ @ RegularExpressionNode (location: (40,6)-(40,14)) + │ │ │ │ ├── flags: forced_us_ascii_encoding + │ │ │ │ ├── opening_loc: (40,6)-(40,7) = "/" + │ │ │ │ ├── content_loc: (40,7)-(40,13) = "(?)" + │ │ │ │ ├── closing_loc: (40,13)-(40,14) = "/" + │ │ │ │ └── unescaped: "(?)" + │ │ │ ├── call_operator_loc: ∅ + │ │ │ ├── name: :=~ + │ │ │ ├── message_loc: (40,15)-(40,17) = "=~" + │ │ │ ├── opening_loc: ∅ + │ │ │ ├── arguments: + │ │ │ │ @ ArgumentsNode (location: (40,18)-(40,22)) + │ │ │ │ ├── flags: ∅ + │ │ │ │ └── arguments: (length: 1) + │ │ │ │ └── @ CallNode (location: (40,18)-(40,22)) + │ │ │ │ ├── flags: variable_call, ignore_visibility + │ │ │ │ ├── receiver: ∅ + │ │ │ │ ├── call_operator_loc: ∅ + │ │ │ │ ├── name: :to_s + │ │ │ │ ├── message_loc: (40,18)-(40,22) = "to_s" + │ │ │ │ ├── opening_loc: ∅ + │ │ │ │ ├── arguments: ∅ + │ │ │ │ ├── closing_loc: ∅ + │ │ │ │ └── block: ∅ + │ │ │ ├── closing_loc: ∅ + │ │ │ └── block: ∅ + │ │ └── targets: (length: 1) + │ │ └── @ LocalVariableTargetNode (location: (40,10)-(40,11)) + │ │ ├── name: :a + │ │ └── depth: 1 + │ ├── opening_loc: (40,4)-(40,5) = "{" + │ └── closing_loc: (40,23)-(40,24) = "}" + ├── @ MatchWriteNode (location: (42,0)-(42,16)) + │ ├── call: + │ │ @ CallNode (location: (42,0)-(42,16)) + │ │ ├── flags: ∅ + │ │ ├── receiver: + │ │ │ @ RegularExpressionNode (location: (42,0)-(42,10)) + │ │ │ ├── flags: forced_us_ascii_encoding + │ │ │ ├── opening_loc: (42,0)-(42,1) = "/" + │ │ │ ├── content_loc: (42,1)-(42,9) = "(?)" + │ │ │ ├── closing_loc: (42,9)-(42,10) = "/" + │ │ │ └── unescaped: "(?)" + │ │ ├── call_operator_loc: ∅ + │ │ ├── name: :=~ + │ │ ├── message_loc: (42,11)-(42,13) = "=~" + │ │ ├── opening_loc: ∅ + │ │ ├── arguments: + │ │ │ @ ArgumentsNode (location: (42,14)-(42,16)) + │ │ │ ├── flags: ∅ + │ │ │ └── arguments: (length: 1) + │ │ │ └── @ StringNode (location: (42,14)-(42,16)) + │ │ │ ├── flags: ∅ + │ │ │ ├── opening_loc: (42,14)-(42,15) = "\"" + │ │ │ ├── content_loc: (42,15)-(42,15) = "" + │ │ │ ├── closing_loc: (42,15)-(42,16) = "\"" + │ │ │ └── unescaped: "" + │ │ ├── closing_loc: ∅ + │ │ └── block: ∅ + │ └── targets: (length: 1) + │ └── @ LocalVariableTargetNode (location: (42,4)-(42,7)) + │ ├── name: :foo + │ └── depth: 0 + ├── @ CallNode (location: (43,0)-(43,16)) + │ ├── flags: ∅ + │ ├── receiver: + │ │ @ RegularExpressionNode (location: (43,0)-(43,10)) + │ │ ├── flags: forced_us_ascii_encoding + │ │ ├── opening_loc: (43,0)-(43,1) = "/" + │ │ ├── content_loc: (43,1)-(43,9) = "(?)" + │ │ ├── closing_loc: (43,9)-(43,10) = "/" + │ │ └── unescaped: "(?)" + │ ├── call_operator_loc: ∅ + │ ├── name: :=~ + │ ├── message_loc: (43,11)-(43,13) = "=~" + │ ├── opening_loc: ∅ + │ ├── arguments: + │ │ @ ArgumentsNode (location: (43,14)-(43,16)) + │ │ ├── flags: ∅ + │ │ └── arguments: (length: 1) + │ │ └── @ StringNode (location: (43,14)-(43,16)) + │ │ ├── flags: ∅ + │ │ ├── opening_loc: (43,14)-(43,15) = "\"" + │ │ ├── content_loc: (43,15)-(43,15) = "" + │ │ ├── closing_loc: (43,15)-(43,16) = "\"" + │ │ └── unescaped: "" + │ ├── closing_loc: ∅ + │ └── block: ∅ + └── @ CallNode (location: (44,0)-(44,16)) + ├── flags: ∅ + ├── receiver: + │ @ RegularExpressionNode (location: (44,0)-(44,10)) + │ ├── flags: forced_us_ascii_encoding + │ ├── opening_loc: (44,0)-(44,1) = "/" + │ ├── content_loc: (44,1)-(44,9) = "(?)" + │ ├── closing_loc: (44,9)-(44,10) = "/" + │ └── unescaped: "(?)" ├── call_operator_loc: ∅ - ├── name: :tap - ├── message_loc: (40,0)-(40,3) = "tap" + ├── name: :=~ + ├── message_loc: (44,11)-(44,13) = "=~" ├── opening_loc: ∅ - ├── arguments: ∅ + ├── arguments: + │ @ ArgumentsNode (location: (44,14)-(44,16)) + │ ├── flags: ∅ + │ └── arguments: (length: 1) + │ └── @ StringNode (location: (44,14)-(44,16)) + │ ├── flags: ∅ + │ ├── opening_loc: (44,14)-(44,15) = "\"" + │ ├── content_loc: (44,15)-(44,15) = "" + │ ├── closing_loc: (44,15)-(44,16) = "\"" + │ └── unescaped: "" ├── closing_loc: ∅ - └── block: - @ BlockNode (location: (40,4)-(40,24)) - ├── locals: [] - ├── parameters: ∅ - ├── body: - │ @ StatementsNode (location: (40,6)-(40,22)) - │ └── body: (length: 1) - │ └── @ MatchWriteNode (location: (40,6)-(40,22)) - │ ├── call: - │ │ @ CallNode (location: (40,6)-(40,22)) - │ │ ├── flags: ∅ - │ │ ├── receiver: - │ │ │ @ RegularExpressionNode (location: (40,6)-(40,14)) - │ │ │ ├── flags: forced_us_ascii_encoding - │ │ │ ├── opening_loc: (40,6)-(40,7) = "/" - │ │ │ ├── content_loc: (40,7)-(40,13) = "(?)" - │ │ │ ├── closing_loc: (40,13)-(40,14) = "/" - │ │ │ └── unescaped: "(?)" - │ │ ├── call_operator_loc: ∅ - │ │ ├── name: :=~ - │ │ ├── message_loc: (40,15)-(40,17) = "=~" - │ │ ├── opening_loc: ∅ - │ │ ├── arguments: - │ │ │ @ ArgumentsNode (location: (40,18)-(40,22)) - │ │ │ ├── flags: ∅ - │ │ │ └── arguments: (length: 1) - │ │ │ └── @ CallNode (location: (40,18)-(40,22)) - │ │ │ ├── flags: variable_call, ignore_visibility - │ │ │ ├── receiver: ∅ - │ │ │ ├── call_operator_loc: ∅ - │ │ │ ├── name: :to_s - │ │ │ ├── message_loc: (40,18)-(40,22) = "to_s" - │ │ │ ├── opening_loc: ∅ - │ │ │ ├── arguments: ∅ - │ │ │ ├── closing_loc: ∅ - │ │ │ └── block: ∅ - │ │ ├── closing_loc: ∅ - │ │ └── block: ∅ - │ └── targets: (length: 1) - │ └── @ LocalVariableTargetNode (location: (40,10)-(40,11)) - │ ├── name: :a - │ └── depth: 1 - ├── opening_loc: (40,4)-(40,5) = "{" - └── closing_loc: (40,23)-(40,24) = "}" + └── block: ∅ From a8ec347ca2511bec811908e70923f96b4ddcda21 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 28 Mar 2024 15:48:44 -0400 Subject: [PATCH 143/211] [ruby/prism] Allow writing to keywords with named captures if they are already locals https://github.com/ruby/prism/commit/418318e1c8 --- prism/prism.c | 16 ++--- test/prism/fixtures/regex.txt | 2 + test/prism/snapshots/regex.txt | 114 +++++++++++++++++++++++++-------- 3 files changed, 96 insertions(+), 36 deletions(-) diff --git a/prism/prism.c b/prism/prism.c index 1cfcf704bb87ef..db71a9c3f46b1f 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -18107,9 +18107,6 @@ parse_regular_expression_named_capture(pm_parser_t *parser, const uint8_t *sourc cursor += width; } - // Finally, validate that the identifier is not a keywor. - if (pm_local_is_keyword((const char *) source, length)) return false; - return cursor == source + length; } @@ -18170,16 +18167,19 @@ parse_regular_expression_named_captures(pm_parser_t *parser, const pm_string_t * if (pm_constant_id_list_includes(&names, name)) continue; pm_constant_id_list_append(&names, name); - // Here we lazily create the MatchWriteNode since we know we're - // about to add a target. - if (match == NULL) match = pm_match_write_node_create(parser, call); - - // First, find the depth of the local that is being assigned. int depth; if ((depth = pm_parser_local_depth_constant_id(parser, name)) == -1) { + // If the identifier is not already a local, then we'll add + // it to the local table unless it's a keyword. + if (pm_local_is_keyword((const char *) source, length)) continue; + pm_parser_local_add(parser, name); } + // Here we lazily create the MatchWriteNode since we know we're + // about to add a target. + if (match == NULL) match = pm_match_write_node_create(parser, call); + // Next, create the local variable target and add it to the // list of targets for the match. pm_node_t *target = (pm_node_t *) pm_local_variable_target_node_create_values(parser, &location, name, depth == -1 ? 0 : (uint32_t) depth); diff --git a/test/prism/fixtures/regex.txt b/test/prism/fixtures/regex.txt index 1010ffedc38fb7..18200e5cbd230a 100644 --- a/test/prism/fixtures/regex.txt +++ b/test/prism/fixtures/regex.txt @@ -41,4 +41,6 @@ tap { /(?)/ =~ to_s } /(?)/ =~ "" /(?)/ =~ "" + /(?)/ =~ "" +def foo(nil:) = /(?)/ =~ "" diff --git a/test/prism/snapshots/regex.txt b/test/prism/snapshots/regex.txt index 44657260c5e62e..ef576b8bc0affe 100644 --- a/test/prism/snapshots/regex.txt +++ b/test/prism/snapshots/regex.txt @@ -1,8 +1,8 @@ -@ ProgramNode (location: (1,0)-(44,16)) +@ ProgramNode (location: (1,0)-(46,32)) ├── locals: [:foo, :ab, :abc, :a] └── statements: - @ StatementsNode (location: (1,0)-(44,16)) - └── body: (length: 24) + @ StatementsNode (location: (1,0)-(46,32)) + └── body: (length: 25) ├── @ CallNode (location: (1,0)-(1,9)) │ ├── flags: ignore_visibility │ ├── receiver: ∅ @@ -425,28 +425,86 @@ │ │ └── unescaped: "" │ ├── closing_loc: ∅ │ └── block: ∅ - └── @ CallNode (location: (44,0)-(44,16)) - ├── flags: ∅ - ├── receiver: - │ @ RegularExpressionNode (location: (44,0)-(44,10)) - │ ├── flags: forced_us_ascii_encoding - │ ├── opening_loc: (44,0)-(44,1) = "/" - │ ├── content_loc: (44,1)-(44,9) = "(?)" - │ ├── closing_loc: (44,9)-(44,10) = "/" - │ └── unescaped: "(?)" - ├── call_operator_loc: ∅ - ├── name: :=~ - ├── message_loc: (44,11)-(44,13) = "=~" - ├── opening_loc: ∅ - ├── arguments: - │ @ ArgumentsNode (location: (44,14)-(44,16)) - │ ├── flags: ∅ - │ └── arguments: (length: 1) - │ └── @ StringNode (location: (44,14)-(44,16)) - │ ├── flags: ∅ - │ ├── opening_loc: (44,14)-(44,15) = "\"" - │ ├── content_loc: (44,15)-(44,15) = "" - │ ├── closing_loc: (44,15)-(44,16) = "\"" - │ └── unescaped: "" - ├── closing_loc: ∅ - └── block: ∅ + ├── @ CallNode (location: (45,0)-(45,16)) + │ ├── flags: ∅ + │ ├── receiver: + │ │ @ RegularExpressionNode (location: (45,0)-(45,10)) + │ │ ├── flags: forced_us_ascii_encoding + │ │ ├── opening_loc: (45,0)-(45,1) = "/" + │ │ ├── content_loc: (45,1)-(45,9) = "(?)" + │ │ ├── closing_loc: (45,9)-(45,10) = "/" + │ │ └── unescaped: "(?)" + │ ├── call_operator_loc: ∅ + │ ├── name: :=~ + │ ├── message_loc: (45,11)-(45,13) = "=~" + │ ├── opening_loc: ∅ + │ ├── arguments: + │ │ @ ArgumentsNode (location: (45,14)-(45,16)) + │ │ ├── flags: ∅ + │ │ └── arguments: (length: 1) + │ │ └── @ StringNode (location: (45,14)-(45,16)) + │ │ ├── flags: ∅ + │ │ ├── opening_loc: (45,14)-(45,15) = "\"" + │ │ ├── content_loc: (45,15)-(45,15) = "" + │ │ ├── closing_loc: (45,15)-(45,16) = "\"" + │ │ └── unescaped: "" + │ ├── closing_loc: ∅ + │ └── block: ∅ + └── @ DefNode (location: (46,0)-(46,32)) + ├── name: :foo + ├── name_loc: (46,4)-(46,7) = "foo" + ├── receiver: ∅ + ├── parameters: + │ @ ParametersNode (location: (46,8)-(46,12)) + │ ├── requireds: (length: 0) + │ ├── optionals: (length: 0) + │ ├── rest: ∅ + │ ├── posts: (length: 0) + │ ├── keywords: (length: 1) + │ │ └── @ RequiredKeywordParameterNode (location: (46,8)-(46,12)) + │ │ ├── flags: ∅ + │ │ ├── name: :nil + │ │ └── name_loc: (46,8)-(46,12) = "nil:" + │ ├── keyword_rest: ∅ + │ └── block: ∅ + ├── body: + │ @ StatementsNode (location: (46,16)-(46,32)) + │ └── body: (length: 1) + │ └── @ MatchWriteNode (location: (46,16)-(46,32)) + │ ├── call: + │ │ @ CallNode (location: (46,16)-(46,32)) + │ │ ├── flags: ∅ + │ │ ├── receiver: + │ │ │ @ RegularExpressionNode (location: (46,16)-(46,26)) + │ │ │ ├── flags: forced_us_ascii_encoding + │ │ │ ├── opening_loc: (46,16)-(46,17) = "/" + │ │ │ ├── content_loc: (46,17)-(46,25) = "(?)" + │ │ │ ├── closing_loc: (46,25)-(46,26) = "/" + │ │ │ └── unescaped: "(?)" + │ │ ├── call_operator_loc: ∅ + │ │ ├── name: :=~ + │ │ ├── message_loc: (46,27)-(46,29) = "=~" + │ │ ├── opening_loc: ∅ + │ │ ├── arguments: + │ │ │ @ ArgumentsNode (location: (46,30)-(46,32)) + │ │ │ ├── flags: ∅ + │ │ │ └── arguments: (length: 1) + │ │ │ └── @ StringNode (location: (46,30)-(46,32)) + │ │ │ ├── flags: ∅ + │ │ │ ├── opening_loc: (46,30)-(46,31) = "\"" + │ │ │ ├── content_loc: (46,31)-(46,31) = "" + │ │ │ ├── closing_loc: (46,31)-(46,32) = "\"" + │ │ │ └── unescaped: "" + │ │ ├── closing_loc: ∅ + │ │ └── block: ∅ + │ └── targets: (length: 1) + │ └── @ LocalVariableTargetNode (location: (46,20)-(46,23)) + │ ├── name: :nil + │ └── depth: 0 + ├── locals: [:nil] + ├── def_keyword_loc: (46,0)-(46,3) = "def" + ├── operator_loc: ∅ + ├── lparen_loc: (46,7)-(46,8) = "(" + ├── rparen_loc: (46,12)-(46,13) = ")" + ├── equal_loc: (46,14)-(46,15) = "=" + └── end_keyword_loc: ∅ From 817eecf685cf2408ff468ba9c3b814e3c6389ce7 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 28 Mar 2024 17:35:54 -0400 Subject: [PATCH 144/211] [PRISM] Enable passing regexp test --- test/.excludes-prism/TestRegexp.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/test/.excludes-prism/TestRegexp.rb b/test/.excludes-prism/TestRegexp.rb index f2b817d79a7266..090515bbe4efd0 100644 --- a/test/.excludes-prism/TestRegexp.rb +++ b/test/.excludes-prism/TestRegexp.rb @@ -1,6 +1,5 @@ exclude(:test_unicode_age_14_0, "unknown") exclude(:test_invalid_fragment, "unknown") -exclude(:test_assign_named_capture_to_reserved_word, "unknown") exclude(:test_unicode_age_15_0, "unknown") exclude(:test_unescape, "unknown") exclude(:test_invalid_escape_error, "unknown") From f3c35749feb559db5aa3597a1c91a30c2550e85c Mon Sep 17 00:00:00 2001 From: Alan Wu Date: Thu, 28 Mar 2024 15:46:08 -0400 Subject: [PATCH 145/211] YJIT: Optimize putobject+opt_ltlt for integers In `jit_rb_int_lshift()`, we guard against the right hand side changing since we want to avoid generating variable length shifts. When control reaches a `putobject` and `opt_ltlt` pair, though, we know that the right hand side never changes. This commit detects this situation and substitutes an implementation that does not guard against the right hand side changing, saving that work. Deleted some `putobject` Rust tests since they aren't that valuable and cause linking issues. Nice boost to `optcarrot` and `protoboeuf`: ``` ---------- ------------------ bench yjit-pre/yjit-post optcarrot 1.09 protoboeuf 1.12 ---------- ------------------ ``` --- bootstraptest/test_yjit.rb | 17 +++++ yjit/src/codegen.rs | 137 ++++++++++++++++++++++--------------- 2 files changed, 98 insertions(+), 56 deletions(-) diff --git a/bootstraptest/test_yjit.rb b/bootstraptest/test_yjit.rb index e5a160fa9fa342..6b7d4839265d04 100644 --- a/bootstraptest/test_yjit.rb +++ b/bootstraptest/test_yjit.rb @@ -4769,3 +4769,20 @@ def tests tests } + +# test integer left shift with constant rhs +assert_equal [0x80000000000, 'a+', :ok].inspect, %q{ + def shift(val) = val << 43 + + def tests + int = shift(1) + str = shift("a") + + Integer.define_method(:<<) { |_| :ok } + redef = shift(1) + + [int, str, redef] + end + + tests +} diff --git a/yjit/src/codegen.rs b/yjit/src/codegen.rs index 0d536907c1abad..c6e50beab1b156 100644 --- a/yjit/src/codegen.rs +++ b/yjit/src/codegen.rs @@ -30,6 +30,7 @@ pub use crate::virtualmem::CodePtr; /// Status returned by code generation functions #[derive(PartialEq, Debug)] enum CodegenStatus { + SkipNextInsn, KeepCompiling, EndBlock, } @@ -1197,6 +1198,13 @@ pub fn gen_single_block( // Move to the next instruction to compile insn_idx += insn_len(opcode) as u16; + // Move past next instruction when instructed + if status == Some(SkipNextInsn) { + let next_pc = unsafe { rb_iseq_pc_at_idx(iseq, insn_idx.into()) }; + let next_opcode: usize = unsafe { rb_iseq_opcode_at_pc(iseq, next_pc) }.try_into().unwrap(); + insn_idx += insn_len(next_opcode) as u16; + } + // If the instruction terminates this block if status == Some(EndBlock) { break; @@ -1343,7 +1351,7 @@ fn jit_putobject(asm: &mut Assembler, arg: VALUE) { fn gen_putobject_int2fix( jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, + ocb: &mut OutlinedCb, ) -> Option { let opcode = jit.opcode; let cst_val: usize = if opcode == YARVINSN_putobject_INT2FIX_0_.as_usize() { @@ -1351,22 +1359,86 @@ fn gen_putobject_int2fix( } else { 1 }; + let cst_val = VALUE::fixnum_from_usize(cst_val); + + if let Some(result) = fuse_putobject_opt_ltlt(jit, asm, cst_val, ocb) { + return Some(result); + } - jit_putobject(asm, VALUE::fixnum_from_usize(cst_val)); + jit_putobject(asm, cst_val); Some(KeepCompiling) } fn gen_putobject( jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, + ocb: &mut OutlinedCb, ) -> Option { let arg: VALUE = jit.get_arg(0); + if let Some(result) = fuse_putobject_opt_ltlt(jit, asm, arg, ocb) { + return Some(result); + } + jit_putobject(asm, arg); Some(KeepCompiling) } +/// Combine `putobject` and and `opt_ltlt` together if profitable, for example when +/// left shifting an integer by a constant amount. +fn fuse_putobject_opt_ltlt( + jit: &mut JITState, + asm: &mut Assembler, + constant_object: VALUE, + ocb: &mut OutlinedCb, +) -> Option { + let next_opcode = unsafe { rb_vm_insn_addr2opcode(jit.pc.add(insn_len(jit.opcode).as_usize()).read().as_ptr()) }; + if next_opcode == YARVINSN_opt_ltlt as i32 && constant_object.fixnum_p() { + // Untag the fixnum shift amount + let shift_amt = constant_object.as_isize() >> 1; + if shift_amt > 63 || shift_amt < 0 { + return None; + } + if !jit.at_current_insn() { + defer_compilation(jit, asm, ocb); + return Some(EndBlock); + } + + let lhs = jit.peek_at_stack(&asm.ctx, 0); + if !lhs.fixnum_p() { + return None; + } + + if !assume_bop_not_redefined(jit, asm, ocb, INTEGER_REDEFINED_OP_FLAG, BOP_LTLT) { + return None; + } + + asm_comment!(asm, "integer left shift with rhs={shift_amt}"); + let lhs = asm.stack_opnd(0); + + // Guard that lhs is a fixnum if necessary + let lhs_type = asm.ctx.get_opnd_type(lhs.into()); + if lhs_type != Type::Fixnum { + asm_comment!(asm, "guard arg0 fixnum"); + asm.test(lhs, Opnd::UImm(RUBY_FIXNUM_FLAG as u64)); + + jit_chain_guard( + JCC_JZ, + jit, + asm, + ocb, + SEND_MAX_DEPTH, + Counter::guard_send_not_fixnums, + ); + } + + asm.stack_pop(1); + fixnum_left_shift_body(asm, lhs, shift_amt as u64); + return Some(SkipNextInsn); + } + return None; +} + fn gen_putself( _jit: &mut JITState, asm: &mut Assembler, @@ -5073,8 +5145,13 @@ fn jit_rb_int_lshift( Counter::lshift_amount_changed, ); + fixnum_left_shift_body(asm, lhs, shift_amt as u64); + true +} + +fn fixnum_left_shift_body(asm: &mut Assembler, lhs: Opnd, shift_amt: u64) { let in_val = asm.sub(lhs, 1.into()); - let shift_opnd = Opnd::UImm(shift_amt as u64); + let shift_opnd = Opnd::UImm(shift_amt); let out_val = asm.lshift(in_val, shift_opnd); let unshifted = asm.rshift(out_val, shift_opnd); @@ -5087,7 +5164,6 @@ fn jit_rb_int_lshift( let ret_opnd = asm.stack_push(Type::Fixnum); asm.mov(ret_opnd, out_val); - true } fn jit_rb_int_rshift( @@ -10384,57 +10460,6 @@ mod tests { assert!(cb.get_write_pos() > 0); } - #[test] - fn test_putobject_qtrue() { - // Test gen_putobject with Qtrue - let (mut jit, _context, mut asm, mut cb, mut ocb) = setup_codegen(); - - let mut value_array: [u64; 2] = [0, Qtrue.into()]; - let pc: *mut VALUE = &mut value_array as *mut u64 as *mut VALUE; - jit.pc = pc; - - let status = gen_putobject(&mut jit, &mut asm, &mut ocb); - - let tmp_type_top = asm.ctx.get_opnd_type(StackOpnd(0)); - - assert_eq!(status, Some(KeepCompiling)); - assert_eq!(tmp_type_top, Type::True); - asm.compile(&mut cb, None).unwrap(); - assert!(cb.get_write_pos() > 0); - } - - #[test] - fn test_putobject_fixnum() { - // Test gen_putobject with a Fixnum to test another conditional branch - let (mut jit, _context, mut asm, mut cb, mut ocb) = setup_codegen(); - - // The Fixnum 7 is encoded as 7 * 2 + 1, or 15 - let mut value_array: [u64; 2] = [0, 15]; - let pc: *mut VALUE = &mut value_array as *mut u64 as *mut VALUE; - jit.pc = pc; - - let status = gen_putobject(&mut jit, &mut asm, &mut ocb); - - let tmp_type_top = asm.ctx.get_opnd_type(StackOpnd(0)); - - assert_eq!(status, Some(KeepCompiling)); - assert_eq!(tmp_type_top, Type::Fixnum); - asm.compile(&mut cb, None).unwrap(); - assert!(cb.get_write_pos() > 0); - } - - #[test] - fn test_int2fix() { - let (mut jit, _context, mut asm, _cb, mut ocb) = setup_codegen(); - jit.opcode = YARVINSN_putobject_INT2FIX_0_.as_usize(); - let status = gen_putobject_int2fix(&mut jit, &mut asm, &mut ocb); - - let tmp_type_top = asm.ctx.get_opnd_type(StackOpnd(0)); - - // Right now we're not testing the generated machine code to make sure a literal 1 or 0 was pushed. I've checked locally. - assert_eq!(status, Some(KeepCompiling)); - assert_eq!(tmp_type_top, Type::Fixnum); - } #[test] fn test_putself() { From 02d40b6c171db2e6ae0c3f259b470c873f746d70 Mon Sep 17 00:00:00 2001 From: "Daisuke Fujimura (fd0)" Date: Tue, 26 Dec 2023 10:27:01 +0900 Subject: [PATCH 146/211] Use ubf list on cygwin --- thread_pthread.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/thread_pthread.c b/thread_pthread.c index cdaf6f240c436a..82b5e362ccc0eb 100644 --- a/thread_pthread.c +++ b/thread_pthread.c @@ -305,7 +305,7 @@ event_name(rb_event_flag_t event) static rb_serial_t current_fork_gen = 1; /* We can't use GET_VM()->fork_gen */ -#if defined(SIGVTALRM) && !defined(__CYGWIN__) && !defined(__EMSCRIPTEN__) +#if defined(SIGVTALRM) && !defined(__EMSCRIPTEN__) # define USE_UBF_LIST 1 #endif From e5def27fbfaca4f08ff3a9bcf4c27742371a1da9 Mon Sep 17 00:00:00 2001 From: Ellen Marie Dash Date: Sat, 11 Sep 2021 17:15:21 -0400 Subject: [PATCH 147/211] [rubygems/rubygems] Add "gem rebuild" command. https://github.com/rubygems/rubygems/commit/6d661573f0 --- lib/rubygems/command_manager.rb | 1 + lib/rubygems/commands/rebuild_command.rb | 262 ++++++++++++++++++ .../test_gem_commands_rebuild_command.rb | 145 ++++++++++ 3 files changed, 408 insertions(+) create mode 100644 lib/rubygems/commands/rebuild_command.rb create mode 100644 test/rubygems/test_gem_commands_rebuild_command.rb diff --git a/lib/rubygems/command_manager.rb b/lib/rubygems/command_manager.rb index c2e4f4ce49ebc4..8e578dc1966172 100644 --- a/lib/rubygems/command_manager.rb +++ b/lib/rubygems/command_manager.rb @@ -60,6 +60,7 @@ class Gem::CommandManager :push, :query, :rdoc, + :rebuild, :search, :server, :signin, diff --git a/lib/rubygems/commands/rebuild_command.rb b/lib/rubygems/commands/rebuild_command.rb new file mode 100644 index 00000000000000..731d14328c729b --- /dev/null +++ b/lib/rubygems/commands/rebuild_command.rb @@ -0,0 +1,262 @@ +# frozen_string_literal: true + +require "date" +require "digest" +require "fileutils" +require_relative "../package" + +class Gem::Commands::RebuildCommand < Gem::Command + DATE_FORMAT = "%Y-%m-%d %H:%M:%S.%N Z" + + def initialize + super "rebuild", "Attempt to reproduce a build of a gem." + + add_option "--force", "Skip validation of the spec." do |_value, options| + options[:force] = true + end + + add_option "--strict", "Consider warnings as errors when validating the spec." do |_value, options| + options[:strict] = true + end + + add_option "--source GEM_SOURCE", "Specify the source to download the gem from." do |value, options| + options[:source] = value + end + + add_option "--original GEM_FILE", "Specify a local file to compare against (instead of downloading it)." do |value, options| + options[:original_gem_file] = value + end + + add_option "--gemspec GEMSPEC_FILE", "Specify the name of the gemspec file." do |value, options| + options[:gemspec_file] = value + end + + add_option "-C PATH", "Run as if gem build was started in instead of the current working directory." do |value, options| + options[:build_path] = value + end + end + + def arguments # :nodoc: + "GEM_NAME gem name on gem server\n" \ + "GEM_VERSION gem version you are attempting to rebuild" + end + + def description # :nodoc: + <<-EOF +The rebuild command allows you to (attempt to) reproduce a build of a gem +from a ruby gemspec. + +This command assumes the gemspec can be built with the `gem build` command. +If you use either `gem build` or `rake build`/`rake release` to build/release +a gem, it is a potential candidate. + +If the gem includes top-level files that change frequently (e.g. Gemfile.lock), +it may require more effort to reproduce a build. For example, it might require +more precisely matched versions of Ruby, RubyGems, or Bundler to be used. + +An example of reproducing a gem build: + + $ pwd + /usr/home/puppy/test/rebuild + $ ls -a + ./ ../ + $ git clone --branch v12.0.2 https://github.com/duckinator/okay.git + $ gem rebuild -C ./okay --gemspec okay.gemspec okay 12.0.2 + Fetching okay-12.0.2.gem + Downloaded okay version 12.0.2 as /usr/home/puppy/test/rebuild/rebuild/old/okay-12.0.2.gem. + Successfully built RubyGem + Name: okay + Version: 12.0.2 + File: okay-12.0.2.gem + + Built at: 2023-08-31 21:39:02 EDT (1693532342) + Original build saved to: /usr/home/puppy/test/rebuild/rebuild/old/okay-12.0.2.gem + Reproduced build saved to: /usr/home/puppy/test/rebuild/rebuild/new/okay-12.0.2.gem + Working directory: ./okay + + Hash comparison: + 38a8bd78ce10bc19189ead0b56fa490d03788a2f926a6481b2f1f5d5fa5ab75b /usr/home/puppy/test/rebuild/rebuild/old/okay-12.0.2.gem + 38a8bd78ce10bc19189ead0b56fa490d03788a2f926a6481b2f1f5d5fa5ab75b /usr/home/puppy/test/rebuild/rebuild/new/okay-12.0.2.gem + + SUCCESS - original and rebuild hashes matched + EOF + end + + def usage # :nodoc: + "#{program_name} GEM_NAME GEM_VERSION" + end + + def execute + gem_name, gem_version = get_gem_name_and_version + + old_dir, new_dir = prep_dirs + + gem_filename = "#{gem_name}-#{gem_version}.gem" + old_file = File.join(old_dir, gem_filename) + new_file = File.join(new_dir, gem_filename) + + if options[:original_gem_file] + FileUtils.copy_file(options[:original_gem_file], old_file) + else + download_gem(gem_name, gem_version, old_file) + end + + source_date_epoch = get_timestamp(old_file).to_s + + if build_path = options[:build_path] + Dir.chdir(build_path) { build_gem(gem_name, source_date_epoch, new_file) } + else + build_gem(gem_name, source_date_epoch, new_file) + end + + compare(source_date_epoch, old_file, new_file) + end + + private + + def sha256(file) + Digest::SHA256.hexdigest(File.read(file)) + end + + def get_timestamp(file) + mtime = nil + Gem::Package::TarReader.new(File.open(file)) do |tar| + mtime = tar.seek("metadata.gz") {|f| f.header.mtime } + end + + mtime + end + + def compare(source_date_epoch, old_file, new_file) + date = Time.at(source_date_epoch.to_i).strftime("%F %T %Z") + + old_hash = sha256(old_file) + new_hash = sha256(new_file) + + say + say "Built at: #{date} (#{source_date_epoch})" + say "Original build saved to: #{old_file}" + say "Reproduced build saved to: #{new_file}" + say "Working directory: #{options[:build_path] || Dir.pwd}" + say + say "Hash comparison:" + say " #{old_hash}\t#{old_file}" + say " #{new_hash}\t#{new_file}" + say + + if old_hash == new_hash + say "SUCCESS - original and rebuild hashes matched" + else + say "FAILURE - original and rebuild hashes did not match" + terminate_interaction 1 + end + end + + def prep_dirs + rebuild_dir = File.expand_path("rebuild") + old_dir = File.join(rebuild_dir, "old") + new_dir = File.join(rebuild_dir, "new") + + if File.directory?(rebuild_dir) + FileUtils.remove_dir(rebuild_dir) + end + + FileUtils.mkdir_p(old_dir) + FileUtils.mkdir_p(new_dir) + + [old_dir, new_dir] + end + + def get_gem_name_and_version + args = options[:args] || [] + if args.length == 2 + gem_name, gem_version = args + elsif args.length > 2 + raise Gem::CommandLineError, "Too many arguments" + else + raise Gem::CommandLineError, "Expected GEM_NAME and GEM_VERSION arguments (gem rebuild GEM_NAME GEM_VERSION)" + end + + [gem_name, gem_version] + end + + def build_gem(gem_name, source_date_epoch, output_file) + gemspec = options[:gemspec_file] || find_gemspec("#{gem_name}.gemspec") + + if gemspec + build_package(gemspec, source_date_epoch, output_file) + else + alert_error error_message(gem_name) + terminate_interaction(1) + end + end + + def build_package(gemspec, source_date_epoch, output_file) + with_source_date_epoch(source_date_epoch) do + spec = Gem::Specification.load(gemspec) + if spec + Gem::Package.build( + spec, + options[:force], + options[:strict], + output_file + ) + else + alert_error "Error loading gemspec. Aborting." + terminate_interaction 1 + end + end + end + + def with_source_date_epoch(source_date_epoch) + old_sde = ENV["SOURCE_DATE_EPOCH"] + ENV["SOURCE_DATE_EPOCH"] = source_date_epoch.to_s + + yield + ensure + ENV["SOURCE_DATE_EPOCH"] = old_sde + end + + def find_gemspec(glob = "*.gemspec") + gemspecs = Dir.glob(glob).sort + + if gemspecs.size > 1 + alert_error "Multiple gemspecs found: #{gemspecs}, please specify one" + terminate_interaction(1) + end + + gemspecs.first + end + + def error_message(gem_name) + if gem_name + "Couldn't find a gemspec file matching '#{gem_name}' in #{Dir.pwd}" + else + "Couldn't find a gemspec file in #{Dir.pwd}" + end + end + + def download_gem(gem_name, gem_version, old_file) + # This code was based loosely off the `gem fetch` command. + version = "= #{gem_version}" + dep = Gem::Dependency.new gem_name, version + + specs_and_sources, errors = + Gem::SpecFetcher.fetcher.spec_for_dependency dep + + # There should never be more than one item in specs_and_sources, + # since we search for an exact version. + spec, source = specs_and_sources[0] + + if spec.nil? + show_lookup_failure gem_name, version, errors, options[:domain] + terminate_interaction 1 + end + + download_path = source.download spec + + FileUtils.move(download_path, old_file) + + say "Downloaded #{gem_name} version #{gem_version} as #{old_file}." + end +end diff --git a/test/rubygems/test_gem_commands_rebuild_command.rb b/test/rubygems/test_gem_commands_rebuild_command.rb new file mode 100644 index 00000000000000..5e8c797e2d11ad --- /dev/null +++ b/test/rubygems/test_gem_commands_rebuild_command.rb @@ -0,0 +1,145 @@ +# frozen_string_literal: true + +require_relative "helper" +require "rubygems/commands/build_command" +require "rubygems/commands/rebuild_command" +require "rubygems/package" + +class TestGemCommandsRebuildCommand < Gem::TestCase + def setup + super + + readme_file = File.join(@tempdir, "README.md") + + begin + umask_orig = File.umask(2) + File.open readme_file, "w" do |f| + f.write "My awesome gem" + end + ensure + File.umask(umask_orig) + end + + @gem_name = "rebuild_test_gem" + @gem_version = "1.0.0" + @gem = util_spec @gem_name do |s| + s.version = @gem_version + s.license = "AGPL-3.0" + s.files = ["README.md"] + end + end + + def util_test_build_gem(gem, args) + @ui = Gem::MockGemUi.new + + cmd = Gem::Commands::BuildCommand.new + + cmd.options[:args] = args + cmd.options[:build_path] = @tempdir + use_ui @ui do + cmd.execute + end + gem_file = "#{@gem_name}-#{@gem_version}.gem" + output = @ui.output.split "\n" + assert_equal " Successfully built RubyGem", output.shift + assert_equal " Name: #{@gem_name}", output.shift + assert_equal " Version: #{@gem_version}", output.shift + assert_equal " File: #{gem_file}", output.shift + assert_equal [], output + + gem_file = File.join(@tempdir, gem_file) + assert File.exist?(gem_file) + + spec = Gem::Package.new(gem_file).spec + + assert_equal @gem_name, spec.name + assert_equal "this is a summary", spec.summary + gem_file + end + + def util_test_rebuild_gem(gem, args, original_gem_file, gemspec_file, timestamp) + @ui = Gem::MockGemUi.new + + cmd = Gem::Commands::RebuildCommand.new + + cmd.options[:args] = args + cmd.options[:original_gem_file] = original_gem_file + cmd.options[:build_path] = @tempdir + cmd.options[:gemspec_file] = gemspec_file + use_ui @ui do + cmd.execute + end + gem_file = "#{@gem_name}-#{@gem_version}.gem" + output = @ui.output.split "\n" + + assert_equal " Successfully built RubyGem", output.shift + assert_equal " Name: #{@gem_name}", output.shift + assert_equal " Version: #{@gem_version}", output.shift + assert_equal " File: #{gem_file}", output.shift + assert_empty output.shift + assert_match(/^Built at: .+ \(#{timestamp}\)/, output.shift) + original_line = output.shift + original = original_line.split(" ")[-1] + assert_match(/^Original build saved to: /, original_line) + reproduced_line = output.shift + reproduced = reproduced_line.split(" ")[-1] + assert_match(/^Reproduced build saved to: /, reproduced_line) + assert_equal "Working directory: #{@tempdir}", output.shift + assert_equal "", output.shift + assert_equal "Hash comparison:", output.shift + output.shift # " #{old_hash}\t#{old_file}" + output.shift # " #{new_hash}\t#{new_file}" + assert_empty output.shift + assert_equal "SUCCESS - original and rebuild hashes matched", output.shift + assert_equal [], output + + assert File.exist?(original) + assert File.exist?(reproduced) + + old_spec = Gem::Package.new(original).spec + new_spec = Gem::Package.new(reproduced).spec + + assert_equal @gem_name, old_spec.name + assert_equal "this is a summary", old_spec.summary + + assert_equal old_spec.name, new_spec.name + assert_equal old_spec.summary, new_spec.summary + + reproduced + end + + def test_build_is_reproducible + # Back up SOURCE_DATE_EPOCH to restore later. + epoch = ENV["SOURCE_DATE_EPOCH"] + + gemspec_file = File.join(@tempdir, @gem.spec_name) + + # Initial Build + + # Set SOURCE_DATE_EPOCH to 2001-02-03 04:05:06 -0500. + ENV["SOURCE_DATE_EPOCH"] = timestamp = Time.new(2001, 2, 3, 4, 5, 6).to_i.to_s + File.write(gemspec_file, @gem.to_ruby) + gem_file = util_test_build_gem @gem, [gemspec_file] + + build_contents = File.read(gem_file) + + gem_file_dir = File.dirname(gem_file) + gem_file_name = File.basename(gem_file) + original_gem_file = File.join(gem_file_dir, "original-" + gem_file_name) + File.rename(gem_file, original_gem_file) + + # Rebuild + + # Set SOURCE_DATE_EPOCH to a different value, meaning we are + # also testing that `gem rebuild` overrides the value. + ENV["SOURCE_DATE_EPOCH"] = Time.new(2007, 8, 9, 10, 11, 12).to_s + + rebuild_gem_file = util_test_rebuild_gem(@gem, [@gem_name, @gem_version], original_gem_file, gemspec_file, timestamp) + + rebuild_contents = File.read(rebuild_gem_file) + + assert_equal build_contents, rebuild_contents + ensure + ENV["SOURCE_DATE_EPOCH"] = epoch + end +end From cd12dfd38888cc3d3f4696146d154da15412bc93 Mon Sep 17 00:00:00 2001 From: Ellen Marie Dash Date: Tue, 30 Jan 2024 18:31:05 -0500 Subject: [PATCH 148/211] [rubygems/rubygems] [rebuild_command] Avoid leaking files. https://github.com/rubygems/rubygems/commit/3b88553d0d --- lib/rubygems/commands/rebuild_command.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/rubygems/commands/rebuild_command.rb b/lib/rubygems/commands/rebuild_command.rb index 731d14328c729b..234d0af85617f2 100644 --- a/lib/rubygems/commands/rebuild_command.rb +++ b/lib/rubygems/commands/rebuild_command.rb @@ -120,8 +120,10 @@ def sha256(file) def get_timestamp(file) mtime = nil - Gem::Package::TarReader.new(File.open(file)) do |tar| - mtime = tar.seek("metadata.gz") {|f| f.header.mtime } + File.open(file, "rb") do |f| + Gem::Package::TarReader.new(f) do |tar| + mtime = tar.seek("metadata.gz") {|tf| tf.header.mtime } + end end mtime From a28087affc5cef24376643507f7cb17c5f9c0110 Mon Sep 17 00:00:00 2001 From: Ellen Marie Dash Date: Tue, 30 Jan 2024 22:05:56 -0500 Subject: [PATCH 149/211] [rubygems/rubygems] [rebuild_command] Add --diff flag to try using diffoscope. https://github.com/rubygems/rubygems/commit/3e9545193a --- lib/rubygems/commands/rebuild_command.rb | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/lib/rubygems/commands/rebuild_command.rb b/lib/rubygems/commands/rebuild_command.rb index 234d0af85617f2..9747171e5cdfd5 100644 --- a/lib/rubygems/commands/rebuild_command.rb +++ b/lib/rubygems/commands/rebuild_command.rb @@ -11,6 +11,10 @@ class Gem::Commands::RebuildCommand < Gem::Command def initialize super "rebuild", "Attempt to reproduce a build of a gem." + add_option "--diff", "If the files don't match, compare them using diffoscope." do |_value, options| + options[:diff] = true + end + add_option "--force", "Skip validation of the spec." do |_value, options| options[:force] = true end @@ -150,6 +154,14 @@ def compare(source_date_epoch, old_file, new_file) say "SUCCESS - original and rebuild hashes matched" else say "FAILURE - original and rebuild hashes did not match" + + if options[:diff] + say + if system("diffoscope", old_file, new_file).nil? + alert_error "error: could not find `diffoscope` executable" + end + end + terminate_interaction 1 end end From 54e0b8073b1236689ac0a2506b62b8b80535cf73 Mon Sep 17 00:00:00 2001 From: Ellen Marie Dash Date: Tue, 30 Jan 2024 23:12:04 -0500 Subject: [PATCH 150/211] [rubygems/rubygems] [rebuild_command] Use temporary directory instead of the working directory. https://github.com/rubygems/rubygems/commit/f2e4e5b56f --- lib/rubygems/commands/rebuild_command.rb | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/lib/rubygems/commands/rebuild_command.rb b/lib/rubygems/commands/rebuild_command.rb index 9747171e5cdfd5..295d3f9c880720 100644 --- a/lib/rubygems/commands/rebuild_command.rb +++ b/lib/rubygems/commands/rebuild_command.rb @@ -3,6 +3,7 @@ require "date" require "digest" require "fileutils" +require "tmpdir" require_relative "../package" class Gem::Commands::RebuildCommand < Gem::Command @@ -167,14 +168,10 @@ def compare(source_date_epoch, old_file, new_file) end def prep_dirs - rebuild_dir = File.expand_path("rebuild") + rebuild_dir = Dir.mktmpdir("gem_rebuild") old_dir = File.join(rebuild_dir, "old") new_dir = File.join(rebuild_dir, "new") - if File.directory?(rebuild_dir) - FileUtils.remove_dir(rebuild_dir) - end - FileUtils.mkdir_p(old_dir) FileUtils.mkdir_p(new_dir) From fe096f64e881cd3426e766eb999740ed0fbfa4ba Mon Sep 17 00:00:00 2001 From: Ellen Marie Dash Date: Tue, 30 Jan 2024 23:18:48 -0500 Subject: [PATCH 151/211] [rubygems/rubygems] [rebuild_command] Clean up help text. https://github.com/rubygems/rubygems/commit/4446389f2e --- lib/rubygems/commands/rebuild_command.rb | 35 ++++-------------------- 1 file changed, 6 insertions(+), 29 deletions(-) diff --git a/lib/rubygems/commands/rebuild_command.rb b/lib/rubygems/commands/rebuild_command.rb index 295d3f9c880720..16c3013effc36a 100644 --- a/lib/rubygems/commands/rebuild_command.rb +++ b/lib/rubygems/commands/rebuild_command.rb @@ -55,35 +55,12 @@ def description # :nodoc: If you use either `gem build` or `rake build`/`rake release` to build/release a gem, it is a potential candidate. -If the gem includes top-level files that change frequently (e.g. Gemfile.lock), -it may require more effort to reproduce a build. For example, it might require -more precisely matched versions of Ruby, RubyGems, or Bundler to be used. - -An example of reproducing a gem build: - - $ pwd - /usr/home/puppy/test/rebuild - $ ls -a - ./ ../ - $ git clone --branch v12.0.2 https://github.com/duckinator/okay.git - $ gem rebuild -C ./okay --gemspec okay.gemspec okay 12.0.2 - Fetching okay-12.0.2.gem - Downloaded okay version 12.0.2 as /usr/home/puppy/test/rebuild/rebuild/old/okay-12.0.2.gem. - Successfully built RubyGem - Name: okay - Version: 12.0.2 - File: okay-12.0.2.gem - - Built at: 2023-08-31 21:39:02 EDT (1693532342) - Original build saved to: /usr/home/puppy/test/rebuild/rebuild/old/okay-12.0.2.gem - Reproduced build saved to: /usr/home/puppy/test/rebuild/rebuild/new/okay-12.0.2.gem - Working directory: ./okay - - Hash comparison: - 38a8bd78ce10bc19189ead0b56fa490d03788a2f926a6481b2f1f5d5fa5ab75b /usr/home/puppy/test/rebuild/rebuild/old/okay-12.0.2.gem - 38a8bd78ce10bc19189ead0b56fa490d03788a2f926a6481b2f1f5d5fa5ab75b /usr/home/puppy/test/rebuild/rebuild/new/okay-12.0.2.gem - - SUCCESS - original and rebuild hashes matched +You will likely need to match the RubyGems version used, since this is +included in the Gem metadata. + +If the gem includes lockfiles (e.g. Gemfile.lock) and similar, it will require +more effort to reproduce a build. For example, it might require more precisely +matched versions of Ruby and/or Bundler to be used. EOF end From 88d7be46b5be03a77686f507ec3e8d11a9dafe60 Mon Sep 17 00:00:00 2001 From: Ellen Marie Dash Date: Wed, 31 Jan 2024 12:34:20 -0500 Subject: [PATCH 152/211] [rubygems/rubygems] [rebuild_command] Use Gem.* helpers. https://github.com/rubygems/rubygems/commit/8644ce7193 --- lib/rubygems/commands/rebuild_command.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/rubygems/commands/rebuild_command.rb b/lib/rubygems/commands/rebuild_command.rb index 16c3013effc36a..4a4f76678d3a00 100644 --- a/lib/rubygems/commands/rebuild_command.rb +++ b/lib/rubygems/commands/rebuild_command.rb @@ -97,12 +97,12 @@ def execute private def sha256(file) - Digest::SHA256.hexdigest(File.read(file)) + Digest::SHA256.hexdigest(Gem.read_binary(file)) end def get_timestamp(file) mtime = nil - File.open(file, "rb") do |f| + File.open(file, Gem.binary_mode) do |f| Gem::Package::TarReader.new(f) do |tar| mtime = tar.seek("metadata.gz") {|tf| tf.header.mtime } end From dfe83df03e1a9a056070ef37999f150bcf5a6fc0 Mon Sep 17 00:00:00 2001 From: Ellen Marie Dash Date: Wed, 31 Jan 2024 13:07:07 -0500 Subject: [PATCH 153/211] [rubygems/rubygems] [rebuild_command] Bail early if the RubyGems version doesn't match. https://github.com/rubygems/rubygems/commit/a691170dc7 --- lib/rubygems/commands/rebuild_command.rb | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/lib/rubygems/commands/rebuild_command.rb b/lib/rubygems/commands/rebuild_command.rb index 4a4f76678d3a00..5386354f1038b4 100644 --- a/lib/rubygems/commands/rebuild_command.rb +++ b/lib/rubygems/commands/rebuild_command.rb @@ -83,6 +83,22 @@ def execute download_gem(gem_name, gem_version, old_file) end + rg_version = rubygems_version(old_file) + unless rg_version == Gem::VERSION + alert_error <<-EOF +You need to use the same RubyGems version #{gem_name} v#{gem_version} was built with. + +#{gem_name} v#{gem_version} was built using RubyGems v#{rg_version}. +Gem files include the version of RubyGems used to build them. +This means in order to reproduce #{gem_filename}, you must also use RubyGems v#{rg_version}. + +You're using RubyGems v#{Gem::VERSION}. + +Please install RubyGems v#{rg_version} and try again. + EOF + terminate_interaction 1 + end + source_date_epoch = get_timestamp(old_file).to_s if build_path = options[:build_path] @@ -247,4 +263,8 @@ def download_gem(gem_name, gem_version, old_file) say "Downloaded #{gem_name} version #{gem_version} as #{old_file}." end + + def rubygems_version(gem_file) + Gem::Package.new(gem_file).spec.rubygems_version + end end From 54d90e1355180587bf7dda8f56d5d59600a7da23 Mon Sep 17 00:00:00 2001 From: Ellen Marie Dash Date: Tue, 27 Feb 2024 19:38:43 -0500 Subject: [PATCH 154/211] [rubygems/rubygems] [rebuild] If --diff is not passed and a rebuild fails, suggest passing --diff. https://github.com/rubygems/rubygems/commit/7caadd182c --- lib/rubygems/commands/rebuild_command.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/rubygems/commands/rebuild_command.rb b/lib/rubygems/commands/rebuild_command.rb index 5386354f1038b4..419e9bfdb0837b 100644 --- a/lib/rubygems/commands/rebuild_command.rb +++ b/lib/rubygems/commands/rebuild_command.rb @@ -154,6 +154,9 @@ def compare(source_date_epoch, old_file, new_file) if system("diffoscope", old_file, new_file).nil? alert_error "error: could not find `diffoscope` executable" end + else + say + say "Pass --diff for more details (requires diffoscope to be installed)." end terminate_interaction 1 From d916dbcb849b816b15717c64ea0c0ce756c15fb5 Mon Sep 17 00:00:00 2001 From: Ellen Marie Dash Date: Tue, 27 Feb 2024 19:48:41 -0500 Subject: [PATCH 155/211] [rubygems/rubygems] Improve formatting of "gem rebuild --help" output. https://github.com/rubygems/rubygems/commit/701550f9dd --- lib/rubygems/commands/rebuild_command.rb | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/lib/rubygems/commands/rebuild_command.rb b/lib/rubygems/commands/rebuild_command.rb index 419e9bfdb0837b..4163e0d244f1c8 100644 --- a/lib/rubygems/commands/rebuild_command.rb +++ b/lib/rubygems/commands/rebuild_command.rb @@ -52,15 +52,15 @@ def description # :nodoc: from a ruby gemspec. This command assumes the gemspec can be built with the `gem build` command. -If you use either `gem build` or `rake build`/`rake release` to build/release -a gem, it is a potential candidate. +If you use any of `gem build`, `rake build`, or`rake release` in the +build/release process for a gem, it is a potential candidate. -You will likely need to match the RubyGems version used, since this is -included in the Gem metadata. +You will need to match the RubyGems version used, since this is included in +the Gem metadata. -If the gem includes lockfiles (e.g. Gemfile.lock) and similar, it will require -more effort to reproduce a build. For example, it might require more precisely -matched versions of Ruby and/or Bundler to be used. +If the gem includes lockfiles (e.g. Gemfile.lock) and similar, it will +require more effort to reproduce a build. For example, it might require +more precisely matched versions of Ruby and/or Bundler to be used. EOF end @@ -148,14 +148,13 @@ def compare(source_date_epoch, old_file, new_file) say "SUCCESS - original and rebuild hashes matched" else say "FAILURE - original and rebuild hashes did not match" + say if options[:diff] - say if system("diffoscope", old_file, new_file).nil? alert_error "error: could not find `diffoscope` executable" end else - say say "Pass --diff for more details (requires diffoscope to be installed)." end From d19744fbd6e8ede579eb73109e6b2ec936a6ab43 Mon Sep 17 00:00:00 2001 From: Ellen Marie Dash Date: Thu, 29 Feb 2024 18:38:40 -0500 Subject: [PATCH 156/211] [rubygems/rubygems] [build, rebuild] Split common find_gemspec() out to GemspecHelpers. https://github.com/rubygems/rubygems/commit/2f80a595c4 --- lib/rubygems/commands/build_command.rb | 13 ++----------- lib/rubygems/commands/rebuild_command.rb | 14 +++----------- lib/rubygems/gemspec_helpers.rb | 20 ++++++++++++++++++++ 3 files changed, 25 insertions(+), 22 deletions(-) create mode 100644 lib/rubygems/gemspec_helpers.rb diff --git a/lib/rubygems/commands/build_command.rb b/lib/rubygems/commands/build_command.rb index 0ebdec565b5606..2ec83241418d13 100644 --- a/lib/rubygems/commands/build_command.rb +++ b/lib/rubygems/commands/build_command.rb @@ -1,11 +1,13 @@ # frozen_string_literal: true require_relative "../command" +require_relative "../gemspec_helpers" require_relative "../package" require_relative "../version_option" class Gem::Commands::BuildCommand < Gem::Command include Gem::VersionOption + include Gem::GemspecHelpers def initialize super "build", "Build a gem from a gemspec" @@ -75,17 +77,6 @@ def execute private - def find_gemspec(glob = "*.gemspec") - gemspecs = Dir.glob(glob).sort - - if gemspecs.size > 1 - alert_error "Multiple gemspecs found: #{gemspecs}, please specify one" - terminate_interaction(1) - end - - gemspecs.first - end - def build_gem gemspec = resolve_gem_name diff --git a/lib/rubygems/commands/rebuild_command.rb b/lib/rubygems/commands/rebuild_command.rb index 4163e0d244f1c8..97f05ef79c84aa 100644 --- a/lib/rubygems/commands/rebuild_command.rb +++ b/lib/rubygems/commands/rebuild_command.rb @@ -4,9 +4,12 @@ require "digest" require "fileutils" require "tmpdir" +require_relative "../gemspec_helpers" require_relative "../package" class Gem::Commands::RebuildCommand < Gem::Command + include Gem::GemspecHelpers + DATE_FORMAT = "%Y-%m-%d %H:%M:%S.%N Z" def initialize @@ -223,17 +226,6 @@ def with_source_date_epoch(source_date_epoch) ENV["SOURCE_DATE_EPOCH"] = old_sde end - def find_gemspec(glob = "*.gemspec") - gemspecs = Dir.glob(glob).sort - - if gemspecs.size > 1 - alert_error "Multiple gemspecs found: #{gemspecs}, please specify one" - terminate_interaction(1) - end - - gemspecs.first - end - def error_message(gem_name) if gem_name "Couldn't find a gemspec file matching '#{gem_name}' in #{Dir.pwd}" diff --git a/lib/rubygems/gemspec_helpers.rb b/lib/rubygems/gemspec_helpers.rb new file mode 100644 index 00000000000000..a4ce0756e3a7e8 --- /dev/null +++ b/lib/rubygems/gemspec_helpers.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +require_relative "../rubygems" + +## +# Mixin methods for commands that work with gemspecs. + +module Gem::GemspecHelpers + def find_gemspec(glob = "*.gemspec") + gemspecs = Dir.glob(glob).sort + + if gemspecs.size > 1 + alert_error "Multiple gemspecs found: #{gemspecs}, please specify one" + terminate_interaction(1) + end + + gemspecs.first + end + +end From 38331c8981cd692669392f3d02b09e15b8f892b1 Mon Sep 17 00:00:00 2001 From: Ellen Marie Dash Date: Thu, 29 Feb 2024 18:47:39 -0500 Subject: [PATCH 157/211] [rubygems/rubygems] [gemspec_helpers] Fix Rubocop warning. https://github.com/rubygems/rubygems/commit/4ebf6ee5ac --- lib/rubygems/gemspec_helpers.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/rubygems/gemspec_helpers.rb b/lib/rubygems/gemspec_helpers.rb index a4ce0756e3a7e8..2b20fcafa12120 100644 --- a/lib/rubygems/gemspec_helpers.rb +++ b/lib/rubygems/gemspec_helpers.rb @@ -16,5 +16,4 @@ def find_gemspec(glob = "*.gemspec") gemspecs.first end - end From 8191735b73eac4486eebac6530bd92080ee23b9a Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Thu, 28 Mar 2024 14:15:26 -0400 Subject: [PATCH 158/211] [PRISM] Fix BEGIN{} execution order --- prism_compile.c | 74 ++++++++++++++++++++++++++++++++++++++++-------- prism_compile.h | 9 ++++++ spec/prism.mspec | 1 - 3 files changed, 71 insertions(+), 13 deletions(-) diff --git a/prism_compile.c b/prism_compile.c index 81b1e7dd11d42c..5aeef191f780a7 100644 --- a/prism_compile.c +++ b/prism_compile.c @@ -6808,27 +6808,38 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, // ^^^^^^^^ const pm_pre_execution_node_t *cast = (const pm_pre_execution_node_t *) node; - DECL_ANCHOR(pre_ex); - INIT_ANCHOR(pre_ex); + LINK_ANCHOR *outer_pre = scope_node->pre_execution_anchor; + RUBY_ASSERT(outer_pre != NULL); + + // BEGIN{} nodes can be nested, so here we're going to do the same thing + // that we did for the top-level compilation where we create two + // anchors and then join them in the correct order into the resulting + // anchor. + DECL_ANCHOR(inner_pre); + INIT_ANCHOR(inner_pre); + scope_node->pre_execution_anchor = inner_pre; + + DECL_ANCHOR(inner_body); + INIT_ANCHOR(inner_body); if (cast->statements != NULL) { const pm_node_list_t *body = &cast->statements->body; + for (size_t index = 0; index < body->size; index++) { - pm_compile_node(iseq, body->nodes[index], pre_ex, true, scope_node); + pm_compile_node(iseq, body->nodes[index], inner_body, true, scope_node); } } if (!popped) { - PUSH_INSN(pre_ex, location, putnil); + PUSH_INSN(inner_body, location, putnil); } - pre_ex->last->next = ret->anchor.next; - ret->anchor.next = pre_ex->anchor.next; - ret->anchor.next->prev = pre_ex->anchor.next; - - if (ret->last == (LINK_ELEMENT *)ret) { - ret->last = pre_ex->last; - } + // Now that everything has been compiled, join both anchors together + // into the correct outer pre execution anchor, and reset the value so + // that subsequent BEGIN{} nodes can be compiled correctly. + ADD_SEQ(outer_pre, inner_pre); + ADD_SEQ(outer_pre, inner_body); + scope_node->pre_execution_anchor = outer_pre; return; } @@ -8342,6 +8353,20 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, } } +/** True if the given iseq can have pre execution blocks. */ +static inline bool +pm_iseq_pre_execution_p(rb_iseq_t *iseq) +{ + switch (ISEQ_BODY(iseq)->type) { + case ISEQ_TYPE_TOP: + case ISEQ_TYPE_EVAL: + case ISEQ_TYPE_MAIN: + return true; + default: + return false; + } +} + /** * This is the main entry-point into the prism compiler. It accepts the iseq * that it should be compiling instruction into and a pointer to the scope node @@ -8355,7 +8380,32 @@ pm_iseq_compile_node(rb_iseq_t *iseq, pm_scope_node_t *node) DECL_ANCHOR(ret); INIT_ANCHOR(ret); - pm_compile_node(iseq, (const pm_node_t *) node, ret, false, node); + if (pm_iseq_pre_execution_p(iseq)) { + // Because these ISEQs can have BEGIN{}, we're going to create two + // anchors to compile them, a "pre" and a "body". We'll mark the "pre" + // on the scope node so that when BEGIN{} is found, its contents will be + // added to the "pre" anchor. + DECL_ANCHOR(pre); + INIT_ANCHOR(pre); + node->pre_execution_anchor = pre; + + // Now we'll compile the body as normal. We won't compile directly into + // the "ret" anchor yet because we want to add the "pre" anchor to the + // beginning of the "ret" anchor first. + DECL_ANCHOR(body); + INIT_ANCHOR(body); + pm_compile_node(iseq, (const pm_node_t *) node, body, false, node); + + // Now we'll join both anchors together so that the content is in the + // correct order. + ADD_SEQ(ret, pre); + ADD_SEQ(ret, body); + } + else { + // In other circumstances, we can just compile the node directly into + // the "ret" anchor. + pm_compile_node(iseq, (const pm_node_t *) node, ret, false, node); + } CHECK(iseq_setup_insn(iseq, ret)); return iseq_setup(iseq, ret); diff --git a/prism_compile.h b/prism_compile.h index 32f8c128442820..e58bed271f2b66 100644 --- a/prism_compile.h +++ b/prism_compile.h @@ -13,6 +13,9 @@ typedef struct pm_local_index_struct { int index, level; } pm_local_index_t; +// A declaration for the struct that lives in compile.c. +struct iseq_link_anchor; + // ScopeNodes are helper nodes, and will never be part of the AST. We manually // declare them here to avoid generating them. typedef struct pm_scope_node { @@ -40,6 +43,12 @@ typedef struct pm_scope_node { ID *constants; st_table *index_lookup_table; + + /** + * This will only be set on the top-level scope node. It will contain all of + * the instructions pertaining to BEGIN{} nodes. + */ + struct iseq_link_anchor *pre_execution_anchor; } pm_scope_node_t; void pm_scope_node_init(const pm_node_t *node, pm_scope_node_t *scope, pm_scope_node_t *previous); diff --git a/spec/prism.mspec b/spec/prism.mspec index b1b0fa8c6bd958..1d4d6f3f21adb1 100644 --- a/spec/prism.mspec +++ b/spec/prism.mspec @@ -1,7 +1,6 @@ # frozen_string_literal: true ## Language -MSpec.register(:exclude, "The BEGIN keyword runs multiple begins in FIFO order") MSpec.register(:exclude, "Executing break from within a block works when passing through a super call") MSpec.register(:exclude, "The defined? keyword when called with a method name in a void context warns about the void context when parsing it") MSpec.register(:exclude, "Hash literal expands an '**{}' or '**obj' element with the last key/value pair taking precedence") From 718c7d4a378f41695a634adc87be30b1ab05a27d Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 29 Mar 2024 11:30:30 -0400 Subject: [PATCH 159/211] [ruby/prism] Handle NULL byte terminators for strings, regexps, and lists https://github.com/ruby/prism/commit/79a75258a4 --- prism/prism.c | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/prism/prism.c b/prism/prism.c index db71a9c3f46b1f..67f530edd9b6a0 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -260,10 +260,13 @@ lex_mode_push_list(pm_parser_t *parser, bool interpolation, uint8_t delimiter) { // We'll use strpbrk to find the first of these characters. uint8_t *breakpoints = lex_mode.as.list.breakpoints; memcpy(breakpoints, "\\ \t\f\r\v\n\0\0\0", sizeof(lex_mode.as.list.breakpoints)); - - // Now we'll add the terminator to the list of breakpoints. size_t index = 7; - breakpoints[index++] = terminator; + + // Now we'll add the terminator to the list of breakpoints. If the + // terminator is not already a NULL byte, add it to the list. + if (terminator != '\0') { + breakpoints[index++] = terminator; + } // If interpolation is allowed, then we're going to check for the # // character. Otherwise we'll only look for escapes and the terminator. @@ -309,13 +312,16 @@ lex_mode_push_regexp(pm_parser_t *parser, uint8_t incrementor, uint8_t terminato // characters. uint8_t *breakpoints = lex_mode.as.regexp.breakpoints; memcpy(breakpoints, "\r\n\\#\0\0", sizeof(lex_mode.as.regexp.breakpoints)); + size_t index = 4; // First we'll add the terminator. - breakpoints[4] = terminator; + if (terminator != '\0') { + breakpoints[index++] = terminator; + } // Next, if there is an incrementor, then we'll check for that as well. if (incrementor != '\0') { - breakpoints[5] = incrementor; + breakpoints[index++] = incrementor; } return lex_mode_push(parser, lex_mode); @@ -341,10 +347,13 @@ lex_mode_push_string(pm_parser_t *parser, bool interpolation, bool label_allowed // string. We'll use strpbrk to find the first of these characters. uint8_t *breakpoints = lex_mode.as.string.breakpoints; memcpy(breakpoints, "\r\n\\\0\0\0", sizeof(lex_mode.as.string.breakpoints)); - - // Now add in the terminator. size_t index = 3; - breakpoints[index++] = terminator; + + // Now add in the terminator. If the terminator is not already a NULL byte, + // then we'll add it. + if (terminator != '\0') { + breakpoints[index++] = terminator; + } // If interpolation is allowed, then we're going to check for the # // character. Otherwise we'll only look for escapes and the terminator. @@ -10752,12 +10761,6 @@ parser_lex(pm_parser_t *parser) { pm_token_buffer_t token_buffer = { 0 }; while (breakpoint != NULL) { - // If we hit a null byte, skip directly past it. - if (*breakpoint == '\0') { - breakpoint = pm_strpbrk(parser, breakpoint + 1, breakpoints, parser->end - (breakpoint + 1), true); - continue; - } - // If we hit whitespace, then we must have received content by // now, so we can return an element of the list. if (pm_char_is_whitespace(*breakpoint)) { @@ -10794,6 +10797,12 @@ parser_lex(pm_parser_t *parser) { LEX(PM_TOKEN_STRING_END); } + // If we hit a null byte, skip directly past it. + if (*breakpoint == '\0') { + breakpoint = pm_strpbrk(parser, breakpoint + 1, breakpoints, parser->end - (breakpoint + 1), true); + continue; + } + // If we hit escapes, then we need to treat the next token // literally. In this case we'll skip past the next character // and find the next breakpoint. From 729a39685b68a53a9f91ca972e6c7c393aed1b52 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 29 Mar 2024 11:33:45 -0400 Subject: [PATCH 160/211] [ruby/prism] Fix calloc argument order https://github.com/ruby/prism/commit/9947ab13c0 --- prism/prism.c | 4 ++-- prism/templates/src/diagnostic.c.erb | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/prism/prism.c b/prism/prism.c index 67f530edd9b6a0..68e016c22a3f50 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -7591,7 +7591,7 @@ parser_lex_magic_comment(pm_parser_t *parser, bool semantic_token_seen) { // Allocate a new magic comment node to append to the parser's list. pm_magic_comment_t *magic_comment; - if ((magic_comment = (pm_magic_comment_t *) xcalloc(sizeof(pm_magic_comment_t), 1)) != NULL) { + if ((magic_comment = (pm_magic_comment_t *) xcalloc(1, sizeof(pm_magic_comment_t))) != NULL) { magic_comment->key_start = key_start; magic_comment->value_start = value_start; magic_comment->key_length = (uint32_t) key_length; @@ -9086,7 +9086,7 @@ parser_lex_callback(pm_parser_t *parser) { */ static inline pm_comment_t * parser_comment(pm_parser_t *parser, pm_comment_type_t type) { - pm_comment_t *comment = (pm_comment_t *) xcalloc(sizeof(pm_comment_t), 1); + pm_comment_t *comment = (pm_comment_t *) xcalloc(1, sizeof(pm_comment_t)); if (comment == NULL) return NULL; *comment = (pm_comment_t) { diff --git a/prism/templates/src/diagnostic.c.erb b/prism/templates/src/diagnostic.c.erb index 4e6b6ec4eefb18..8badf579e5031c 100644 --- a/prism/templates/src/diagnostic.c.erb +++ b/prism/templates/src/diagnostic.c.erb @@ -383,7 +383,7 @@ pm_diagnostic_level(pm_diagnostic_id_t diag_id) { */ bool pm_diagnostic_list_append(pm_list_t *list, const uint8_t *start, const uint8_t *end, pm_diagnostic_id_t diag_id) { - pm_diagnostic_t *diagnostic = (pm_diagnostic_t *) xcalloc(sizeof(pm_diagnostic_t), 1); + pm_diagnostic_t *diagnostic = (pm_diagnostic_t *) xcalloc(1, sizeof(pm_diagnostic_t)); if (diagnostic == NULL) return false; *diagnostic = (pm_diagnostic_t) { @@ -415,7 +415,7 @@ pm_diagnostic_list_append_format(pm_list_t *list, const uint8_t *start, const ui return false; } - pm_diagnostic_t *diagnostic = (pm_diagnostic_t *) xcalloc(sizeof(pm_diagnostic_t), 1); + pm_diagnostic_t *diagnostic = (pm_diagnostic_t *) xcalloc(1, sizeof(pm_diagnostic_t)); if (diagnostic == NULL) { return false; } From f57c7fef6b260ca7d04458efd2f9e9fd784c9d89 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 29 Mar 2024 11:16:45 -0400 Subject: [PATCH 161/211] [PRISM] Have RubyVM::InstructionSequence.compile respect --parser=prism --- .github/workflows/prism.yml | 2 +- iseq.c | 80 ++++++++++---------- test/.excludes-prism/TestCall.rb | 3 + test/.excludes-prism/TestClass.rb | 3 + test/.excludes-prism/TestIRB/RubyLexTest.rb | 1 + test/.excludes-prism/TestISeq.rb | 6 ++ test/.excludes-prism/TestIseqLoad.rb | 5 ++ test/.excludes-prism/TestParse.rb | 37 +++++++++ test/.excludes-prism/TestPatternMatching.rb | 4 + test/.excludes-prism/TestRubyLiteral.rb | 4 + test/.excludes-prism/TestRubyOptimization.rb | 1 + test/.excludes-prism/TestRubyVM.rb | 1 + test/.excludes-prism/TestSyntax.rb | 45 +++++++++++ 13 files changed, 151 insertions(+), 41 deletions(-) create mode 100644 test/.excludes-prism/TestCall.rb create mode 100644 test/.excludes-prism/TestClass.rb create mode 100644 test/.excludes-prism/TestIRB/RubyLexTest.rb create mode 100644 test/.excludes-prism/TestIseqLoad.rb create mode 100644 test/.excludes-prism/TestRubyOptimization.rb create mode 100644 test/.excludes-prism/TestRubyVM.rb diff --git a/.github/workflows/prism.yml b/.github/workflows/prism.yml index 8b295f7f815349..7fc10e2ab12775 100644 --- a/.github/workflows/prism.yml +++ b/.github/workflows/prism.yml @@ -92,7 +92,7 @@ jobs: timeout-minutes: 40 env: GNUMAKEFLAGS: '' - RUBY_TESTOPTS: '-q --tty=no --excludes-dir="../src/test/.excludes-prism" --exclude="test_ast.rb" --exclude="error_highlight/test_error_highlight.rb" --exclude="prism/encoding_test.rb"' + RUBY_TESTOPTS: '-q --tty=no --excludes-dir="../src/test/.excludes-prism" --exclude="test_ast.rb" --exclude="error_highlight/test_error_highlight.rb" --exclude="prism/encoding_test.rb" --exclude="prism/locals_test.rb" --exclude="prism/newline_test.rb"' RUN_OPTS: ${{ matrix.run_opts }} - name: make test-prism-spec diff --git a/iseq.c b/iseq.c index 2d412dfcc9265d..fffdb79edc7ff2 100644 --- a/iseq.c +++ b/iseq.c @@ -1430,6 +1430,44 @@ rb_iseqw_new(const rb_iseq_t *iseq) return iseqw_new(iseq); } +/** + * Accept the options given to InstructionSequence.compile and + * InstructionSequence.compile_prism and share the logic for creating the + * instruction sequence. + */ +static VALUE +iseqw_s_compile_parser(int argc, VALUE *argv, VALUE self, bool prism) +{ + VALUE src, file = Qnil, path = Qnil, line = Qnil, opt = Qnil; + int i; + + i = rb_scan_args(argc, argv, "1*:", &src, NULL, &opt); + if (i > 4+NIL_P(opt)) rb_error_arity(argc, 1, 5); + switch (i) { + case 5: opt = argv[--i]; + case 4: line = argv[--i]; + case 3: path = argv[--i]; + case 2: file = argv[--i]; + } + + if (NIL_P(file)) file = rb_fstring_lit(""); + if (NIL_P(path)) path = file; + if (NIL_P(line)) line = INT2FIX(1); + + Check_Type(path, T_STRING); + Check_Type(file, T_STRING); + + rb_iseq_t *iseq; + if (prism) { + iseq = pm_iseq_compile_with_option(src, file, path, line, opt); + } + else { + iseq = rb_iseq_compile_with_option(src, file, path, line, opt); + } + + return iseqw_new(iseq); +} + /* * call-seq: * InstructionSequence.compile(source[, file[, path[, line[, options]]]]) -> iseq @@ -1470,26 +1508,7 @@ rb_iseqw_new(const rb_iseq_t *iseq) static VALUE iseqw_s_compile(int argc, VALUE *argv, VALUE self) { - VALUE src, file = Qnil, path = Qnil, line = Qnil, opt = Qnil; - int i; - - i = rb_scan_args(argc, argv, "1*:", &src, NULL, &opt); - if (i > 4+NIL_P(opt)) rb_error_arity(argc, 1, 5); - switch (i) { - case 5: opt = argv[--i]; - case 4: line = argv[--i]; - case 3: path = argv[--i]; - case 2: file = argv[--i]; - } - - if (NIL_P(file)) file = rb_fstring_lit(""); - if (NIL_P(path)) path = file; - if (NIL_P(line)) line = INT2FIX(1); - - Check_Type(path, T_STRING); - Check_Type(file, T_STRING); - - return iseqw_new(rb_iseq_compile_with_option(src, file, path, line, opt)); + return iseqw_s_compile_parser(argc, argv, self, *rb_ruby_prism_ptr()); } /* @@ -1531,26 +1550,7 @@ iseqw_s_compile(int argc, VALUE *argv, VALUE self) static VALUE iseqw_s_compile_prism(int argc, VALUE *argv, VALUE self) { - VALUE src, file = Qnil, path = Qnil, line = Qnil, opt = Qnil; - int i; - - i = rb_scan_args(argc, argv, "1*:", &src, NULL, &opt); - if (i > 4+NIL_P(opt)) rb_error_arity(argc, 1, 5); - switch (i) { - case 5: opt = argv[--i]; - case 4: line = argv[--i]; - case 3: path = argv[--i]; - case 2: file = argv[--i]; - } - - if (NIL_P(file)) file = rb_fstring_lit(""); - if (NIL_P(path)) path = file; - if (NIL_P(line)) line = INT2FIX(1); - - Check_Type(path, T_STRING); - Check_Type(file, T_STRING); - - return iseqw_new(pm_iseq_compile_with_option(src, file, path, line, opt)); + return iseqw_s_compile_parser(argc, argv, self, true); } /* diff --git a/test/.excludes-prism/TestCall.rb b/test/.excludes-prism/TestCall.rb new file mode 100644 index 00000000000000..08a949fc017475 --- /dev/null +++ b/test/.excludes-prism/TestCall.rb @@ -0,0 +1,3 @@ +exclude(:test_call_op_asgn_keywords, "unknown") +exclude(:test_kwsplat_block_order_op_asgn, "unknown") +exclude(:test_call_op_asgn_keywords_mutable, "unknown") diff --git a/test/.excludes-prism/TestClass.rb b/test/.excludes-prism/TestClass.rb new file mode 100644 index 00000000000000..14b768b9b709e0 --- /dev/null +++ b/test/.excludes-prism/TestClass.rb @@ -0,0 +1,3 @@ +exclude(:test_invalid_break_from_class_definition, "unknown") +exclude(:test_invalid_next_from_class_definition, "unknown") +exclude(:test_invalid_return_from_class_definition, "unknown") diff --git a/test/.excludes-prism/TestIRB/RubyLexTest.rb b/test/.excludes-prism/TestIRB/RubyLexTest.rb new file mode 100644 index 00000000000000..2274ae62cfd589 --- /dev/null +++ b/test/.excludes-prism/TestIRB/RubyLexTest.rb @@ -0,0 +1 @@ +exclude(:test_code_block_open_with_should_continue, "symbol encoding") diff --git a/test/.excludes-prism/TestISeq.rb b/test/.excludes-prism/TestISeq.rb index de7052cb112d1f..71d158367792bb 100644 --- a/test/.excludes-prism/TestISeq.rb +++ b/test/.excludes-prism/TestISeq.rb @@ -1 +1,7 @@ +exclude(:test_each_child, "unknown") +exclude(:test_frozen_string_literal_compile_option, "unknown") exclude(:test_syntax_error_message, "unknown") +exclude(:test_to_binary_class_tracepoint, "unknown") +exclude(:test_to_binary_end_tracepoint, "unknown") +exclude(:test_trace_points, "unknown") +exclude(:test_unreachable_syntax_error, "unknown") diff --git a/test/.excludes-prism/TestIseqLoad.rb b/test/.excludes-prism/TestIseqLoad.rb new file mode 100644 index 00000000000000..036e03f7e4f500 --- /dev/null +++ b/test/.excludes-prism/TestIseqLoad.rb @@ -0,0 +1,5 @@ +exclude(:test_bug8543, "unknown") +exclude(:test_case_when, "unknown") +exclude(:test_hidden, "unknown") +exclude(:test_kwarg, "unknown") +exclude(:test_splatsplat, "unknown") diff --git a/test/.excludes-prism/TestParse.rb b/test/.excludes-prism/TestParse.rb index f2ef81cd119693..126ead1800e152 100644 --- a/test/.excludes-prism/TestParse.rb +++ b/test/.excludes-prism/TestParse.rb @@ -1,7 +1,44 @@ +exclude(:test_alias_backref, "unknown") +exclude(:test_bad_arg, "unknown") +exclude(:test_block_dup, "unknown") +exclude(:test_class_module, "unknown") +exclude(:test_disallowed_class_variable, "unknown") +exclude(:test_disallowed_gloal_variable, "unknown") +exclude(:test_disallowed_instance_variable, "unknown") +exclude(:test_duplicate_argument, "unknown") +exclude(:test_dynamic_constant_assignment, "unknown") +exclude(:test_else_without_rescue, "unknown") +exclude(:test_embedded_rd_error, "unknown") +exclude(:test_embedded_rd, "unknown") +exclude(:test_error_def_in_argument, "unknown") +exclude(:test_float, "unknown") +exclude(:test_global_variable, "unknown") +exclude(:test_here_document, "unknown") +exclude(:test_heredoc_unterminated_interpolation, "unknown") +exclude(:test_invalid_break_from_class_definition, "unknown") +exclude(:test_invalid_char, "unknown") +exclude(:test_invalid_class_variable, "unknown") +exclude(:test_invalid_instance_variable, "unknown") +exclude(:test_invalid_next_from_class_definition, "unknown") +exclude(:test_location_of_invalid_token, "unknown") +exclude(:test_no_blockarg, "unknown") +exclude(:test_op_asgn1_with_block, "unknown") +exclude(:test_parse_string, "unknown") +exclude(:test_percent, "unknown") +exclude(:test_question, "unknown") +exclude(:test_shareable_constant_value_ignored, "unknown") exclude(:test_shareable_constant_value_nested, "ractor support") exclude(:test_shareable_constant_value_nonliteral, "ractor support") exclude(:test_shareable_constant_value_simple, "ractor support") exclude(:test_shareable_constant_value_unfrozen, "ractor support") exclude(:test_shareable_constant_value_unshareable_literal, "ractor support") +exclude(:test_string, "unknown") +exclude(:test_truncated_source_line, "unknown") +exclude(:test_unassignable, "unknown") +exclude(:test_unexpected_eof, "unknown") +exclude(:test_unexpected_token_after_numeric, "unknown") +exclude(:test_unterminated_regexp_error, "unknown") exclude(:test_unused_variable, "missing warning") exclude(:test_void_expr_stmts_value, "missing warning") +exclude(:test_void_value_in_rhs, "unknown") +exclude(:test_words, "unknown") diff --git a/test/.excludes-prism/TestPatternMatching.rb b/test/.excludes-prism/TestPatternMatching.rb index cb64a6d818787c..40d9d6d99cbde0 100644 --- a/test/.excludes-prism/TestPatternMatching.rb +++ b/test/.excludes-prism/TestPatternMatching.rb @@ -1 +1,5 @@ +exclude(:test_array_pattern, "duplicated variable error missing") +exclude(:test_find_pattern, "duplicated variable error missing") exclude(:test_hash_pattern, "useless literal warning missing") +exclude(:test_invalid_syntax, "duplicated variable error missing") +exclude(:test_var_pattern, "duplicated variable error missing") diff --git a/test/.excludes-prism/TestRubyLiteral.rb b/test/.excludes-prism/TestRubyLiteral.rb index a5ad6d6d9d0bf7..1174baa95ffb46 100644 --- a/test/.excludes-prism/TestRubyLiteral.rb +++ b/test/.excludes-prism/TestRubyLiteral.rb @@ -1,2 +1,6 @@ +exclude(:test_debug_frozen_string_in_array_literal, "unknown") +exclude(:test_debug_frozen_string, "unknown") exclude(:test_dregexp, "unknown") +exclude(:test_hash_value_omission, "unknown") +exclude(:test_integer, "unknown") exclude(:test_string, "https://github.com/ruby/prism/issues/2331") diff --git a/test/.excludes-prism/TestRubyOptimization.rb b/test/.excludes-prism/TestRubyOptimization.rb new file mode 100644 index 00000000000000..df22ca4f714860 --- /dev/null +++ b/test/.excludes-prism/TestRubyOptimization.rb @@ -0,0 +1 @@ +exclude(:test_peephole_string_literal_range, "unknown") diff --git a/test/.excludes-prism/TestRubyVM.rb b/test/.excludes-prism/TestRubyVM.rb new file mode 100644 index 00000000000000..6d4c3ca6fe4513 --- /dev/null +++ b/test/.excludes-prism/TestRubyVM.rb @@ -0,0 +1 @@ +exclude(:test_keep_script_lines, "unknown") diff --git a/test/.excludes-prism/TestSyntax.rb b/test/.excludes-prism/TestSyntax.rb index 680974906bcb6f..8f0cfcf4f45a7a 100644 --- a/test/.excludes-prism/TestSyntax.rb +++ b/test/.excludes-prism/TestSyntax.rb @@ -1,4 +1,49 @@ +exclude(:test__END___cr, "unknown") +exclude(:test_anonymous_block_forwarding, "unknown") +exclude(:test_anonymous_keyword_rest_forwarding, "unknown") +exclude(:test_anonymous_rest_forwarding, "unknown") +exclude(:test_argument_forwarding_with_anon_rest_kwrest_and_block, "unknown") +exclude(:test_argument_forwarding_with_super, "unknown") +exclude(:test_argument_forwarding, "unknown") +exclude(:test_brace_after_literal_argument, "unknown") exclude(:test_dedented_heredoc_concatenation, "unknown") exclude(:test_dedented_heredoc_continued_line, "unknown") +exclude(:test_dedented_heredoc_invalid_identifer, "unknown") +exclude(:test_duplicated_arg, "unknown") +exclude(:test_duplicated_kw_kwrest, "unknown") +exclude(:test_duplicated_kw, "unknown") +exclude(:test_duplicated_opt_kw, "unknown") +exclude(:test_duplicated_opt_kwrest, "unknown") +exclude(:test_duplicated_opt_post, "unknown") +exclude(:test_duplicated_opt_rest, "unknown") +exclude(:test_duplicated_opt, "unknown") +exclude(:test_duplicated_rest_kw, "unknown") +exclude(:test_duplicated_rest_kwrest, "unknown") +exclude(:test_duplicated_rest_opt, "unknown") +exclude(:test_duplicated_rest_post, "unknown") +exclude(:test_duplicated_rest, "unknown") exclude(:test_duplicated_when, "unknown") +exclude(:test_error_message_encoding, "unknown") +exclude(:test_heredoc_cr, "unknown") +exclude(:test_heredoc_no_terminator, "unknown") +exclude(:test_invalid_break, "unknown") +exclude(:test_invalid_encoding_symbol, "unknown") +exclude(:test_invalid_literal_message, "unknown") +exclude(:test_invalid_next, "unknown") exclude(:test_it, "https://github.com/ruby/prism/issues/2323") +exclude(:test_keyword_invalid_name, "unknown") +exclude(:test_keyword_self_reference, "unknown") +exclude(:test_keywords_specified_and_not_accepted, "unknown") +exclude(:test_methoddef_endless_command, "unknown") +exclude(:test_numbered_parameter, "unknown") +exclude(:test_optional_self_reference, "unknown") +exclude(:test_parenthesised_statement_argument, "unknown") +exclude(:test_range_at_eol, "unknown") +exclude(:test_safe_call_in_massign_lhs, "unknown") +exclude(:test_syntax_error_at_newline, "unknown") +exclude(:test_unassignable, "unknown") +exclude(:test_unexpected_fraction, "unknown") +exclude(:test_unterminated_heredoc_cr, "unknown") +exclude(:test_unterminated_heredoc, "unknown") +exclude(:test_warn_balanced, "unknown") +exclude(:test_warn_unreachable, "unknown") From d7d59ea172384cac5eae63b39b61c3f09c1f43e3 Mon Sep 17 00:00:00 2001 From: Alex Robbin Date: Fri, 29 Mar 2024 12:23:24 -0400 Subject: [PATCH 162/211] [rubygems/rubygems] add test case to ensure updating with multiple sources + caching maintains the right lockfile https://github.com/rubygems/rubygems/commit/65839757e6 --- spec/bundler/commands/update_spec.rb | 84 ++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/spec/bundler/commands/update_spec.rb b/spec/bundler/commands/update_spec.rb index 2a09e0531ba7ea..cfb86ebb54832f 100644 --- a/spec/bundler/commands/update_spec.rb +++ b/spec/bundler/commands/update_spec.rb @@ -864,6 +864,90 @@ expect(exitstatus).to eq(22) end + context "with multiple sources and caching enabled" do + before do + build_repo2 do + build_gem "rack", "1.0.0" + + build_gem "request_store", "1.0.0" do |s| + s.add_dependency "rack", "1.0.0" + end + end + + build_repo4 do + # set up repo with no gems + end + + gemfile <<~G + source "#{file_uri_for(gem_repo2)}" + + gem "request_store" + + source "#{file_uri_for(gem_repo4)}" do + end + G + + lockfile <<~L + GEM + remote: #{file_uri_for(gem_repo2)}/ + specs: + rack (1.0.0) + request_store (1.0.0) + rack (= 1.0.0) + + GEM + remote: #{file_uri_for(gem_repo4)}/ + specs: + + PLATFORMS + #{local_platform} + + DEPENDENCIES + request_store + + BUNDLED WITH + #{Bundler::VERSION} + L + end + + it "works" do + bundle :install + bundle :cache + + update_repo2 do + build_gem "request_store", "1.1.0" do |s| + s.add_dependency "rack", "1.0.0" + end + end + + bundle "update request_store" + + expect(out).to include("Bundle updated!") + + expect(lockfile).to eq <<~L + GEM + remote: #{file_uri_for(gem_repo2)}/ + specs: + rack (1.0.0) + request_store (1.1.0) + rack (= 1.0.0) + + GEM + remote: #{file_uri_for(gem_repo4)}/ + specs: + + PLATFORMS + #{local_platform} + + DEPENDENCIES + request_store + + BUNDLED WITH + #{Bundler::VERSION} + L + end + end + context "with multiple, duplicated sources, with lockfile in old format", bundler: "< 3" do before do build_repo2 do From cdb8d208c919bbc72b3b07d24c118d3a4af95d11 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 29 Mar 2024 18:25:39 -0400 Subject: [PATCH 163/211] [PRISM] Fix error message for duplicate parameter name --- prism/config.yml | 2 +- prism/prism.c | 2 +- prism/templates/src/diagnostic.c.erb | 2 +- test/.excludes-prism/TestCall.rb | 2 +- test/.excludes-prism/TestParse.rb | 1 - test/.excludes-prism/TestRegexp.rb | 6 +++--- test/.excludes-prism/TestSyntax.rb | 13 ------------- test/prism/errors_test.rb | 12 ++++++------ 8 files changed, 13 insertions(+), 27 deletions(-) diff --git a/prism/config.yml b/prism/config.yml index 5ed4cf1b2128ea..edbe4b32d84be5 100644 --- a/prism/config.yml +++ b/prism/config.yml @@ -166,7 +166,7 @@ errors: - PARAMETER_BLOCK_MULTI - PARAMETER_CIRCULAR - PARAMETER_METHOD_NAME - - PARAMETER_NAME_REPEAT + - PARAMETER_NAME_DUPLICATED - PARAMETER_NO_DEFAULT - PARAMETER_NO_DEFAULT_KW - PARAMETER_NUMBERED_RESERVED diff --git a/prism/prism.c b/prism/prism.c index 68e016c22a3f50..1f4b9ced5c67fd 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -7176,7 +7176,7 @@ pm_parser_parameter_name_check(pm_parser_t *parser, const pm_token_t *name) { if (pm_constant_id_list_includes(&parser->current_scope->locals, constant_id)) { // Add an error if the parameter doesn't start with _ and has been seen before if ((name->start < name->end) && (*name->start != '_')) { - pm_parser_err_token(parser, name, PM_ERR_PARAMETER_NAME_REPEAT); + pm_parser_err_token(parser, name, PM_ERR_PARAMETER_NAME_DUPLICATED); } return true; } diff --git a/prism/templates/src/diagnostic.c.erb b/prism/templates/src/diagnostic.c.erb index 8badf579e5031c..b608c44f01afce 100644 --- a/prism/templates/src/diagnostic.c.erb +++ b/prism/templates/src/diagnostic.c.erb @@ -249,7 +249,7 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_ERR_PARAMETER_BLOCK_MULTI] = { "multiple block parameters; only one block is allowed", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_PARAMETER_CIRCULAR] = { "parameter default value references itself", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_PARAMETER_METHOD_NAME] = { "unexpected name for a parameter", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_PARAMETER_NAME_REPEAT] = { "repeated parameter name", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PARAMETER_NAME_DUPLICATED] = { "duplicated argument name", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_PARAMETER_NO_DEFAULT] = { "expected a default value for the parameter", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_PARAMETER_NO_DEFAULT_KW] = { "expected a default value for the keyword parameter", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_PARAMETER_NUMBERED_RESERVED] = { "%.2s is reserved for numbered parameters", PM_ERROR_LEVEL_SYNTAX }, diff --git a/test/.excludes-prism/TestCall.rb b/test/.excludes-prism/TestCall.rb index 08a949fc017475..fa84af685b5196 100644 --- a/test/.excludes-prism/TestCall.rb +++ b/test/.excludes-prism/TestCall.rb @@ -1,3 +1,3 @@ exclude(:test_call_op_asgn_keywords, "unknown") -exclude(:test_kwsplat_block_order_op_asgn, "unknown") exclude(:test_call_op_asgn_keywords_mutable, "unknown") +exclude(:test_kwsplat_block_order_op_asgn, "unknown") diff --git a/test/.excludes-prism/TestParse.rb b/test/.excludes-prism/TestParse.rb index 126ead1800e152..50a82029b5f838 100644 --- a/test/.excludes-prism/TestParse.rb +++ b/test/.excludes-prism/TestParse.rb @@ -5,7 +5,6 @@ exclude(:test_disallowed_class_variable, "unknown") exclude(:test_disallowed_gloal_variable, "unknown") exclude(:test_disallowed_instance_variable, "unknown") -exclude(:test_duplicate_argument, "unknown") exclude(:test_dynamic_constant_assignment, "unknown") exclude(:test_else_without_rescue, "unknown") exclude(:test_embedded_rd_error, "unknown") diff --git a/test/.excludes-prism/TestRegexp.rb b/test/.excludes-prism/TestRegexp.rb index 090515bbe4efd0..2cf1902348455b 100644 --- a/test/.excludes-prism/TestRegexp.rb +++ b/test/.excludes-prism/TestRegexp.rb @@ -1,6 +1,6 @@ -exclude(:test_unicode_age_14_0, "unknown") +exclude(:test_invalid_escape_error, "unknown") exclude(:test_invalid_fragment, "unknown") -exclude(:test_unicode_age_15_0, "unknown") exclude(:test_unescape, "unknown") -exclude(:test_invalid_escape_error, "unknown") +exclude(:test_unicode_age_14_0, "unknown") +exclude(:test_unicode_age_15_0, "unknown") exclude(:test_unicode_age, "unknown") diff --git a/test/.excludes-prism/TestSyntax.rb b/test/.excludes-prism/TestSyntax.rb index 8f0cfcf4f45a7a..f2f13e3161c674 100644 --- a/test/.excludes-prism/TestSyntax.rb +++ b/test/.excludes-prism/TestSyntax.rb @@ -9,19 +9,6 @@ exclude(:test_dedented_heredoc_concatenation, "unknown") exclude(:test_dedented_heredoc_continued_line, "unknown") exclude(:test_dedented_heredoc_invalid_identifer, "unknown") -exclude(:test_duplicated_arg, "unknown") -exclude(:test_duplicated_kw_kwrest, "unknown") -exclude(:test_duplicated_kw, "unknown") -exclude(:test_duplicated_opt_kw, "unknown") -exclude(:test_duplicated_opt_kwrest, "unknown") -exclude(:test_duplicated_opt_post, "unknown") -exclude(:test_duplicated_opt_rest, "unknown") -exclude(:test_duplicated_opt, "unknown") -exclude(:test_duplicated_rest_kw, "unknown") -exclude(:test_duplicated_rest_kwrest, "unknown") -exclude(:test_duplicated_rest_opt, "unknown") -exclude(:test_duplicated_rest_post, "unknown") -exclude(:test_duplicated_rest, "unknown") exclude(:test_duplicated_when, "unknown") exclude(:test_error_message_encoding, "unknown") exclude(:test_heredoc_cr, "unknown") diff --git a/test/prism/errors_test.rb b/test/prism/errors_test.rb index 6e6e74ee5d5a1d..6cc71f9647172e 100644 --- a/test/prism/errors_test.rb +++ b/test/prism/errors_test.rb @@ -1149,7 +1149,7 @@ def test_duplicated_parameter_names ) assert_errors expected, "def foo(a,b,a);end", [ - ["repeated parameter name", 12..13] + ["duplicated argument name", 12..13] ] end @@ -1169,7 +1169,7 @@ def test_duplicated_parameter_names ) assert_errors expected, "def foo(a,b,*a);end", [ - ["repeated parameter name", 13..14] + ["duplicated argument name", 13..14] ] expected = DefNode( @@ -1188,7 +1188,7 @@ def test_duplicated_parameter_names ) assert_errors expected, "def foo(a,b,**a);end", [ - ["repeated parameter name", 14..15] + ["duplicated argument name", 14..15] ] expected = DefNode( @@ -1207,7 +1207,7 @@ def test_duplicated_parameter_names ) assert_errors expected, "def foo(a,b,&a);end", [ - ["repeated parameter name", 13..14] + ["duplicated argument name", 13..14] ] expected = DefNode( @@ -1482,7 +1482,7 @@ def test_shadow_args_in_lambda def test_shadow_args_in_block source = "tap{|a;a|}" assert_errors expression(source), source, [ - ["repeated parameter name", 7..8], + ["duplicated argument name", 7..8], ] end @@ -1491,7 +1491,7 @@ def test_repeated_parameter_name_in_destructured_params # In Ruby 3.0.x, `Ripper.sexp_raw` does not return `nil` for this case. compare_ripper = RUBY_ENGINE == "ruby" && (RUBY_VERSION.split('.').map { |x| x.to_i } <=> [3, 1]) >= 1 assert_errors expression(source), source, [ - ["repeated parameter name", 14..15], + ["duplicated argument name", 14..15], ], compare_ripper: compare_ripper end From a6d8837d2b1c0f71a397804a0f7ecbc27bc687ea Mon Sep 17 00:00:00 2001 From: git Date: Sat, 30 Mar 2024 06:58:50 +0000 Subject: [PATCH 164/211] Update bundled gems list as of 2024-03-29 --- NEWS.md | 2 +- gems/bundled_gems | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/NEWS.md b/NEWS.md index ea274937758cba..874794bff9dbd6 100644 --- a/NEWS.md +++ b/NEWS.md @@ -60,7 +60,7 @@ The following bundled gems are updated. * net-smtp 0.5.0 * rbs 3.4.4 * typeprof 0.21.11 -* debug 1.9.1 +* debug 1.9.2 The following bundled gems are promoted from default gems. diff --git a/gems/bundled_gems b/gems/bundled_gems index 5dfd755706a379..408b5426d4a450 100644 --- a/gems/bundled_gems +++ b/gems/bundled_gems @@ -20,7 +20,7 @@ matrix 0.4.2 https://github.com/ruby/matrix prime 0.1.2 https://github.com/ruby/prime rbs 3.4.4 https://github.com/ruby/rbs ba7872795d5de04adb8ff500c0e6afdc81a041dd typeprof 0.21.11 https://github.com/ruby/typeprof b19a6416da3a05d57fadd6ffdadb382b6d236ca5 -debug 1.9.1 https://github.com/ruby/debug 2d602636d99114d55a32fedd652c9c704446a749 +debug 1.9.2 https://github.com/ruby/debug racc 1.7.3 https://github.com/ruby/racc mutex_m 0.2.0 https://github.com/ruby/mutex_m getoptlong 0.2.1 https://github.com/ruby/getoptlong From f697d3242feeb18dae3b9f5e9addebb11e5442ff Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Sat, 30 Mar 2024 18:34:45 +0900 Subject: [PATCH 165/211] Manage required baseruby version in one place --- configure.ac | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/configure.ac b/configure.ac index 530c073764caa2..990f28143c26a0 100644 --- a/configure.ac +++ b/configure.ac @@ -75,8 +75,12 @@ AC_ARG_WITH(baseruby, [ AC_PATH_PROG([BASERUBY], [ruby], [false]) ]) -# BASERUBY must be >= 3.0.0. Note that `"3.0.0" > "3.0"` is true. -AS_IF([test "$HAVE_BASERUBY" != no -a "`RUBYOPT=- $BASERUBY --disable=gems -e 'print true if RUBY_VERSION > "3.0"' 2>/dev/null`" = true], [ +required_baseruby_version=`${tooldir}/missing-baseruby.bat 2>&1 | sed '/.* BASERUBY must be /!d;s///;s/^Ruby *//;s/ .*//'` +required_baseruby_version=${required_baseruby_version%.0} # Note that `"x.y.0" > "x.y"` is true. +AS_IF([test "$HAVE_BASERUBY" != no], [ + HAVE_BASERUBY="`RUBYOPT=- $BASERUBY --disable=gems -e 'print :yes if RUBY_VERSION > "'$required_baseruby_version'"' 2>/dev/null`" +]) +AS_IF([test "${HAVE_BASERUBY:=no}" != no], [ AS_CASE(["$build_os"], [mingw*], [ # Can MSys shell run a command with a drive letter? RUBYOPT=- `cygpath -ma "$BASERUBY"` --disable=gems -e exit 2>/dev/null || HAVE_BASERUBY=no @@ -84,12 +88,10 @@ AS_IF([test "$HAVE_BASERUBY" != no -a "`RUBYOPT=- $BASERUBY --disable=gems -e 'p RUBY_APPEND_OPTION(BASERUBY, "--disable=gems") BASERUBY_VERSION=`$BASERUBY -v` $BASERUBY -C "$srcdir" tool/downloader.rb -d tool -e gnu config.guess config.sub >&AS_MESSAGE_FD -], [ - HAVE_BASERUBY=no ]) AS_IF([test "$HAVE_BASERUBY" = no], [ AS_IF([test "$cross_compiling" = yes], [AC_MSG_ERROR([executable host ruby is required for cross-compiling])]) - BASERUBY=$srcdir/tool/missing-baseruby.bat + BASERUBY=${tooldir}/missing-baseruby.bat ]) AC_SUBST(BASERUBY) AC_SUBST(HAVE_BASERUBY) From 376ae22235dd50fee32ab7660c17137b7f3a245e Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Sat, 30 Mar 2024 18:34:45 +0900 Subject: [PATCH 166/211] Manage required baseruby version in one place Add a Ruby script mode to `tool/missing-baseruby.bat` that checks if `RUBY_VERSION` meets the required version. This will enable similar checks on mswin as well. --- configure.ac | 4 +--- tool/missing-baseruby.bat | 7 +++++-- win32/Makefile.sub | 7 ++++--- win32/setup.mak | 1 + 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/configure.ac b/configure.ac index 990f28143c26a0..7badc5f46a8849 100644 --- a/configure.ac +++ b/configure.ac @@ -75,10 +75,8 @@ AC_ARG_WITH(baseruby, [ AC_PATH_PROG([BASERUBY], [ruby], [false]) ]) -required_baseruby_version=`${tooldir}/missing-baseruby.bat 2>&1 | sed '/.* BASERUBY must be /!d;s///;s/^Ruby *//;s/ .*//'` -required_baseruby_version=${required_baseruby_version%.0} # Note that `"x.y.0" > "x.y"` is true. AS_IF([test "$HAVE_BASERUBY" != no], [ - HAVE_BASERUBY="`RUBYOPT=- $BASERUBY --disable=gems -e 'print :yes if RUBY_VERSION > "'$required_baseruby_version'"' 2>/dev/null`" + RUBYOPT=- $BASERUBY --disable=gems "${tooldir}/missing-baseruby.bat" || HAVE_BASERUBY=no ]) AS_IF([test "${HAVE_BASERUBY:=no}" != no], [ AS_CASE(["$build_os"], [mingw*], [ diff --git a/tool/missing-baseruby.bat b/tool/missing-baseruby.bat index 34b37361e5ef6f..87a9857e066fcc 100755 --- a/tool/missing-baseruby.bat +++ b/tool/missing-baseruby.bat @@ -1,12 +1,14 @@ -: " +:"" == " @echo off || ( :warn echo>&2.%~1 goto :eof :abort exit /b 1 +)||( +:)"||( + s = %^# ) -: " : ; call() { local call=${1#:}; shift; $call "$@"; } : ; warn() { echo "$1" >&2; } : ; abort () { exit 1; } @@ -14,3 +16,4 @@ call :warn "executable host ruby is required. use --with-baseruby option." call :warn "Note that BASERUBY must be Ruby 3.0.0 or later." call :abort +: || (:^; abort if RUBY_VERSION < s[%r"warn .*Ruby ([\d.]+)(?:\.0)?",1]) diff --git a/win32/Makefile.sub b/win32/Makefile.sub index 80a43d00dae209..7dbb020eee11fc 100644 --- a/win32/Makefile.sub +++ b/win32/Makefile.sub @@ -11,13 +11,15 @@ PATH_SEPARATOR = ; TZ = # skip timezone tests PWD = $(MAKEDIR) empty = +tooldir = $(srcdir)/tool !ifndef MFLAGS MFLAGS=-l !endif !if "$(BASERUBY)" == "" -! if [for %I in (ruby.exe) do @echo BASERUBY = %~s$$PATH:I > baseruby.mk] +! if [ruby $(tooldir)/missing-baseruby.bat 2> nul] +! else if [for %I in (ruby.exe) do @echo BASERUBY = %~s$$PATH:I > baseruby.mk] ! else ! include baseruby.mk ! endif @@ -27,7 +29,7 @@ MFLAGS=-l BASERUBY = !endif !if "$(BASERUBY)" == "" -BASERUBY = echo executable host ruby is required. use --with-baseruby option.^& exit 1 +BASERUBY = $(tooldir:/=\)\missing-baseruby.bat HAVE_BASERUBY = no !else HAVE_BASERUBY = yes @@ -485,7 +487,6 @@ EXTOBJS = dmyext.$(OBJEXT) arch_hdrdir = $(EXTOUT)/include/$(arch) top_srcdir = $(srcdir) hdrdir = $(srcdir)/include -tooldir = $(srcdir)/tool VPATH = $(arch_hdrdir)/ruby;$(hdrdir)/ruby;$(srcdir);$(srcdir)/missing;$(win_srcdir) !ifndef GIT diff --git a/win32/setup.mak b/win32/setup.mak index d4178127bf2282..8c279948215f9c 100644 --- a/win32/setup.mak +++ b/win32/setup.mak @@ -66,6 +66,7 @@ RJIT_SUPPORT = $(RJIT_SUPPORT) # TOOLS << !if defined(BASERUBY) + $(BASERUBY:/=\) "$(srcdir)/tool/missing-baseruby.bat" @echo BASERUBY = $(BASERUBY:/=\)>> $(MAKEFILE) !endif !if "$(RUBY_DEVEL)" == "yes" From 9579cf45d59f313e70a6a8dab2e9173743513e91 Mon Sep 17 00:00:00 2001 From: Aaron Patterson Date: Sat, 30 Mar 2024 17:29:09 -0700 Subject: [PATCH 167/211] If we have a shape cache we should use it If there is a shape cache, then we should believe the results instead of doing a linear search for non-existent items This fixes a case where checking the index of an undefined ivar would result in an O(n) search. Now we get O(log n). Benchmark is as follows: ```ruby N = ARGV[0].to_i class ManyIVs class_eval "def initialize;" + N.times.map { "@a#{_1} = #{_1}" }.join("\n") + "end" def check defined?(@not) end end class Subclass < ManyIVs def initialize super @foo = 123 end end def t s = Process.clock_gettime Process::CLOCK_MONOTONIC yield Process.clock_gettime(Process::CLOCK_MONOTONIC) - s end def test a = ManyIVs.new b = Subclass.new t { 200000.times { a.check; b.check } } end puts "#{N},#{test}" ``` On the master branch: ``` $ for i in (seq 1 3 32); ./miniruby test.rb $i; end 1,0.015619999991031364 4,0.013061000005109236 7,0.013365999999223277 10,0.015474999992875382 13,0.017674999980954453 16,0.020055999979376793 19,0.02260500000556931 22,0.0254080000158865 25,0.02806599999894388 28,0.031244999991031364 31,0.034568000002764165 ``` On this branch: ``` $ for i in (seq 1 3 32); ./miniruby test.rb $i; end 1,0.015848999988520518 4,0.013225000002421439 7,0.013049000001046807 10,0.010697999998228624 13,0.010902000009082258 16,0.011448000004747882 19,0.01151199999731034 22,0.011539999977685511 25,0.01173300002119504 28,0.011900000012246892 31,0.012278999987756833 ``` --- shape.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/shape.c b/shape.c index 8241f67d6ab5d6..1158aad52c5f11 100644 --- a/shape.c +++ b/shape.c @@ -863,7 +863,13 @@ rb_shape_get_iv_index(rb_shape_t *shape, ID id, attr_index_t *value) RUBY_ASSERT(rb_shape_id(shape) != OBJ_TOO_COMPLEX_SHAPE_ID); if (!shape_cache_get_iv_index(shape, id, value)) { - return shape_get_iv_index(shape, id, value); + // If it wasn't in the ancestor cache, then don't do a linear search + if (shape->ancestor_index && shape->next_iv_index >= ANCESTOR_CACHE_THRESHOLD) { + return false; + } + else { + return shape_get_iv_index(shape, id, value); + } } return true; From 174b67169975160aa682d9b2c6ac5ccde2652105 Mon Sep 17 00:00:00 2001 From: Ellen Marie Dash Date: Sat, 30 Mar 2024 21:07:31 -0400 Subject: [PATCH 168/211] [rubygems/rubygems] [commands/rebuild] Remove unused DATE_FORMAT constant. https://github.com/rubygems/rubygems/commit/3c4e3fadc9 --- lib/rubygems/commands/rebuild_command.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/rubygems/commands/rebuild_command.rb b/lib/rubygems/commands/rebuild_command.rb index 97f05ef79c84aa..77a474ef1ddf32 100644 --- a/lib/rubygems/commands/rebuild_command.rb +++ b/lib/rubygems/commands/rebuild_command.rb @@ -10,8 +10,6 @@ class Gem::Commands::RebuildCommand < Gem::Command include Gem::GemspecHelpers - DATE_FORMAT = "%Y-%m-%d %H:%M:%S.%N Z" - def initialize super "rebuild", "Attempt to reproduce a build of a gem." From 9d0a5148ae062a0481a4a18fbeb9cfd01dc10428 Mon Sep 17 00:00:00 2001 From: KJ Tsanaktsidis Date: Sat, 30 Mar 2024 12:58:01 +1100 Subject: [PATCH 169/211] Add missing RB_GC_GUARDs related to DATA_PTR I discovered the problem in `compile.c` from a failing TestIseqLoad#test_stressful_roundtrip test with ASAN enabled. The other two changes in array.c and string.c I found by auditing similar usages of DATA_PTR in the codebase. [Bug #20402] --- array.c | 1 + compile.c | 1 + string.c | 1 + 3 files changed, 3 insertions(+) diff --git a/array.c b/array.c index 00f5ab91812d82..bcf98fc0125ae1 100644 --- a/array.c +++ b/array.c @@ -6643,6 +6643,7 @@ ary_sample(rb_execution_context_t *ec, VALUE ary, VALUE randgen, VALUE nv, VALUE }); DATA_PTR(vmemo) = 0; st_free_table(memo); + RB_GC_GUARD(vmemo); } else { result = rb_ary_dup(ary); diff --git a/compile.c b/compile.c index ab041e6e89d6f6..a66d5e1b93d81a 100644 --- a/compile.c +++ b/compile.c @@ -11318,6 +11318,7 @@ iseq_build_from_ary_body(rb_iseq_t *iseq, LINK_ANCHOR *const anchor, } } DATA_PTR(labels_wrapper) = 0; + RB_GC_GUARD(labels_wrapper); validate_labels(iseq, labels_table); if (!ret) return ret; return iseq_setup(iseq, anchor); diff --git a/string.c b/string.c index 5c29718dff8647..ecd6b97f883ff1 100644 --- a/string.c +++ b/string.c @@ -1148,6 +1148,7 @@ str_cat_conv_enc_opts(VALUE newstr, long ofs, const char *ptr, long len, rb_str_resize(newstr, olen); } DATA_PTR(econv_wrapper) = 0; + RB_GC_GUARD(econv_wrapper); rb_econv_close(ec); switch (ret) { case econv_finished: From e02a06fbf25001f0adc56e2e798b25d16ba84f54 Mon Sep 17 00:00:00 2001 From: KJ Tsanaktsidis Date: Sun, 31 Mar 2024 20:41:22 +1100 Subject: [PATCH 170/211] Document how to run the tests under ASAN now that they pass! --- doc/contributing/building_ruby.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/doc/contributing/building_ruby.md b/doc/contributing/building_ruby.md index 18464458896b74..2991385b94a6e6 100644 --- a/doc/contributing/building_ruby.md +++ b/doc/contributing/building_ruby.md @@ -181,12 +181,16 @@ mkdir build && cd build ../configure CC=clang cflags="-fsanitize=address -fno-omit-frame-pointer -DUSE_MN_THREADS=0" # and any other options you might like make ``` -The compiled Ruby will now automatically crash with a report and a backtrace if ASAN detects a memory safety issue. +The compiled Ruby will now automatically crash with a report and a backtrace if ASAN detects a memory safety issue. To run Ruby's test suite under ASAN, issue the following command. Note that this will take quite a long time (over two hours on my laptop); the `RUBY_TEST_TIMEOUT_SCALE` and `SYNTAX_SUGEST_TIMEOUT` variables are required to make sure tests don't spuriously fail with timeouts when in fact they're just slow. + +``` shell +RUBY_TEST_TIMEOUT_SCALE=5 SYNTAX_SUGGEST_TIMEOUT=600 make check +``` Please note, however, the following caveats! -* ASAN will not work properly on any currently released version of Ruby; the necessary support is currently only present on Ruby's master branch. -* Due to [this bug](https://bugs.ruby-lang.org/issues/20243), Clang generates code for threadlocal variables which doesn't work with M:N threading. Thus, it's necessary to disable M:N threading support at build time for now. +* ASAN will not work properly on any currently released version of Ruby; the necessary support is currently only present on Ruby's master branch (and the whole test suite passes only as of commit [9d0a5148ae062a0481a4a18fbeb9cfd01dc10428](https://bugs.ruby-lang.org/projects/ruby-master/repository/git/revisions/9d0a5148ae062a0481a4a18fbeb9cfd01dc10428)) +* Due to [this bug](https://bugs.ruby-lang.org/issues/20243), Clang generates code for threadlocal variables which doesn't work with M:N threading. Thus, it's necessary to disable M:N threading support at build time for now (with the `-DUSE_MN_THREADS=0` configure argument). * Currently, ASAN will only work correctly when using a recent head build of LLVM/Clang - it requires [this bugfix](https://github.com/llvm/llvm-project/pull/75290) related to multithreaded `fork`, which is not yet in any released version. See [here](https://llvm.org/docs/CMake.html) for instructions on how to build LLVM/Clang from source (note you will need at least the `clang` and `compiler-rt` projects enabled). Then, you will need to replace `CC=clang` in the instructions with an explicit path to your built Clang binary. * ASAN has only been tested so far with Clang on Linux. It may or may not work with other compilers or on other platforms - please file an issue on [https://bugs.ruby-lang.org](https://bugs.ruby-lang.org) if you run into problems with such configurations (or, to report that they actually work properly!) * In particular, although I have not yet tried it, I have reason to believe ASAN will _not_ work properly on macOS yet - the fix for the multithreaded fork issue was actually reverted for macOS (see [here](https://github.com/llvm/llvm-project/commit/2a03854e4ce9bb1bcd79a211063bc63c4657f92c)). Please open an issue on [https://bugs.ruby-lang.org](https://bugs.ruby-lang.org) if this is a problem for you. From 80bda107c84187d90eeff9497d465e086364b420 Mon Sep 17 00:00:00 2001 From: KJ Tsanaktsidis Date: Mon, 1 Apr 2024 10:50:18 +1100 Subject: [PATCH 171/211] [ruby/resolv] Add an explicit with_udp_and_tcp helper to test_dns.rb This helper tries to bind UDP and TCP sockets to the same port, by retrying the bind if the randomly-assinged UDP port is already taken in TCP. This fixes a flaky test. [Bug #20403] https://github.com/ruby/resolv/commit/3d135f99d9 --- test/resolv/test_dns.rb | 334 ++++++++++++++++++++++------------------ 1 file changed, 188 insertions(+), 146 deletions(-) diff --git a/test/resolv/test_dns.rb b/test/resolv/test_dns.rb index 20c3408cd6b9ca..40c5406db83525 100644 --- a/test/resolv/test_dns.rb +++ b/test/resolv/test_dns.rb @@ -64,6 +64,50 @@ def with_udp(host, port) end end + def with_udp_and_tcp(host, port) + if port == 0 + # Automatic port; we might need to retry until we find a port which is free on both UDP _and_ TCP. + retries_remaining = 5 + t = nil + u = nil + begin + begin + u = UDPSocket.new + u.bind(host, 0) + _, udp_port, _, _ = u.addr + t = TCPServer.new(host, udp_port) + t.listen(1) + rescue Errno::EADDRINUSE, Errno::EACCES + # ADDRINUSE is what should get thrown if we try and bind a port which is already bound on UNIXen, + # but windows can sometimes throw EACCESS. + # See: https://stackoverflow.com/questions/48478869/cannot-bind-to-some-ports-due-to-permission-denied + retries_remaining -= 1 + if retries_remaining > 0 + t&.close + t = nil + u&.close + u = nil + retry + end + raise + end + + # If we get to this point, we have a valid t & u socket + yield u, t + ensure + t&.close + u&.close + end + else + # Explicitly specified port, don't retry the bind. + with_udp(host, port) do |u| + with_tcp(host, port) do |t| + yield u, t + end + end + end + end + # [ruby-core:65836] def test_resolve_with_2_ndots conf = Resolv::DNS::Config.new :nameserver => ['127.0.0.1'], :ndots => 2 @@ -176,156 +220,154 @@ def test_query_ipv4_address_truncated_tcp_fallback num_records = 50 - with_udp('127.0.0.1', 0) {|u| + with_udp_and_tcp('127.0.0.1', 0) {|u, t| _, server_port, _, server_address = u.addr - with_tcp('127.0.0.1', server_port) {|t| - client_thread = Thread.new { - Resolv::DNS.open(:nameserver_port => [[server_address, server_port]]) {|dns| - dns.getresources("foo.example.org", Resolv::DNS::Resource::IN::A) - } - } - udp_server_thread = Thread.new { - msg, (_, client_port, _, client_address) = Timeout.timeout(5) {u.recvfrom(4096)} - id, word2, qdcount, ancount, nscount, arcount = msg.unpack("nnnnnn") - qr = (word2 & 0x8000) >> 15 - opcode = (word2 & 0x7800) >> 11 - aa = (word2 & 0x0400) >> 10 - tc = (word2 & 0x0200) >> 9 - rd = (word2 & 0x0100) >> 8 - ra = (word2 & 0x0080) >> 7 - z = (word2 & 0x0070) >> 4 - rcode = word2 & 0x000f - rest = msg[12..-1] - assert_equal(0, qr) # 0:query 1:response - assert_equal(0, opcode) # 0:QUERY 1:IQUERY 2:STATUS - assert_equal(0, aa) # Authoritative Answer - assert_equal(0, tc) # TrunCation - assert_equal(1, rd) # Recursion Desired - assert_equal(0, ra) # Recursion Available - assert_equal(0, z) # Reserved for future use - assert_equal(0, rcode) # 0:No-error 1:Format-error 2:Server-failure 3:Name-Error 4:Not-Implemented 5:Refused - assert_equal(1, qdcount) # number of entries in the question section. - assert_equal(0, ancount) # number of entries in the answer section. - assert_equal(0, nscount) # number of entries in the authority records section. - assert_equal(0, arcount) # number of entries in the additional records section. - name = [3, "foo", 7, "example", 3, "org", 0].pack("Ca*Ca*Ca*C") - assert_operator(rest, :start_with?, name) - rest = rest[name.length..-1] - assert_equal(4, rest.length) - qtype, _ = rest.unpack("nn") - assert_equal(1, qtype) # A - assert_equal(1, qtype) # IN - id = id - qr = 1 - opcode = opcode - aa = 0 - tc = 1 - rd = rd - ra = 1 - z = 0 - rcode = 0 - qdcount = 0 - ancount = num_records - nscount = 0 - arcount = 0 - word2 = (qr << 15) | - (opcode << 11) | - (aa << 10) | - (tc << 9) | - (rd << 8) | - (ra << 7) | - (z << 4) | - rcode - msg = [id, word2, qdcount, ancount, nscount, arcount].pack("nnnnnn") - type = 1 - klass = 1 - ttl = 3600 - rdlength = 4 - num_records.times do |i| - rdata = [192,0,2,i].pack("CCCC") # 192.0.2.x (TEST-NET address) RFC 3330 - rr = [name, type, klass, ttl, rdlength, rdata].pack("a*nnNna*") - msg << rr - end - u.send(msg[0...512], 0, client_address, client_port) - } - tcp_server_thread = Thread.new { - ct = t.accept - msg = ct.recv(512) - msg.slice!(0..1) # Size (only for TCP) - id, word2, qdcount, ancount, nscount, arcount = msg.unpack("nnnnnn") - qr = (word2 & 0x8000) >> 15 - opcode = (word2 & 0x7800) >> 11 - aa = (word2 & 0x0400) >> 10 - tc = (word2 & 0x0200) >> 9 - rd = (word2 & 0x0100) >> 8 - ra = (word2 & 0x0080) >> 7 - z = (word2 & 0x0070) >> 4 - rcode = word2 & 0x000f - rest = msg[12..-1] - assert_equal(0, qr) # 0:query 1:response - assert_equal(0, opcode) # 0:QUERY 1:IQUERY 2:STATUS - assert_equal(0, aa) # Authoritative Answer - assert_equal(0, tc) # TrunCation - assert_equal(1, rd) # Recursion Desired - assert_equal(0, ra) # Recursion Available - assert_equal(0, z) # Reserved for future use - assert_equal(0, rcode) # 0:No-error 1:Format-error 2:Server-failure 3:Name-Error 4:Not-Implemented 5:Refused - assert_equal(1, qdcount) # number of entries in the question section. - assert_equal(0, ancount) # number of entries in the answer section. - assert_equal(0, nscount) # number of entries in the authority records section. - assert_equal(0, arcount) # number of entries in the additional records section. - name = [3, "foo", 7, "example", 3, "org", 0].pack("Ca*Ca*Ca*C") - assert_operator(rest, :start_with?, name) - rest = rest[name.length..-1] - assert_equal(4, rest.length) - qtype, _ = rest.unpack("nn") - assert_equal(1, qtype) # A - assert_equal(1, qtype) # IN - id = id - qr = 1 - opcode = opcode - aa = 0 - tc = 0 - rd = rd - ra = 1 - z = 0 - rcode = 0 - qdcount = 0 - ancount = num_records - nscount = 0 - arcount = 0 - word2 = (qr << 15) | - (opcode << 11) | - (aa << 10) | - (tc << 9) | - (rd << 8) | - (ra << 7) | - (z << 4) | - rcode - msg = [id, word2, qdcount, ancount, nscount, arcount].pack("nnnnnn") - type = 1 - klass = 1 - ttl = 3600 - rdlength = 4 - num_records.times do |i| - rdata = [192,0,2,i].pack("CCCC") # 192.0.2.x (TEST-NET address) RFC 3330 - rr = [name, type, klass, ttl, rdlength, rdata].pack("a*nnNna*") - msg << rr - end - msg = "#{[msg.bytesize].pack("n")}#{msg}" # Prefix with size - ct.send(msg, 0) - ct.close + client_thread = Thread.new { + Resolv::DNS.open(:nameserver_port => [[server_address, server_port]]) {|dns| + dns.getresources("foo.example.org", Resolv::DNS::Resource::IN::A) } - result, _ = assert_join_threads([client_thread, udp_server_thread, tcp_server_thread]) - assert_instance_of(Array, result) - assert_equal(50, result.length) - result.each_with_index do |rr, i| - assert_instance_of(Resolv::DNS::Resource::IN::A, rr) - assert_instance_of(Resolv::IPv4, rr.address) - assert_equal("192.0.2.#{i}", rr.address.to_s) - assert_equal(3600, rr.ttl) + } + udp_server_thread = Thread.new { + msg, (_, client_port, _, client_address) = Timeout.timeout(5) {u.recvfrom(4096)} + id, word2, qdcount, ancount, nscount, arcount = msg.unpack("nnnnnn") + qr = (word2 & 0x8000) >> 15 + opcode = (word2 & 0x7800) >> 11 + aa = (word2 & 0x0400) >> 10 + tc = (word2 & 0x0200) >> 9 + rd = (word2 & 0x0100) >> 8 + ra = (word2 & 0x0080) >> 7 + z = (word2 & 0x0070) >> 4 + rcode = word2 & 0x000f + rest = msg[12..-1] + assert_equal(0, qr) # 0:query 1:response + assert_equal(0, opcode) # 0:QUERY 1:IQUERY 2:STATUS + assert_equal(0, aa) # Authoritative Answer + assert_equal(0, tc) # TrunCation + assert_equal(1, rd) # Recursion Desired + assert_equal(0, ra) # Recursion Available + assert_equal(0, z) # Reserved for future use + assert_equal(0, rcode) # 0:No-error 1:Format-error 2:Server-failure 3:Name-Error 4:Not-Implemented 5:Refused + assert_equal(1, qdcount) # number of entries in the question section. + assert_equal(0, ancount) # number of entries in the answer section. + assert_equal(0, nscount) # number of entries in the authority records section. + assert_equal(0, arcount) # number of entries in the additional records section. + name = [3, "foo", 7, "example", 3, "org", 0].pack("Ca*Ca*Ca*C") + assert_operator(rest, :start_with?, name) + rest = rest[name.length..-1] + assert_equal(4, rest.length) + qtype, _ = rest.unpack("nn") + assert_equal(1, qtype) # A + assert_equal(1, qtype) # IN + id = id + qr = 1 + opcode = opcode + aa = 0 + tc = 1 + rd = rd + ra = 1 + z = 0 + rcode = 0 + qdcount = 0 + ancount = num_records + nscount = 0 + arcount = 0 + word2 = (qr << 15) | + (opcode << 11) | + (aa << 10) | + (tc << 9) | + (rd << 8) | + (ra << 7) | + (z << 4) | + rcode + msg = [id, word2, qdcount, ancount, nscount, arcount].pack("nnnnnn") + type = 1 + klass = 1 + ttl = 3600 + rdlength = 4 + num_records.times do |i| + rdata = [192,0,2,i].pack("CCCC") # 192.0.2.x (TEST-NET address) RFC 3330 + rr = [name, type, klass, ttl, rdlength, rdata].pack("a*nnNna*") + msg << rr end + u.send(msg[0...512], 0, client_address, client_port) } + tcp_server_thread = Thread.new { + ct = t.accept + msg = ct.recv(512) + msg.slice!(0..1) # Size (only for TCP) + id, word2, qdcount, ancount, nscount, arcount = msg.unpack("nnnnnn") + qr = (word2 & 0x8000) >> 15 + opcode = (word2 & 0x7800) >> 11 + aa = (word2 & 0x0400) >> 10 + tc = (word2 & 0x0200) >> 9 + rd = (word2 & 0x0100) >> 8 + ra = (word2 & 0x0080) >> 7 + z = (word2 & 0x0070) >> 4 + rcode = word2 & 0x000f + rest = msg[12..-1] + assert_equal(0, qr) # 0:query 1:response + assert_equal(0, opcode) # 0:QUERY 1:IQUERY 2:STATUS + assert_equal(0, aa) # Authoritative Answer + assert_equal(0, tc) # TrunCation + assert_equal(1, rd) # Recursion Desired + assert_equal(0, ra) # Recursion Available + assert_equal(0, z) # Reserved for future use + assert_equal(0, rcode) # 0:No-error 1:Format-error 2:Server-failure 3:Name-Error 4:Not-Implemented 5:Refused + assert_equal(1, qdcount) # number of entries in the question section. + assert_equal(0, ancount) # number of entries in the answer section. + assert_equal(0, nscount) # number of entries in the authority records section. + assert_equal(0, arcount) # number of entries in the additional records section. + name = [3, "foo", 7, "example", 3, "org", 0].pack("Ca*Ca*Ca*C") + assert_operator(rest, :start_with?, name) + rest = rest[name.length..-1] + assert_equal(4, rest.length) + qtype, _ = rest.unpack("nn") + assert_equal(1, qtype) # A + assert_equal(1, qtype) # IN + id = id + qr = 1 + opcode = opcode + aa = 0 + tc = 0 + rd = rd + ra = 1 + z = 0 + rcode = 0 + qdcount = 0 + ancount = num_records + nscount = 0 + arcount = 0 + word2 = (qr << 15) | + (opcode << 11) | + (aa << 10) | + (tc << 9) | + (rd << 8) | + (ra << 7) | + (z << 4) | + rcode + msg = [id, word2, qdcount, ancount, nscount, arcount].pack("nnnnnn") + type = 1 + klass = 1 + ttl = 3600 + rdlength = 4 + num_records.times do |i| + rdata = [192,0,2,i].pack("CCCC") # 192.0.2.x (TEST-NET address) RFC 3330 + rr = [name, type, klass, ttl, rdlength, rdata].pack("a*nnNna*") + msg << rr + end + msg = "#{[msg.bytesize].pack("n")}#{msg}" # Prefix with size + ct.send(msg, 0) + ct.close + } + result, _ = assert_join_threads([client_thread, udp_server_thread, tcp_server_thread]) + assert_instance_of(Array, result) + assert_equal(50, result.length) + result.each_with_index do |rr, i| + assert_instance_of(Resolv::DNS::Resource::IN::A, rr) + assert_instance_of(Resolv::IPv4, rr.address) + assert_equal("192.0.2.#{i}", rr.address.to_s) + assert_equal(3600, rr.ttl) + end } end From e07178d52613cb7090e6c5d8e8e57e8e1f938527 Mon Sep 17 00:00:00 2001 From: Joshua Young Date: Mon, 1 Apr 2024 11:06:10 +1000 Subject: [PATCH 172/211] [DOC] Fix scope resolution operator typo in `Process#wait` docs --- process.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/process.c b/process.c index 8d08da76505635..72483815d68f1b 100644 --- a/process.c +++ b/process.c @@ -1452,7 +1452,7 @@ proc_wait(int argc, VALUE *argv) * or as the logical OR of both: * * - Process::WNOHANG: Does not block if no child process is available. - * - Process:WUNTRACED: May return a stopped child process, even if not yet reported. + * - Process::WUNTRACED: May return a stopped child process, even if not yet reported. * * Not all flags are available on all platforms. * From 0774232bf3c1eab0f6a414578988b051c9dda3cf Mon Sep 17 00:00:00 2001 From: S-H-GAMELINKS Date: Wed, 27 Mar 2024 00:47:36 +0900 Subject: [PATCH 173/211] Remove unnecessary macros and functions for Universal Parser --- ruby_parser.c | 16 ---------------- rubyparser.h | 4 ---- universal_parser.c | 4 ---- 3 files changed, 24 deletions(-) diff --git a/ruby_parser.c b/ruby_parser.c index 6d85a72c5b0c18..83539612e882e4 100644 --- a/ruby_parser.c +++ b/ruby_parser.c @@ -263,18 +263,6 @@ enc_from_encoding(void *enc) return rb_enc_from_encoding((rb_encoding *)enc); } -static int -encoding_get(VALUE obj) -{ - return ENCODING_GET(obj); -} - -static void -encoding_set(VALUE obj, int encindex) -{ - ENCODING_SET(obj, encindex); -} - static int encoding_is_ascii8bit(VALUE obj) { @@ -603,8 +591,6 @@ static const rb_parser_config_t rb_global_parser_config = { .ascii8bit_encoding = ascii8bit_encoding, .enc_codelen = enc_codelen, .enc_mbcput = enc_mbcput, - .char_to_option_kcode = rb_char_to_option_kcode, - .ascii8bit_encindex = rb_ascii8bit_encindex, .enc_find_index = rb_enc_find_index, .enc_from_index = enc_from_index, .enc_associate_index = rb_enc_associate_index, @@ -613,8 +599,6 @@ static const rb_parser_config_t rb_global_parser_config = { .enc_coderange_unknown = ENC_CODERANGE_UNKNOWN, .enc_compatible = enc_compatible, .enc_from_encoding = enc_from_encoding, - .encoding_get = encoding_get, - .encoding_set = encoding_set, .encoding_is_ascii8bit = encoding_is_ascii8bit, .usascii_encoding = usascii_encoding, .enc_coderange_broken = ENC_CODERANGE_BROKEN, diff --git a/rubyparser.h b/rubyparser.h index 9a809aa059f220..b0bef9d56feb6c 100644 --- a/rubyparser.h +++ b/rubyparser.h @@ -1356,16 +1356,12 @@ typedef struct rb_parser_config_struct { rb_encoding *(*ascii8bit_encoding)(void); int (*enc_codelen)(int c, rb_encoding *enc); int (*enc_mbcput)(unsigned int c, void *buf, rb_encoding *enc); - int (*char_to_option_kcode)(int c, int *option, int *kcode); - int (*ascii8bit_encindex)(void); int (*enc_find_index)(const char *name); rb_encoding *(*enc_from_index)(int idx); VALUE (*enc_associate_index)(VALUE obj, int encindex); int (*enc_isspace)(OnigCodePoint c, rb_encoding *enc); rb_encoding *(*enc_compatible)(VALUE str1, VALUE str2); VALUE (*enc_from_encoding)(rb_encoding *enc); - int (*encoding_get)(VALUE obj); - void (*encoding_set)(VALUE obj, int encindex); int (*encoding_is_ascii8bit)(VALUE obj); rb_encoding *(*usascii_encoding)(void); int enc_coderange_broken; diff --git a/universal_parser.c b/universal_parser.c index 08fdfe5b4a8008..4a02675eec303d 100644 --- a/universal_parser.c +++ b/universal_parser.c @@ -211,8 +211,6 @@ struct rb_imemo_tmpbuf_struct { #define rb_ascii8bit_encoding p->config->ascii8bit_encoding #define rb_enc_codelen p->config->enc_codelen #define rb_enc_mbcput p->config->enc_mbcput -#define rb_char_to_option_kcode p->config->char_to_option_kcode -#define rb_ascii8bit_encindex p->config->ascii8bit_encindex #define rb_enc_find_index p->config->enc_find_index #define rb_enc_from_index p->config->enc_from_index #define rb_enc_associate_index p->config->enc_associate_index @@ -221,8 +219,6 @@ struct rb_imemo_tmpbuf_struct { #define ENC_CODERANGE_UNKNOWN p->config->enc_coderange_unknown #define rb_enc_compatible p->config->enc_compatible #define rb_enc_from_encoding p->config->enc_from_encoding -#define ENCODING_GET p->config->encoding_get -#define ENCODING_SET p->config->encoding_set #define ENCODING_IS_ASCII8BIT p->config->encoding_is_ascii8bit #define rb_usascii_encoding p->config->usascii_encoding From acfef7c4f06696bf5f6d0ade06ddc32683a2f7bb Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Mon, 1 Apr 2024 13:16:33 +0900 Subject: [PATCH 174/211] Use dummy data generated by RubyGems --- test/rubygems/fixtures/prerelease_specs.4.8.gz | Bin 0 -> 24 bytes test/rubygems/fixtures/specs.4.8.gz | Bin 0 -> 559 bytes test/rubygems/test_gem_source.rb | 4 ++-- 3 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 test/rubygems/fixtures/prerelease_specs.4.8.gz create mode 100644 test/rubygems/fixtures/specs.4.8.gz diff --git a/test/rubygems/fixtures/prerelease_specs.4.8.gz b/test/rubygems/fixtures/prerelease_specs.4.8.gz new file mode 100644 index 0000000000000000000000000000000000000000..15d3a53b430ddcf639d48f74eea91b04fb4b7f74 GIT binary patch literal 24 fcmb2|=3w}=oFR>YIr(8n8UsVl+^U}}3=9kaVR{E= literal 0 HcmV?d00001 diff --git a/test/rubygems/fixtures/specs.4.8.gz b/test/rubygems/fixtures/specs.4.8.gz new file mode 100644 index 0000000000000000000000000000000000000000..0e98784384d309f95fcf72c7156bbf65e36b2649 GIT binary patch literal 559 zcmV+~0?_>*iwFSnrvPRE19enQZ`wc*?V+*^lt5CHs!~oIdb4Ho6=DwYrK-5Jm!yYq zK#N(!7O&SD+fe>}cD=xkQG2jt&&+%C_U(*6+Kb7;p7(D_R4|cp!V(HrV10ji4Q_u> z9t6KBs6JlOY=Pi8VOjVs;ZxVt_yotyg+kNLPT-PJDtkQEE9k`HhovJUAP`JAbh~B z#zxYoRhh1p6^Oxer%27W^HDta$>6?5B919_@(5T~I7nbu7qr(U5cOa;9JylKzlA(U%kkwLP|JF#KF>bMb7_A~sY8{ddDIWErP9C?z>VK_LQ`0Ye_%1RJ5ab(0sX*xOWrOom{Wlqbaz{8P5{BT(OkEv8Jz}03;QA xJjvG7KpVxDr5N!=uKjWOU&8i*l3gObuUqJu&6k`}q^bSL9;qm|l001n25()qS literal 0 HcmV?d00001 diff --git a/test/rubygems/test_gem_source.rb b/test/rubygems/test_gem_source.rb index 4d445f343765d6..9516a1442291ee 100644 --- a/test/rubygems/test_gem_source.rb +++ b/test/rubygems/test_gem_source.rb @@ -54,8 +54,8 @@ def test_dependency_resolver_set_bundler_api end def test_dependency_resolver_set_file_uri - File.write(File.join(@tempdir, "prerelease_specs.4.8.gz"), Gem::Util.gzip("\x04\x08[\x05".b)) - File.write(File.join(@tempdir, "specs.4.8.gz"), Gem::Util.gzip("\x04\x08[\x05".b)) + FileUtils.cp File.expand_path(File.join("fixtures", "prerelease_specs.4.8.gz"), __dir__), @tempdir + FileUtils.cp File.expand_path(File.join("fixtures", "specs.4.8.gz"), __dir__), @tempdir source = Gem::Source.new "file://#{@tempdir}/" From 70645a5acdaa6028a897b828ceacd57efb9cf257 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Mon, 1 Apr 2024 16:03:57 +0900 Subject: [PATCH 175/211] Write gzipped data as binary Be careful when writing binary data on Windows. ``` $ ruby -e 's = Gem::Util.gzip("\x04\x08[\x05".b); p s.index("\n"); puts IO::Buffer.for(s).hexdump' 6 0x00000000 1f 8b 08 00 6c 3d 0a 66 00 03 63 e1 88 66 05 00 ....l=.f..c..f.. 0x00000010 e3 69 10 89 04 00 00 00 .i...... ``` --- test/rubygems/fixtures/prerelease_specs.4.8.gz | Bin 24 -> 0 bytes test/rubygems/fixtures/specs.4.8.gz | Bin 559 -> 0 bytes test/rubygems/test_gem_source.rb | 5 +++-- 3 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 test/rubygems/fixtures/prerelease_specs.4.8.gz delete mode 100644 test/rubygems/fixtures/specs.4.8.gz diff --git a/test/rubygems/fixtures/prerelease_specs.4.8.gz b/test/rubygems/fixtures/prerelease_specs.4.8.gz deleted file mode 100644 index 15d3a53b430ddcf639d48f74eea91b04fb4b7f74..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 24 fcmb2|=3w}=oFR>YIr(8n8UsVl+^U}}3=9kaVR{E= diff --git a/test/rubygems/fixtures/specs.4.8.gz b/test/rubygems/fixtures/specs.4.8.gz deleted file mode 100644 index 0e98784384d309f95fcf72c7156bbf65e36b2649..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 559 zcmV+~0?_>*iwFSnrvPRE19enQZ`wc*?V+*^lt5CHs!~oIdb4Ho6=DwYrK-5Jm!yYq zK#N(!7O&SD+fe>}cD=xkQG2jt&&+%C_U(*6+Kb7;p7(D_R4|cp!V(HrV10ji4Q_u> z9t6KBs6JlOY=Pi8VOjVs;ZxVt_yotyg+kNLPT-PJDtkQEE9k`HhovJUAP`JAbh~B z#zxYoRhh1p6^Oxer%27W^HDta$>6?5B919_@(5T~I7nbu7qr(U5cOa;9JylKzlA(U%kkwLP|JF#KF>bMb7_A~sY8{ddDIWErP9C?z>VK_LQ`0Ye_%1RJ5ab(0sX*xOWrOom{Wlqbaz{8P5{BT(OkEv8Jz}03;QA xJjvG7KpVxDr5N!=uKjWOU&8i*l3gObuUqJu&6k`}q^bSL9;qm|l001n25()qS diff --git a/test/rubygems/test_gem_source.rb b/test/rubygems/test_gem_source.rb index 9516a1442291ee..269f81dc802948 100644 --- a/test/rubygems/test_gem_source.rb +++ b/test/rubygems/test_gem_source.rb @@ -54,8 +54,9 @@ def test_dependency_resolver_set_bundler_api end def test_dependency_resolver_set_file_uri - FileUtils.cp File.expand_path(File.join("fixtures", "prerelease_specs.4.8.gz"), __dir__), @tempdir - FileUtils.cp File.expand_path(File.join("fixtures", "specs.4.8.gz"), __dir__), @tempdir + empty_gzip = Gem::Util.gzip("\x04\x08[\x05".b) + File.binwrite(File.join(@tempdir, "prerelease_specs.4.8.gz"), empty_gzip) + File.binwrite(File.join(@tempdir, "specs.4.8.gz"), empty_gzip) source = Gem::Source.new "file://#{@tempdir}/" From b50c4dc30a1e2db18e62b4dab3d8faabc62e29e3 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Mon, 1 Apr 2024 16:51:28 +0900 Subject: [PATCH 176/211] Rename the variable It is not an empty gzipped data, a gzipped empty dump data. --- test/rubygems/test_gem_source.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/rubygems/test_gem_source.rb b/test/rubygems/test_gem_source.rb index 269f81dc802948..6baa203dcb1829 100644 --- a/test/rubygems/test_gem_source.rb +++ b/test/rubygems/test_gem_source.rb @@ -54,9 +54,9 @@ def test_dependency_resolver_set_bundler_api end def test_dependency_resolver_set_file_uri - empty_gzip = Gem::Util.gzip("\x04\x08[\x05".b) - File.binwrite(File.join(@tempdir, "prerelease_specs.4.8.gz"), empty_gzip) - File.binwrite(File.join(@tempdir, "specs.4.8.gz"), empty_gzip) + empty_dump = Gem::Util.gzip("\x04\x08[\x05".b) + File.binwrite(File.join(@tempdir, "prerelease_specs.4.8.gz"), empty_dump) + File.binwrite(File.join(@tempdir, "specs.4.8.gz"), empty_dump) source = Gem::Source.new "file://#{@tempdir}/" From 1232975398a96af3070463292ec0c01e09a06c50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=9C=E9=83=A8=E6=98=8C=E5=B9=B3?= Date: Mon, 1 Apr 2024 12:10:34 +0900 Subject: [PATCH 177/211] add CI matrix for clang-19 --- .github/workflows/compilers.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/compilers.yml b/.github/workflows/compilers.yml index 9a7855b78475b5..4b58d9d10a1cb6 100644 --- a/.github/workflows/compilers.yml +++ b/.github/workflows/compilers.yml @@ -27,7 +27,7 @@ concurrency: # environment variables (plus the "echo $GITHUB_ENV" hack) is to reroute that # restriction. env: - default_cc: clang-17 + default_cc: clang-18 append_cc: '' # -O1 is faster than -O3 in our tests... Majority of time are consumed trying @@ -81,6 +81,7 @@ jobs: optflags: '-O2' shared: disable # check: true + - { name: clang-19, env: { default_cc: clang-19 } } - { name: clang-18, env: { default_cc: clang-18 } } - { name: clang-17, env: { default_cc: clang-17 } } - { name: clang-16, env: { default_cc: clang-16 } } @@ -213,7 +214,7 @@ jobs: runs-on: ubuntu-latest container: - image: ghcr.io/ruby/ruby-ci-image:${{ matrix.entry.container || matrix.entry.env.default_cc || 'clang-17' }} + image: ghcr.io/ruby/ruby-ci-image:${{ matrix.entry.container || matrix.entry.env.default_cc || 'clang-18' }} options: --user root if: >- From e26ac3ab7106ce6ca5911e34cbd099841c8d6da1 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 28 Mar 2024 15:30:25 -0400 Subject: [PATCH 178/211] Test finalizer is ran in bootstraptest --- bootstraptest/test_finalizer.rb | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/bootstraptest/test_finalizer.rb b/bootstraptest/test_finalizer.rb index 22a16b1220e784..ccfa0b55d613f2 100644 --- a/bootstraptest/test_finalizer.rb +++ b/bootstraptest/test_finalizer.rb @@ -6,3 +6,11 @@ ObjectSpace.define_finalizer(a2,proc{a1.inspect}) ObjectSpace.define_finalizer(a1,proc{}) }, '[ruby-dev:35778]' + +assert_equal 'true', %q{ + obj = Object.new + id = obj.object_id + + ObjectSpace.define_finalizer(obj, proc { |i| print(id == i) }) + nil +} From 3ca0683529a540bf651d5dc94289dc1db79c3338 Mon Sep 17 00:00:00 2001 From: David Rodriguez Date: Thu, 21 Mar 2024 16:01:37 +0100 Subject: [PATCH 179/211] [rubygems/rubygems] Fix typo https://github.com/rubygems/rubygems/commit/0ddf25e5aa --- lib/bundler/resolver/candidate.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/bundler/resolver/candidate.rb b/lib/bundler/resolver/candidate.rb index e695ef08ee0bde..9e8b9133358c69 100644 --- a/lib/bundler/resolver/candidate.rb +++ b/lib/bundler/resolver/candidate.rb @@ -15,7 +15,7 @@ class Resolver # considered separately. # # Some candidates may also keep some information explicitly about the - # package the refer to. These candidates are referred to as "canonical" and + # package they refer to. These candidates are referred to as "canonical" and # are used when materializing resolution results back into RubyGems # specifications that can be installed, written to lock files, and so on. # From d342937e01935ea543cc1f6ac46022715c40db7d Mon Sep 17 00:00:00 2001 From: David Rodriguez Date: Thu, 21 Mar 2024 16:02:17 +0100 Subject: [PATCH 180/211] [rubygems/rubygems] Rename method for clarity And also so that it matches the method used by main PubGrub sample resolver class. https://github.com/rubygems/rubygems/commit/0e612361b8 --- lib/bundler/resolver.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/bundler/resolver.rb b/lib/bundler/resolver.rb index f60069f421ab08..d1c3addea2dbae 100644 --- a/lib/bundler/resolver.rb +++ b/lib/bundler/resolver.rb @@ -158,7 +158,7 @@ def parse_dependency(package, dependency) def versions_for(package, range=VersionRange.any) versions = range.select_versions(@sorted_versions[package]) - sort_versions(package, versions) + sort_versions_by_preferred(package, versions) end def no_versions_incompatibility_for(package, unsatisfied_term) @@ -275,7 +275,7 @@ def all_versions_for(package) groups end - sort_versions(package, versions) + sort_versions_by_preferred(package, versions) end def source_for(name) @@ -357,7 +357,7 @@ def requirement_satisfied_by?(requirement, spec) requirement.satisfied_by?(spec.version) || spec.source.is_a?(Source::Gemspec) end - def sort_versions(package, versions) + def sort_versions_by_preferred(package, versions) if versions.size > 1 @gem_version_promoter.sort_versions(package, versions).reverse else From acbd91e47ff36216459bbba4368b04e6a3079b2a Mon Sep 17 00:00:00 2001 From: David Rodriguez Date: Thu, 21 Mar 2024 16:55:50 +0100 Subject: [PATCH 181/211] [rubygems/rubygems] No need to sort twice when filling versions https://github.com/rubygems/rubygems/commit/13294528c4 --- lib/bundler/gem_version_promoter.rb | 8 +++++--- lib/bundler/resolver.rb | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/bundler/gem_version_promoter.rb b/lib/bundler/gem_version_promoter.rb index c7eacd193049a3..b666c29d32f819 100644 --- a/lib/bundler/gem_version_promoter.rb +++ b/lib/bundler/gem_version_promoter.rb @@ -53,7 +53,7 @@ def level=(value) # @return [Specification] A new instance of the Specification Array sorted and # possibly filtered. def sort_versions(package, specs) - specs = filter_dep_specs(specs, package) if strict + specs = filter_versions(package, specs) sort_dep_specs(specs, package) end @@ -73,9 +73,9 @@ def pre? pre == true end - private + def filter_versions(package, specs) + return specs unless strict - def filter_dep_specs(specs, package) locked_version = package.locked_version return specs if locked_version.nil? || major? @@ -89,6 +89,8 @@ def filter_dep_specs(specs, package) end end + private + def sort_dep_specs(specs, package) locked_version = package.locked_version diff --git a/lib/bundler/resolver.rb b/lib/bundler/resolver.rb index d1c3addea2dbae..1cd94ccf50c6b1 100644 --- a/lib/bundler/resolver.rb +++ b/lib/bundler/resolver.rb @@ -275,7 +275,7 @@ def all_versions_for(package) groups end - sort_versions_by_preferred(package, versions) + @gem_version_promoter.filter_versions(package, versions) end def source_for(name) From 0a1e36964dfa634bfd3088da5362093b89103a33 Mon Sep 17 00:00:00 2001 From: David Rodriguez Date: Thu, 21 Mar 2024 17:20:57 +0100 Subject: [PATCH 182/211] [rubygems/rubygems] Remove unnecessary filtering We do that when first caching versions, and then it's no longer necessary. https://github.com/rubygems/rubygems/commit/ede15847db --- lib/bundler/gem_version_promoter.rb | 52 ++++++++----------- .../bundler/gem_version_promoter_spec.rb | 8 +-- 2 files changed, 27 insertions(+), 33 deletions(-) diff --git a/lib/bundler/gem_version_promoter.rb b/lib/bundler/gem_version_promoter.rb index b666c29d32f819..66141b7b636123 100644 --- a/lib/bundler/gem_version_promoter.rb +++ b/lib/bundler/gem_version_promoter.rb @@ -53,9 +53,30 @@ def level=(value) # @return [Specification] A new instance of the Specification Array sorted and # possibly filtered. def sort_versions(package, specs) - specs = filter_versions(package, specs) + locked_version = package.locked_version - sort_dep_specs(specs, package) + result = specs.sort do |a, b| + unless package.prerelease_specified? || pre? + a_pre = a.prerelease? + b_pre = b.prerelease? + + next -1 if a_pre && !b_pre + next 1 if b_pre && !a_pre + end + + if major? || locked_version.nil? + a <=> b + elsif either_version_older_than_locked?(a, b, locked_version) + a <=> b + elsif segments_do_not_match?(a, b, :major) + b <=> a + elsif !minor? && segments_do_not_match?(a, b, :minor) + b <=> a + else + a <=> b + end + end + post_sort(result, package.unlock?, locked_version) end # @return [bool] Convenience method for testing value of level variable. @@ -91,33 +112,6 @@ def filter_versions(package, specs) private - def sort_dep_specs(specs, package) - locked_version = package.locked_version - - result = specs.sort do |a, b| - unless package.prerelease_specified? || pre? - a_pre = a.prerelease? - b_pre = b.prerelease? - - next -1 if a_pre && !b_pre - next 1 if b_pre && !a_pre - end - - if major? || locked_version.nil? - a <=> b - elsif either_version_older_than_locked?(a, b, locked_version) - a <=> b - elsif segments_do_not_match?(a, b, :major) - b <=> a - elsif !minor? && segments_do_not_match?(a, b, :minor) - b <=> a - else - a <=> b - end - end - post_sort(result, package.unlock?, locked_version) - end - def either_version_older_than_locked?(a, b, locked_version) a.version < locked_version || b.version < locked_version end diff --git a/spec/bundler/bundler/gem_version_promoter_spec.rb b/spec/bundler/bundler/gem_version_promoter_spec.rb index fccbb58fea181e..23228cd3da0b1b 100644 --- a/spec/bundler/bundler/gem_version_promoter_spec.rb +++ b/spec/bundler/bundler/gem_version_promoter_spec.rb @@ -58,18 +58,18 @@ def sorted_versions(candidates:, current:, name: "foo", locked: []) context "when level is minor" do before { gvp.level = :minor } - it "removes downgrades and major upgrades" do + it "sorts highest minor within same major in last position" do versions = sorted_versions(candidates: %w[0.2.0 0.3.0 0.3.1 0.9.0 1.0.0 2.0.1 2.1.0], current: "0.3.0") - expect(versions).to eq %w[0.3.0 0.3.1 0.9.0] + expect(versions).to eq %w[0.2.0 2.0.1 2.1.0 1.0.0 0.3.0 0.3.1 0.9.0] end end context "when level is patch" do before { gvp.level = :patch } - it "removes downgrades and major and minor upgrades" do + it "sorts highest patch within same minor in last position" do versions = sorted_versions(candidates: %w[0.2.0 0.3.0 0.3.1 0.9.0 1.0.0 2.0.1 2.1.0], current: "0.3.0") - expect(versions).to eq %w[0.3.0 0.3.1] + expect(versions).to eq %w[0.2.0 2.1.0 2.0.1 1.0.0 0.9.0 0.3.0 0.3.1] end end end From 2b82b7d192d26b1153186187dbe3bef84de7ed3f Mon Sep 17 00:00:00 2001 From: David Rodriguez Date: Thu, 21 Mar 2024 17:27:04 +0100 Subject: [PATCH 183/211] [rubygems/rubygems] Update docs https://github.com/rubygems/rubygems/commit/ac24a68486 --- lib/bundler/gem_version_promoter.rb | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/lib/bundler/gem_version_promoter.rb b/lib/bundler/gem_version_promoter.rb index 66141b7b636123..c7187654b7612e 100644 --- a/lib/bundler/gem_version_promoter.rb +++ b/lib/bundler/gem_version_promoter.rb @@ -45,13 +45,12 @@ def level=(value) # Given a Resolver::Package and an Array of Specifications of available # versions for a gem, this method will return the Array of Specifications - # sorted (and possibly truncated if strict is true) in an order to give - # preference to the current level (:major, :minor or :patch) when resolution - # is deciding what versions best resolve all dependencies in the bundle. + # sorted in an order to give preference to the current level (:major, :minor + # or :patch) when resolution is deciding what versions best resolve all + # dependencies in the bundle. # @param package [Resolver::Package] The package being resolved. # @param specs [Specification] An array of Specifications for the package. - # @return [Specification] A new instance of the Specification Array sorted and - # possibly filtered. + # @return [Specification] A new instance of the Specification Array sorted. def sort_versions(package, specs) locked_version = package.locked_version @@ -94,6 +93,15 @@ def pre? pre == true end + # Given a Resolver::Package and an Array of Specifications of available + # versions for a gem, this method will truncate the Array if strict + # is true. That means filtering out downgrades from the version currently + # locked, and filtering out upgrades that go past the selected level (major, + # minor, or patch). + # @param package [Resolver::Package] The package being resolved. + # @param specs [Specification] An array of Specifications for the package. + # @return [Specification] A new instance of the Specification Array + # truncated. def filter_versions(package, specs) return specs unless strict From d69ef1cc52b34d3242376ea5b4893b1b55e71517 Mon Sep 17 00:00:00 2001 From: David Rodriguez Date: Thu, 21 Mar 2024 17:55:14 +0100 Subject: [PATCH 184/211] [rubygems/rubygems] Let GemVersionPromoter sort in preferred order directly So that we don't need to reverse the Array. https://github.com/rubygems/rubygems/commit/aeea5e2e00 --- lib/bundler/gem_version_promoter.rb | 20 ++++++------ lib/bundler/resolver.rb | 2 +- .../bundler/gem_version_promoter_spec.rb | 32 +++++++++---------- 3 files changed, 27 insertions(+), 27 deletions(-) diff --git a/lib/bundler/gem_version_promoter.rb b/lib/bundler/gem_version_promoter.rb index c7187654b7612e..ecc65b49560292 100644 --- a/lib/bundler/gem_version_promoter.rb +++ b/lib/bundler/gem_version_promoter.rb @@ -59,20 +59,20 @@ def sort_versions(package, specs) a_pre = a.prerelease? b_pre = b.prerelease? - next -1 if a_pre && !b_pre - next 1 if b_pre && !a_pre + next 1 if a_pre && !b_pre + next -1 if b_pre && !a_pre end if major? || locked_version.nil? - a <=> b + b <=> a elsif either_version_older_than_locked?(a, b, locked_version) - a <=> b - elsif segments_do_not_match?(a, b, :major) b <=> a + elsif segments_do_not_match?(a, b, :major) + a <=> b elsif !minor? && segments_do_not_match?(a, b, :minor) - b <=> a - else a <=> b + else + b <=> a end end post_sort(result, package.unlock?, locked_version) @@ -137,13 +137,13 @@ def post_sort(result, unlock, locked_version) if unlock || locked_version.nil? result else - move_version_to_end(result, locked_version) + move_version_to_beginning(result, locked_version) end end - def move_version_to_end(result, version) + def move_version_to_beginning(result, version) move, keep = result.partition {|s| s.version.to_s == version.to_s } - keep.concat(move) + move.concat(keep) end end end diff --git a/lib/bundler/resolver.rb b/lib/bundler/resolver.rb index 1cd94ccf50c6b1..329540fd3d5e0f 100644 --- a/lib/bundler/resolver.rb +++ b/lib/bundler/resolver.rb @@ -359,7 +359,7 @@ def requirement_satisfied_by?(requirement, spec) def sort_versions_by_preferred(package, versions) if versions.size > 1 - @gem_version_promoter.sort_versions(package, versions).reverse + @gem_version_promoter.sort_versions(package, versions) else versions end diff --git a/spec/bundler/bundler/gem_version_promoter_spec.rb b/spec/bundler/bundler/gem_version_promoter_spec.rb index 23228cd3da0b1b..917daba95de5e5 100644 --- a/spec/bundler/bundler/gem_version_promoter_spec.rb +++ b/spec/bundler/bundler/gem_version_promoter_spec.rb @@ -33,13 +33,13 @@ def sorted_versions(candidates:, current:, name: "foo", locked: []) it "numerically sorts versions" do versions = sorted_versions(candidates: %w[1.7.7 1.7.8 1.7.9 1.7.15 1.8.0], current: "1.7.8") - expect(versions).to eq %w[1.7.7 1.7.8 1.7.9 1.7.15 1.8.0] + expect(versions).to eq %w[1.8.0 1.7.15 1.7.9 1.7.8 1.7.7] end context "with no options" do it "defaults to level=:major, strict=false, pre=false" do versions = sorted_versions(candidates: %w[0.2.0 0.3.0 0.3.1 0.9.0 1.0.0 2.0.1 2.1.0], current: "0.3.0") - expect(versions).to eq %w[0.2.0 0.3.0 0.3.1 0.9.0 1.0.0 2.0.1 2.1.0] + expect(versions).to eq %w[2.1.0 2.0.1 1.0.0 0.9.0 0.3.1 0.3.0 0.2.0] end end @@ -51,25 +51,25 @@ def sorted_versions(candidates:, current:, name: "foo", locked: []) it "keeps downgrades" do versions = sorted_versions(candidates: %w[0.2.0 0.3.0 0.3.1 0.9.0 1.0.0 2.0.1 2.1.0], current: "0.3.0") - expect(versions).to eq %w[0.2.0 0.3.0 0.3.1 0.9.0 1.0.0 2.0.1 2.1.0] + expect(versions).to eq %w[2.1.0 2.0.1 1.0.0 0.9.0 0.3.1 0.3.0 0.2.0] end end context "when level is minor" do before { gvp.level = :minor } - it "sorts highest minor within same major in last position" do + it "sorts highest minor within same major in first position" do versions = sorted_versions(candidates: %w[0.2.0 0.3.0 0.3.1 0.9.0 1.0.0 2.0.1 2.1.0], current: "0.3.0") - expect(versions).to eq %w[0.2.0 2.0.1 2.1.0 1.0.0 0.3.0 0.3.1 0.9.0] + expect(versions).to eq %w[0.9.0 0.3.1 0.3.0 1.0.0 2.1.0 2.0.1 0.2.0] end end context "when level is patch" do before { gvp.level = :patch } - it "sorts highest patch within same minor in last position" do + it "sorts highest patch within same minor in first position" do versions = sorted_versions(candidates: %w[0.2.0 0.3.0 0.3.1 0.9.0 1.0.0 2.0.1 2.1.0], current: "0.3.0") - expect(versions).to eq %w[0.2.0 2.1.0 2.0.1 1.0.0 0.9.0 0.3.0 0.3.1] + expect(versions).to eq %w[0.3.1 0.3.0 0.9.0 1.0.0 2.0.1 2.1.0 0.2.0] end end end @@ -82,25 +82,25 @@ def sorted_versions(candidates:, current:, name: "foo", locked: []) it "orders by version" do versions = sorted_versions(candidates: %w[0.2.0 0.3.0 0.3.1 0.9.0 1.0.0 2.0.1 2.1.0], current: "0.3.0") - expect(versions).to eq %w[0.2.0 0.3.0 0.3.1 0.9.0 1.0.0 2.0.1 2.1.0] + expect(versions).to eq %w[2.1.0 2.0.1 1.0.0 0.9.0 0.3.1 0.3.0 0.2.0] end end context "when level is minor" do before { gvp.level = :minor } - it "favors downgrades, then upgrades by major descending, minor ascending, patch ascending" do + it "favors minor upgrades, then patch upgrades, then major upgrades, then downgrades" do versions = sorted_versions(candidates: %w[0.2.0 0.3.0 0.3.1 0.9.0 1.0.0 2.0.1 2.1.0], current: "0.3.0") - expect(versions).to eq %w[0.2.0 2.0.1 2.1.0 1.0.0 0.3.0 0.3.1 0.9.0] + expect(versions).to eq %w[0.9.0 0.3.1 0.3.0 1.0.0 2.1.0 2.0.1 0.2.0] end end context "when level is patch" do before { gvp.level = :patch } - it "favors downgrades, then upgrades by major descending, minor descending, patch ascending" do + it "favors patch upgrades, then minor upgrades, then major upgrades, then downgrades" do versions = sorted_versions(candidates: %w[0.2.0 0.3.0 0.3.1 0.9.0 1.0.0 2.0.1 2.1.0], current: "0.3.0") - expect(versions).to eq %w[0.2.0 2.1.0 2.0.1 1.0.0 0.9.0 0.3.0 0.3.1] + expect(versions).to eq %w[0.3.1 0.3.0 0.9.0 1.0.0 2.0.1 2.1.0 0.2.0] end end end @@ -110,7 +110,7 @@ def sorted_versions(candidates:, current:, name: "foo", locked: []) it "sorts regardless of prerelease status" do versions = sorted_versions(candidates: %w[1.7.7.pre 1.8.0 1.8.1.pre 1.8.1 2.0.0.pre 2.0.0], current: "1.8.0") - expect(versions).to eq %w[1.7.7.pre 1.8.0 1.8.1.pre 1.8.1 2.0.0.pre 2.0.0] + expect(versions).to eq %w[2.0.0 2.0.0.pre 1.8.1 1.8.1.pre 1.8.0 1.7.7.pre] end end @@ -119,16 +119,16 @@ def sorted_versions(candidates:, current:, name: "foo", locked: []) it "deprioritizes prerelease gems" do versions = sorted_versions(candidates: %w[1.7.7.pre 1.8.0 1.8.1.pre 1.8.1 2.0.0.pre 2.0.0], current: "1.8.0") - expect(versions).to eq %w[1.7.7.pre 1.8.1.pre 2.0.0.pre 1.8.0 1.8.1 2.0.0] + expect(versions).to eq %w[2.0.0 1.8.1 1.8.0 2.0.0.pre 1.8.1.pre 1.7.7.pre] end end context "when locking and not major" do before { gvp.level = :minor } - it "keeps the current version last" do + it "keeps the current version first" do versions = sorted_versions(candidates: %w[0.2.0 0.3.0 0.3.1 0.9.0 1.0.0 2.1.0 2.0.1], current: "0.3.0", locked: ["bar"]) - expect(versions.last).to eq("0.3.0") + expect(versions.first).to eq("0.3.0") end end end From caaafbc35e05b11b597d297e67142e27eab9a012 Mon Sep 17 00:00:00 2001 From: David Rodriguez Date: Thu, 21 Mar 2024 18:19:13 +0100 Subject: [PATCH 185/211] [rubygems/rubygems] Make it look more like BasicPackageSource https://github.com/rubygems/rubygems/commit/bb5727934c --- lib/bundler/resolver.rb | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/bundler/resolver.rb b/lib/bundler/resolver.rb index 329540fd3d5e0f..30e9a7beaa3e57 100644 --- a/lib/bundler/resolver.rb +++ b/lib/bundler/resolver.rb @@ -158,7 +158,13 @@ def parse_dependency(package, dependency) def versions_for(package, range=VersionRange.any) versions = range.select_versions(@sorted_versions[package]) - sort_versions_by_preferred(package, versions) + # Conditional avoids (among other things) calling + # sort_versions_by_preferred with the root package + if versions.size > 1 + sort_versions_by_preferred(package, versions) + else + versions + end end def no_versions_incompatibility_for(package, unsatisfied_term) @@ -358,11 +364,7 @@ def requirement_satisfied_by?(requirement, spec) end def sort_versions_by_preferred(package, versions) - if versions.size > 1 - @gem_version_promoter.sort_versions(package, versions) - else - versions - end + @gem_version_promoter.sort_versions(package, versions) end def repository_for(package) From b6ac37c91a1bcbf51eb7632cabafa037073be764 Mon Sep 17 00:00:00 2001 From: David Rodriguez Date: Thu, 21 Mar 2024 18:41:31 +0100 Subject: [PATCH 186/211] [rubygems/rubygems] No need for any version prioritization when building errors We just need to filter versions belonging to the range, but don't need anything else. https://github.com/rubygems/rubygems/commit/8355a225d7 --- lib/bundler/resolver.rb | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/lib/bundler/resolver.rb b/lib/bundler/resolver.rb index 30e9a7beaa3e57..2b6022cc2757a3 100644 --- a/lib/bundler/resolver.rb +++ b/lib/bundler/resolver.rb @@ -156,7 +156,7 @@ def parse_dependency(package, dependency) end def versions_for(package, range=VersionRange.any) - versions = range.select_versions(@sorted_versions[package]) + versions = select_sorted_versions(package, range) # Conditional avoids (among other things) calling # sort_versions_by_preferred with the root package @@ -381,11 +381,12 @@ def prepare_dependencies(requirements, packages) next [dep_package, dep_constraint] if name == "bundler" - versions = versions_for(dep_package, dep_constraint.range) + dep_range = dep_constraint.range + versions = select_sorted_versions(dep_package, dep_range) if versions.empty? && dep_package.ignores_prereleases? @sorted_versions.delete(dep_package) dep_package.consider_prereleases! - versions = versions_for(dep_package, dep_constraint.range) + versions = select_sorted_versions(dep_package, dep_range) end next [dep_package, dep_constraint] unless versions.empty? @@ -395,6 +396,10 @@ def prepare_dependencies(requirements, packages) end.compact.to_h end + def select_sorted_versions(package, range) + range.select_versions(@sorted_versions[package]) + end + def other_specs_matching_message(specs, requirement) message = String.new("The source contains the following gems matching '#{requirement}':\n") message << specs.map {|s| " * #{s.full_name}" }.join("\n") From bfdbdf7aaeff3c987c3a09cc550358a321d8df32 Mon Sep 17 00:00:00 2001 From: David Rodriguez Date: Thu, 21 Mar 2024 21:20:40 +0100 Subject: [PATCH 187/211] [rubygems/rubygems] No need to check for root package every time https://github.com/rubygems/rubygems/commit/6ca192649f --- lib/bundler/resolver.rb | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/lib/bundler/resolver.rb b/lib/bundler/resolver.rb index 2b6022cc2757a3..78beddd7a32bde 100644 --- a/lib/bundler/resolver.rb +++ b/lib/bundler/resolver.rb @@ -51,25 +51,21 @@ def setup_solver end @sorted_versions = Hash.new do |candidates, package| - candidates[package] = if package.root? - [root_version] - else - all_versions_for(package).sort - end + candidates[package] = all_versions_for(package).sort end + @sorted_versions[root] = [root_version] + root_dependencies = prepare_dependencies(@requirements, @packages) @cached_dependencies = Hash.new do |dependencies, package| - dependencies[package] = if package.root? - { root_version => root_dependencies } - else - Hash.new do |versions, version| - versions[version] = to_dependency_hash(version.dependencies.reject {|d| d.name == package.name }, @packages) - end + dependencies[package] = Hash.new do |versions, version| + versions[version] = to_dependency_hash(version.dependencies.reject {|d| d.name == package.name }, @packages) end end + @cached_dependencies[root] = { root_version => root_dependencies } + logger = Bundler::UI::Shell.new logger.level = debug? ? "debug" : "warn" From f80bb3837c2ba2d46d8906c5f4bdc65f475c76db Mon Sep 17 00:00:00 2001 From: David Rodriguez Date: Thu, 21 Mar 2024 19:19:05 +0100 Subject: [PATCH 188/211] [rubygems/rubygems] Keep unfiltered versions separately https://github.com/rubygems/rubygems/commit/7b5cc51a96 --- lib/bundler/resolver.rb | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/lib/bundler/resolver.rb b/lib/bundler/resolver.rb index 78beddd7a32bde..c148c79b371fab 100644 --- a/lib/bundler/resolver.rb +++ b/lib/bundler/resolver.rb @@ -50,8 +50,12 @@ def setup_solver specs[name] = matches.sort_by {|s| [s.version, s.platform.to_s] } end + @all_versions = Hash.new do |candidates, package| + candidates[package] = all_versions_for(package) + end + @sorted_versions = Hash.new do |candidates, package| - candidates[package] = all_versions_for(package).sort + candidates[package] = filtered_versions_for(package).sort end @sorted_versions[root] = [root_version] @@ -249,7 +253,7 @@ def all_versions_for(package) locked_requirement = base_requirements[name] results = filter_matching_specs(results, locked_requirement) if locked_requirement - versions = results.group_by(&:version).reduce([]) do |groups, (version, specs)| + results.group_by(&:version).reduce([]) do |groups, (version, specs)| platform_specs = package.platforms.map {|platform| select_best_platform_match(specs, platform) } # If package is a top-level dependency, @@ -276,8 +280,6 @@ def all_versions_for(package) groups end - - @gem_version_promoter.filter_versions(package, versions) end def source_for(name) @@ -336,6 +338,10 @@ def raise_not_found!(package) private + def filtered_versions_for(package) + @gem_version_promoter.filter_versions(package, @all_versions[package]) + end + def filter_matching_specs(specs, requirements) Array(requirements).flat_map do |requirement| specs.select {| spec| requirement_satisfied_by?(requirement, spec) } @@ -380,6 +386,7 @@ def prepare_dependencies(requirements, packages) dep_range = dep_constraint.range versions = select_sorted_versions(dep_package, dep_range) if versions.empty? && dep_package.ignores_prereleases? + @all_versions.delete(dep_package) @sorted_versions.delete(dep_package) dep_package.consider_prereleases! versions = select_sorted_versions(dep_package, dep_range) From e2a1d0b53dddc29b03a535286763fde51d4e089d Mon Sep 17 00:00:00 2001 From: David Rodriguez Date: Thu, 21 Mar 2024 20:59:42 +0100 Subject: [PATCH 189/211] [rubygems/rubygems] Improve error message when strict resolution filters out everything https://github.com/rubygems/rubygems/commit/1ea44b3749 --- lib/bundler/resolver.rb | 20 ++++++++++++++++++++ spec/bundler/commands/lock_spec.rb | 16 ++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/lib/bundler/resolver.rb b/lib/bundler/resolver.rb index c148c79b371fab..1a6711ea6fcacd 100644 --- a/lib/bundler/resolver.rb +++ b/lib/bundler/resolver.rb @@ -342,6 +342,17 @@ def filtered_versions_for(package) @gem_version_promoter.filter_versions(package, @all_versions[package]) end + def raise_all_versions_filtered_out!(package) + level = @gem_version_promoter.level + name = package.name + locked_version = package.locked_version + requirement = package.dependency + + raise GemNotFound, + "#{name} is locked to #{locked_version}, while Gemfile is requesting #{requirement}. " \ + "--strict --#{level} was specified, but there are no #{level} level upgrades from #{locked_version} satisfying #{requirement}, so version solving has failed" + end + def filter_matching_specs(specs, requirements) Array(requirements).flat_map do |requirement| specs.select {| spec| requirement_satisfied_by?(requirement, spec) } @@ -391,6 +402,11 @@ def prepare_dependencies(requirements, packages) dep_package.consider_prereleases! versions = select_sorted_versions(dep_package, dep_range) end + + if versions.empty? && select_all_versions(dep_package, dep_range).any? + raise_all_versions_filtered_out!(dep_package) + end + next [dep_package, dep_constraint] unless versions.empty? next unless dep_package.current_platform? @@ -403,6 +419,10 @@ def select_sorted_versions(package, range) range.select_versions(@sorted_versions[package]) end + def select_all_versions(package, range) + range.select_versions(@all_versions[package]) + end + def other_specs_matching_message(specs, requirement) message = String.new("The source contains the following gems matching '#{requirement}':\n") message << specs.map {|s| " * #{s.full_name}" }.join("\n") diff --git a/spec/bundler/commands/lock_spec.rb b/spec/bundler/commands/lock_spec.rb index 0f1aeef910650e..f6793d393ba828 100644 --- a/spec/bundler/commands/lock_spec.rb +++ b/spec/bundler/commands/lock_spec.rb @@ -392,6 +392,22 @@ expect(the_bundle.locked_gems.specs.map(&:full_name)).to eq(%w[foo-1.5.0 bar-2.1.1 qux-1.1.0].sort) end + it "shows proper error when Gemfile changes forbid patch upgrades, and --patch --strict is given" do + # force next minor via Gemfile + gemfile <<-G + source "#{file_uri_for(gem_repo4)}" + gem 'foo', '1.5.0' + gem 'qux' + G + + bundle "lock --update foo --patch --strict", raise_on_error: false + + expect(err).to include( + "foo is locked to 1.4.3, while Gemfile is requesting foo (= 1.5.0). " \ + "--strict --patch was specified, but there are no patch level upgrades from 1.4.3 satisfying foo (= 1.5.0), so version solving has failed" + ) + end + context "pre" do it "defaults to major" do bundle "lock --update --pre" From 508bddc86516ed798365594b70d57e9ec5713e8b Mon Sep 17 00:00:00 2001 From: tomoya ishida Date: Tue, 2 Apr 2024 01:25:17 +0900 Subject: [PATCH 190/211] [ruby/reline] Align completion menu items (https://github.com/ruby/reline/pull/613) https://github.com/ruby/reline/commit/a622704f62 --- lib/reline/line_editor.rb | 30 ++++++++++++++++++++++++++---- test/reline/test_line_editor.rb | 25 +++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 4 deletions(-) diff --git a/lib/reline/line_editor.rb b/lib/reline/line_editor.rb index d202ba02d27a76..d30ab3f36723ee 100644 --- a/lib/reline/line_editor.rb +++ b/lib/reline/line_editor.rb @@ -49,7 +49,29 @@ module CompletionState RenderedScreen = Struct.new(:base_y, :lines, :cursor_y, keyword_init: true) CompletionJourneyData = Struct.new(:preposing, :postposing, :list, :pointer) - MenuInfo = Struct.new(:target, :list) + + class MenuInfo + attr_reader :list + + def initialize(list) + @list = list + end + + def lines(screen_width) + return [] if @list.empty? + + list = @list.sort + sizes = list.map { |item| Reline::Unicode.calculate_width(item) } + item_width = sizes.max + 2 + num_cols = [screen_width / item_width, 1].max + num_rows = list.size.fdiv(num_cols).ceil + list_with_padding = list.zip(sizes).map { |item, size| item + ' ' * (item_width - size) } + aligned = (list_with_padding + [nil] * (num_rows * num_cols - list_with_padding.size)).each_slice(num_rows).to_a.transpose + aligned.map do |row| + row.join.rstrip + end + end + end MINIMUM_SCROLLBAR_HEIGHT = 1 @@ -458,7 +480,7 @@ def render_differential [[0, Reline::Unicode.calculate_width(l, true), l]] end if @menu_info - @menu_info.list.sort!.each do |item| + @menu_info.lines(screen_width).each do |item| new_lines << [[0, Reline::Unicode.calculate_width(item), item]] end @menu_info = nil # TODO: do not change state here @@ -779,8 +801,8 @@ def editing_mode @config.editing_mode end - private def menu(target, list) - @menu_info = MenuInfo.new(target, list) + private def menu(_target, list) + @menu_info = MenuInfo.new(list) end private def complete_internal_proc(list, is_menu) diff --git a/test/reline/test_line_editor.rb b/test/reline/test_line_editor.rb index 0963d2c8fe835c..bf688ac3c6f193 100644 --- a/test/reline/test_line_editor.rb +++ b/test/reline/test_line_editor.rb @@ -127,4 +127,29 @@ def test_complicated end end end + + def test_menu_info_format + list = %w[aa b c d e f g hhh i j k] + col3 = [ + 'aa e i', + 'b f j', + 'c g k', + 'd hhh' + ] + col2 = [ + 'aa g', + 'b hhh', + 'c i', + 'd j', + 'e k', + 'f' + ] + assert_equal(col3, Reline::LineEditor::MenuInfo.new(list).lines(19)) + assert_equal(col3, Reline::LineEditor::MenuInfo.new(list).lines(15)) + assert_equal(col2, Reline::LineEditor::MenuInfo.new(list).lines(14)) + assert_equal(col2, Reline::LineEditor::MenuInfo.new(list).lines(10)) + assert_equal(list, Reline::LineEditor::MenuInfo.new(list).lines(9)) + assert_equal(list, Reline::LineEditor::MenuInfo.new(list).lines(0)) + assert_equal([], Reline::LineEditor::MenuInfo.new([]).lines(10)) + end end From a531cac335653c6df5ba73932321d6227287aff6 Mon Sep 17 00:00:00 2001 From: tomoya ishida Date: Tue, 2 Apr 2024 03:12:23 +0900 Subject: [PATCH 191/211] [ruby/reline] Refactor completion (https://github.com/ruby/reline/pull/647) * Refactor completion: split autocompletion and tabcompletion logic and state * Move completion candidate listup logic from dialog proc to LineEditor https://github.com/ruby/reline/commit/c3c09ac9c2 --- lib/reline.rb | 30 ++-- lib/reline/line_editor.rb | 149 ++++++++++---------- test/reline/yamatanooroti/test_rendering.rb | 13 ++ 3 files changed, 100 insertions(+), 92 deletions(-) diff --git a/lib/reline.rb b/lib/reline.rb index f3fd28b627736a..b43584fc9bccbf 100644 --- a/lib/reline.rb +++ b/lib/reline.rb @@ -219,26 +219,16 @@ def get_screen_size Reline::DEFAULT_DIALOG_PROC_AUTOCOMPLETE = ->() { # autocomplete - return nil unless config.autocompletion - if just_cursor_moving and completion_journey_data.nil? - # Auto complete starts only when edited - return nil - end - pre, target, post = retrieve_completion_block(true) - if target.nil? or target.empty? or (completion_journey_data&.pointer == -1 and target.size <= 3) - return nil - end - if completion_journey_data and completion_journey_data.list - result = completion_journey_data.list.dup - result.shift - pointer = completion_journey_data.pointer - 1 - else - result = call_completion_proc_with_checking_args(pre, target, post) - pointer = nil - end - if result and result.size == 1 and result[0] == target and pointer != 0 - result = nil - end + return unless config.autocompletion + + journey_data = completion_journey_data + return unless journey_data + + target = journey_data.list[journey_data.pointer] + result = journey_data.list.drop(1) + pointer = journey_data.pointer - 1 + return if target.empty? || (result == [target] && pointer < 0) + target_width = Reline::Unicode.calculate_width(target) x = cursor_pos.x - target_width if x < 0 diff --git a/lib/reline/line_editor.rb b/lib/reline/line_editor.rb index d30ab3f36723ee..d5c158ac746e54 100644 --- a/lib/reline/line_editor.rb +++ b/lib/reline/line_editor.rb @@ -41,14 +41,13 @@ module CompletionState NORMAL = :normal COMPLETION = :completion MENU = :menu - JOURNEY = :journey MENU_WITH_PERFECT_MATCH = :menu_with_perfect_match PERFECT_MATCH = :perfect_match end RenderedScreen = Struct.new(:base_y, :lines, :cursor_y, keyword_init: true) - CompletionJourneyData = Struct.new(:preposing, :postposing, :list, :pointer) + CompletionJourneyState = Struct.new(:line_index, :pre, :target, :post, :list, :pointer) class MenuInfo attr_reader :list @@ -221,7 +220,7 @@ def reset_variables(prompt = '', encoding:) @waiting_proc = nil @waiting_operator_proc = nil @waiting_operator_vi_arg = nil - @completion_journey_data = nil + @completion_journey_state = nil @completion_state = CompletionState::NORMAL @perfect_matched = nil @menu_info = nil @@ -559,6 +558,8 @@ def rerender end class DialogProcScope + CompletionJourneyData = Struct.new(:preposing, :postposing, :list, :pointer) + def initialize(line_editor, config, proc_to_exec, context) @line_editor = line_editor @config = config @@ -622,7 +623,7 @@ def preferred_dialog_height end def completion_journey_data - @line_editor.instance_variable_get(:@completion_journey_data) + @line_editor.dialog_proc_scope_completion_journey_data end def config @@ -851,9 +852,9 @@ def editing_mode [target, preposing, completed, postposing] end - private def complete(list, just_show_list = false) + private def complete(list, just_show_list) case @completion_state - when CompletionState::NORMAL, CompletionState::JOURNEY + when CompletionState::NORMAL @completion_state = CompletionState::COMPLETION when CompletionState::PERFECT_MATCH @dig_perfect_match_proc&.(@perfect_matched) @@ -893,46 +894,44 @@ def editing_mode end end - private def move_completed_list(list, direction) - case @completion_state - when CompletionState::NORMAL, CompletionState::COMPLETION, - CompletionState::MENU, CompletionState::MENU_WITH_PERFECT_MATCH - @completion_state = CompletionState::JOURNEY - result = retrieve_completion_block - return if result.nil? - preposing, target, postposing = result - @completion_journey_data = CompletionJourneyData.new( - preposing, postposing, - [target] + list.select{ |item| item.start_with?(target) }, 0) - if @completion_journey_data.list.size == 1 - @completion_journey_data.pointer = 0 - else - case direction - when :up - @completion_journey_data.pointer = @completion_journey_data.list.size - 1 - when :down - @completion_journey_data.pointer = 1 - end - end - @completion_state = CompletionState::JOURNEY - else - case direction - when :up - @completion_journey_data.pointer -= 1 - if @completion_journey_data.pointer < 0 - @completion_journey_data.pointer = @completion_journey_data.list.size - 1 - end - when :down - @completion_journey_data.pointer += 1 - if @completion_journey_data.pointer >= @completion_journey_data.list.size - @completion_journey_data.pointer = 0 - end - end + def dialog_proc_scope_completion_journey_data + return nil unless @completion_journey_state + line_index = @completion_journey_state.line_index + pre_lines = @buffer_of_lines[0...line_index].map { |line| line + "\n" } + post_lines = @buffer_of_lines[(line_index + 1)..-1].map { |line| line + "\n" } + DialogProcScope::CompletionJourneyData.new( + pre_lines.join + @completion_journey_state.pre, + @completion_journey_state.post + post_lines.join, + @completion_journey_state.list, + @completion_journey_state.pointer + ) + end + + private def move_completed_list(direction) + @completion_journey_state ||= retrieve_completion_journey_state + return false unless @completion_journey_state + + if (delta = { up: -1, down: +1 }[direction]) + @completion_journey_state.pointer = (@completion_journey_state.pointer + delta) % @completion_journey_state.list.size end - completed = @completion_journey_data.list[@completion_journey_data.pointer] - line_to_pointer = (@completion_journey_data.preposing + completed).split("\n")[@line_index] || String.new(encoding: @encoding) - new_line = line_to_pointer + (@completion_journey_data.postposing.split("\n").first || '') - set_current_line(new_line, line_to_pointer.bytesize) + completed = @completion_journey_state.list[@completion_journey_state.pointer] + set_current_line(@completion_journey_state.pre + completed + @completion_journey_state.post, @completion_journey_state.pre.bytesize + completed.bytesize) + true + end + + private def retrieve_completion_journey_state + preposing, target, postposing = retrieve_completion_block + list = call_completion_proc + return unless list.is_a?(Array) + + candidates = list.select{ |item| item.start_with?(target) } + return if candidates.empty? + + pre = preposing.split("\n", -1).last || '' + post = postposing.split("\n", -1).first || '' + CompletionJourneyState.new( + @line_index, pre, target, post, [target] + candidates, 0 + ) end private def run_for_operators(key, method_symbol, &block) @@ -1121,50 +1120,56 @@ def input_key(key) @first_char = false completion_occurs = false if @config.editing_mode_is?(:emacs, :vi_insert) and key.char == "\C-i".ord - unless @config.disable_completion - result = call_completion_proc - if result.is_a?(Array) - completion_occurs = true - process_insert - if @config.autocompletion - move_completed_list(result, :down) - else - complete(result) + if !@config.disable_completion + process_insert(force: true) + if @config.autocompletion + @completion_state = CompletionState::NORMAL + completion_occurs = move_completed_list(:down) + else + @completion_journey_state = nil + result = call_completion_proc + if result.is_a?(Array) + completion_occurs = true + complete(result, false) end end end elsif @config.editing_mode_is?(:emacs, :vi_insert) and key.char == :completion_journey_up if not @config.disable_completion and @config.autocompletion - result = call_completion_proc - if result.is_a?(Array) - completion_occurs = true - process_insert - move_completed_list(result, :up) - end + process_insert(force: true) + @completion_state = CompletionState::NORMAL + completion_occurs = move_completed_list(:up) end - elsif not @config.disable_completion and @config.editing_mode_is?(:vi_insert) and ["\C-p".ord, "\C-n".ord].include?(key.char) - unless @config.disable_completion - result = call_completion_proc - if result.is_a?(Array) - completion_occurs = true - process_insert - move_completed_list(result, "\C-p".ord == key.char ? :up : :down) - end + elsif @config.editing_mode_is?(:vi_insert) and ["\C-p".ord, "\C-n".ord].include?(key.char) + # In vi mode, move completed list even if autocompletion is off + if not @config.disable_completion + process_insert(force: true) + @completion_state = CompletionState::NORMAL + completion_occurs = move_completed_list("\C-p".ord == key.char ? :up : :down) end elsif Symbol === key.char and respond_to?(key.char, true) process_key(key.char, key.char) else normal_char(key) end + unless completion_occurs @completion_state = CompletionState::NORMAL - @completion_journey_data = nil + @completion_journey_state = nil end + if @in_pasting clear_dialogs - else - return old_lines != @buffer_of_lines + return + end + + modified = old_lines != @buffer_of_lines + if !completion_occurs && modified && !@config.disable_completion && @config.autocompletion + # Auto complete starts only when edited + process_insert(force: true) + @completion_journey_state = retrieve_completion_journey_state end + modified end def scroll_into_view @@ -2042,7 +2047,7 @@ def finish private def em_delete_or_list(key) if current_line.empty? or @byte_pointer < current_line.bytesize em_delete(key) - else # show completed list + elsif !@config.autocompletion # show completed list result = call_completion_proc if result.is_a?(Array) complete(result, true) diff --git a/test/reline/yamatanooroti/test_rendering.rb b/test/reline/yamatanooroti/test_rendering.rb index efb1e01562b905..53c842a04a41c7 100644 --- a/test/reline/yamatanooroti/test_rendering.rb +++ b/test/reline/yamatanooroti/test_rendering.rb @@ -1108,6 +1108,19 @@ def test_autocomplete_target_is_wrapped EOC end + def test_force_insert_before_autocomplete + start_terminal(20, 20, %W{ruby -I#{@pwd}/lib #{@pwd}/test/reline/yamatanooroti/multiline_repl --autocomplete}, startup_message: 'Multiline REPL.') + write('Sy') + write(";St\t\t") + close + assert_screen(<<~'EOC') + Multiline REPL. + prompt> Sy;Struct + String + Struct + EOC + end + def test_simple_dialog_with_scroll_key start_terminal(20, 50, %W{ruby -I#{@pwd}/lib #{@pwd}/test/reline/yamatanooroti/multiline_repl --dialog long,scrollkey}, startup_message: 'Multiline REPL.') write('a') From f1e385aad978573c04e3eafb24e67ce253e1d302 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 1 Apr 2024 13:08:21 -0400 Subject: [PATCH 192/211] [ruby/prism] Track captures in pattern matching for duplicates https://github.com/ruby/prism/commit/aa2182f064 --- prism/config.yml | 1 + prism/prism.c | 284 +++++++++++++++------------ prism/templates/src/diagnostic.c.erb | 1 + test/prism/errors_test.rb | 16 ++ 4 files changed, 173 insertions(+), 129 deletions(-) diff --git a/prism/config.yml b/prism/config.yml index edbe4b32d84be5..658492a488b930 100644 --- a/prism/config.yml +++ b/prism/config.yml @@ -175,6 +175,7 @@ errors: - PARAMETER_STAR - PARAMETER_UNEXPECTED_FWD - PARAMETER_WILD_LOOSE_COMMA + - PATTERN_CAPTURE_DUPLICATE - PATTERN_EXPRESSION_AFTER_BRACKET - PATTERN_EXPRESSION_AFTER_COMMA - PATTERN_EXPRESSION_AFTER_HROCKET diff --git a/prism/prism.c b/prism/prism.c index 1f4b9ced5c67fd..6d46e06c97483a 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -4939,7 +4939,8 @@ pm_refute_numbered_parameter(pm_parser_t *parser, const uint8_t *start, const ui * name and depth. */ static pm_local_variable_target_node_t * -pm_local_variable_target_node_create_values(pm_parser_t *parser, const pm_location_t *location, pm_constant_id_t name, uint32_t depth) { +pm_local_variable_target_node_create(pm_parser_t *parser, const pm_location_t *location, pm_constant_id_t name, uint32_t depth) { + pm_refute_numbered_parameter(parser, location->start, location->end); pm_local_variable_target_node_t *node = PM_ALLOC_NODE(parser, pm_local_variable_target_node_t); *node = (pm_local_variable_target_node_t) { @@ -4954,36 +4955,6 @@ pm_local_variable_target_node_create_values(pm_parser_t *parser, const pm_locati return node; } -/** - * Allocate and initialize a new LocalVariableTargetNode node. - */ -static pm_local_variable_target_node_t * -pm_local_variable_target_node_create(pm_parser_t *parser, const pm_token_t *name) { - pm_refute_numbered_parameter(parser, name->start, name->end); - - return pm_local_variable_target_node_create_values( - parser, - &(pm_location_t) { .start = name->start, .end = name->end }, - pm_parser_constant_id_token(parser, name), - 0 - ); -} - -/** - * Allocate and initialize a new LocalVariableTargetNode node with the given depth. - */ -static pm_local_variable_target_node_t * -pm_local_variable_target_node_create_depth(pm_parser_t *parser, const pm_token_t *name, uint32_t depth) { - pm_refute_numbered_parameter(parser, name->start, name->end); - - return pm_local_variable_target_node_create_values( - parser, - &(pm_location_t) { .start = name->start, .end = name->end }, - pm_parser_constant_id_token(parser, name), - depth - ); -} - /** * Allocate and initialize a new MatchPredicateNode node. */ @@ -7086,9 +7057,9 @@ pm_parser_local_add_location(pm_parser_t *parser, const uint8_t *start, const ui /** * Add a local variable from a token to the current scope. */ -static inline void +static inline pm_constant_id_t pm_parser_local_add_token(pm_parser_t *parser, pm_token_t *token) { - pm_parser_local_add_location(parser, token->start, token->end); + return pm_parser_local_add_location(parser, token->start, token->end); } /** @@ -14625,13 +14596,27 @@ parse_heredoc_dedent(pm_parser_t *parser, pm_node_list_t *nodes, size_t common_w } static pm_node_t * -parse_pattern(pm_parser_t *parser, bool top_pattern, pm_diagnostic_id_t diag_id); +parse_pattern(pm_parser_t *parser, pm_constant_id_list_t *captures, bool top_pattern, pm_diagnostic_id_t diag_id); + +/** + * Add the newly created local to the list of captures for this pattern matching + * expression. If it is duplicated from a previous local, then we'll need to add + * an error to the parser. + */ +static void +parse_pattern_capture(pm_parser_t *parser, pm_constant_id_list_t *captures, pm_constant_id_t capture, const pm_location_t *location) { + if (pm_constant_id_list_includes(captures, capture)) { + pm_parser_err(parser, location->start, location->end, PM_ERR_PATTERN_CAPTURE_DUPLICATE); + } else { + pm_constant_id_list_append(captures, capture); + } +} /** * Accept any number of constants joined by :: delimiters. */ static pm_node_t * -parse_pattern_constant_path(pm_parser_t *parser, pm_node_t *node) { +parse_pattern_constant_path(pm_parser_t *parser, pm_constant_id_list_t *captures, pm_node_t *node) { // Now, if there are any :: operators that follow, parse them as constant // path nodes. while (accept1(parser, PM_TOKEN_COLON_COLON)) { @@ -14639,7 +14624,7 @@ parse_pattern_constant_path(pm_parser_t *parser, pm_node_t *node) { expect1(parser, PM_TOKEN_CONSTANT, PM_ERR_CONSTANT_PATH_COLON_COLON_CONSTANT); pm_node_t *child = (pm_node_t *) pm_constant_read_node_create(parser, &parser->previous); - node = (pm_node_t *)pm_constant_path_node_create(parser, node, &delimiter, child); + node = (pm_node_t *) pm_constant_path_node_create(parser, node, &delimiter, child); } // If there is a [ or ( that follows, then this is part of a larger pattern @@ -14658,7 +14643,7 @@ parse_pattern_constant_path(pm_parser_t *parser, pm_node_t *node) { accept1(parser, PM_TOKEN_NEWLINE); if (!accept1(parser, PM_TOKEN_BRACKET_RIGHT)) { - inner = parse_pattern(parser, true, PM_ERR_PATTERN_EXPRESSION_AFTER_BRACKET); + inner = parse_pattern(parser, captures, true, PM_ERR_PATTERN_EXPRESSION_AFTER_BRACKET); accept1(parser, PM_TOKEN_NEWLINE); expect1(parser, PM_TOKEN_BRACKET_RIGHT, PM_ERR_PATTERN_TERM_BRACKET); } @@ -14670,7 +14655,7 @@ parse_pattern_constant_path(pm_parser_t *parser, pm_node_t *node) { accept1(parser, PM_TOKEN_NEWLINE); if (!accept1(parser, PM_TOKEN_PARENTHESIS_RIGHT)) { - inner = parse_pattern(parser, true, PM_ERR_PATTERN_EXPRESSION_AFTER_PAREN); + inner = parse_pattern(parser, captures, true, PM_ERR_PATTERN_EXPRESSION_AFTER_PAREN); accept1(parser, PM_TOKEN_NEWLINE); expect1(parser, PM_TOKEN_PARENTHESIS_RIGHT, PM_ERR_PATTERN_TERM_PAREN); } @@ -14753,18 +14738,30 @@ parse_pattern_constant_path(pm_parser_t *parser, pm_node_t *node) { * Parse a rest pattern. */ static pm_splat_node_t * -parse_pattern_rest(pm_parser_t *parser) { +parse_pattern_rest(pm_parser_t *parser, pm_constant_id_list_t *captures) { assert(parser->previous.type == PM_TOKEN_USTAR); pm_token_t operator = parser->previous; pm_node_t *name = NULL; // Rest patterns don't necessarily have a name associated with them. So we - // will check for that here. If they do, then we'll add it to the local table - // since this pattern will cause it to become a local variable. + // will check for that here. If they do, then we'll add it to the local + // table since this pattern will cause it to become a local variable. if (accept1(parser, PM_TOKEN_IDENTIFIER)) { pm_token_t identifier = parser->previous; - pm_parser_local_add_token(parser, &identifier); - name = (pm_node_t *) pm_local_variable_target_node_create(parser, &identifier); + pm_constant_id_t constant_id = pm_parser_constant_id_token(parser, &identifier); + + int depth; + if ((depth = pm_parser_local_depth_constant_id(parser, constant_id)) == -1) { + pm_parser_local_add(parser, constant_id); + } + + parse_pattern_capture(parser, captures, constant_id, &PM_LOCATION_TOKEN_VALUE(&identifier)); + name = (pm_node_t *) pm_local_variable_target_node_create( + parser, + &PM_LOCATION_TOKEN_VALUE(&identifier), + constant_id, + (uint32_t) (depth == -1 ? 0 : depth) + ); } // Finally we can return the created node. @@ -14775,7 +14772,7 @@ parse_pattern_rest(pm_parser_t *parser) { * Parse a keyword rest node. */ static pm_node_t * -parse_pattern_keyword_rest(pm_parser_t *parser) { +parse_pattern_keyword_rest(pm_parser_t *parser, pm_constant_id_list_t *captures) { assert(parser->current.type == PM_TOKEN_USTAR_STAR); parser_lex(parser); @@ -14787,8 +14784,20 @@ parse_pattern_keyword_rest(pm_parser_t *parser) { } if (accept1(parser, PM_TOKEN_IDENTIFIER)) { - pm_parser_local_add_token(parser, &parser->previous); - value = (pm_node_t *) pm_local_variable_target_node_create(parser, &parser->previous); + pm_constant_id_t constant_id = pm_parser_constant_id_token(parser, &parser->previous); + + int depth; + if ((depth = pm_parser_local_depth_constant_id(parser, constant_id)) == -1) { + pm_parser_local_add(parser, constant_id); + } + + parse_pattern_capture(parser, captures, constant_id, &PM_LOCATION_TOKEN_VALUE(&parser->previous)); + value = (pm_node_t *) pm_local_variable_target_node_create( + parser, + &PM_LOCATION_TOKEN_VALUE(&parser->previous), + constant_id, + (uint32_t) (depth == -1 ? 0 : depth) + ); } return (pm_node_t *) pm_assoc_splat_node_create(parser, value, &operator); @@ -14799,21 +14808,23 @@ parse_pattern_keyword_rest(pm_parser_t *parser) { * value. This will use an implicit local variable target. */ static pm_node_t * -parse_pattern_hash_implicit_value(pm_parser_t *parser, pm_symbol_node_t *key) { +parse_pattern_hash_implicit_value(pm_parser_t *parser, pm_constant_id_list_t *captures, pm_symbol_node_t *key) { const pm_location_t *value_loc = &((pm_symbol_node_t *) key)->value_loc; - pm_constant_id_t name = pm_parser_constant_id_location(parser, value_loc->start, value_loc->end); - - int current_depth = pm_parser_local_depth_constant_id(parser, name); - uint32_t depth; + pm_constant_id_t constant_id = pm_parser_constant_id_location(parser, value_loc->start, value_loc->end); - if (current_depth == -1) { - pm_parser_local_add_location(parser, value_loc->start, value_loc->end); - depth = 0; - } else { - depth = (uint32_t) current_depth; + int depth; + if ((depth = pm_parser_local_depth_constant_id(parser, constant_id)) == -1) { + pm_parser_local_add(parser, constant_id); } - pm_local_variable_target_node_t *target = pm_local_variable_target_node_create_values(parser, value_loc, name, depth); + parse_pattern_capture(parser, captures, constant_id, value_loc); + pm_local_variable_target_node_t *target = pm_local_variable_target_node_create( + parser, + value_loc, + constant_id, + (uint32_t) (depth == -1 ? 0 : depth) + ); + return (pm_node_t *) pm_implicit_node_create(parser, (pm_node_t *) target); } @@ -14821,7 +14832,7 @@ parse_pattern_hash_implicit_value(pm_parser_t *parser, pm_symbol_node_t *key) { * Parse a hash pattern. */ static pm_hash_pattern_node_t * -parse_pattern_hash(pm_parser_t *parser, pm_node_t *first_node) { +parse_pattern_hash(pm_parser_t *parser, pm_constant_id_list_t *captures, pm_node_t *first_node) { pm_node_list_t assocs = { 0 }; pm_node_t *rest = NULL; @@ -14834,14 +14845,14 @@ parse_pattern_hash(pm_parser_t *parser, pm_node_t *first_node) { if (pm_symbol_node_label_p(first_node)) { pm_node_t *value; - if (!match7(parser, PM_TOKEN_COMMA, PM_TOKEN_KEYWORD_THEN, PM_TOKEN_BRACE_RIGHT, PM_TOKEN_BRACKET_RIGHT, PM_TOKEN_PARENTHESIS_RIGHT, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON)) { - // Here we have a value for the first assoc in the list, so - // we will parse it now. - value = parse_pattern(parser, false, PM_ERR_PATTERN_EXPRESSION_AFTER_KEY); - } else { + if (match7(parser, PM_TOKEN_COMMA, PM_TOKEN_KEYWORD_THEN, PM_TOKEN_BRACE_RIGHT, PM_TOKEN_BRACKET_RIGHT, PM_TOKEN_PARENTHESIS_RIGHT, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON)) { // Otherwise, we will create an implicit local variable // target for the value. - value = parse_pattern_hash_implicit_value(parser, (pm_symbol_node_t *) first_node); + value = parse_pattern_hash_implicit_value(parser, captures, (pm_symbol_node_t *) first_node); + } else { + // Here we have a value for the first assoc in the list, so + // we will parse it now. + value = parse_pattern(parser, captures, false, PM_ERR_PATTERN_EXPRESSION_AFTER_KEY); } pm_token_t operator = not_provided(parser); @@ -14875,7 +14886,7 @@ parse_pattern_hash(pm_parser_t *parser, pm_node_t *first_node) { } if (match1(parser, PM_TOKEN_USTAR_STAR)) { - pm_node_t *assoc = parse_pattern_keyword_rest(parser); + pm_node_t *assoc = parse_pattern_keyword_rest(parser, captures); if (rest == NULL) { rest = assoc; @@ -14888,12 +14899,10 @@ parse_pattern_hash(pm_parser_t *parser, pm_node_t *first_node) { pm_node_t *key = (pm_node_t *) pm_symbol_node_label_create(parser, &parser->previous); pm_node_t *value = NULL; - if (!match7(parser, PM_TOKEN_COMMA, PM_TOKEN_KEYWORD_THEN, PM_TOKEN_BRACE_RIGHT, PM_TOKEN_BRACKET_RIGHT, PM_TOKEN_PARENTHESIS_RIGHT, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON)) { - value = parse_pattern(parser, false, PM_ERR_PATTERN_EXPRESSION_AFTER_KEY); + if (match7(parser, PM_TOKEN_COMMA, PM_TOKEN_KEYWORD_THEN, PM_TOKEN_BRACE_RIGHT, PM_TOKEN_BRACKET_RIGHT, PM_TOKEN_PARENTHESIS_RIGHT, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON)) { + value = parse_pattern_hash_implicit_value(parser, captures, (pm_symbol_node_t *) key); } else { - const pm_location_t *value_loc = &((pm_symbol_node_t *) key)->value_loc; - pm_parser_local_add_location(parser, value_loc->start, value_loc->end); - value = parse_pattern_hash_implicit_value(parser, (pm_symbol_node_t *) key); + value = parse_pattern(parser, captures, false, PM_ERR_PATTERN_EXPRESSION_AFTER_KEY); } pm_token_t operator = not_provided(parser); @@ -14917,18 +14926,25 @@ parse_pattern_hash(pm_parser_t *parser, pm_node_t *first_node) { * Parse a pattern expression primitive. */ static pm_node_t * -parse_pattern_primitive(pm_parser_t *parser, pm_diagnostic_id_t diag_id) { +parse_pattern_primitive(pm_parser_t *parser, pm_constant_id_list_t *captures, pm_diagnostic_id_t diag_id) { switch (parser->current.type) { case PM_TOKEN_IDENTIFIER: case PM_TOKEN_METHOD_NAME: { parser_lex(parser); - pm_token_t name = parser->previous; - int depth = pm_parser_local_depth(parser, &name); - if (depth < 0) { - depth = 0; - pm_parser_local_add_token(parser, &name); + pm_constant_id_t constant_id = pm_parser_constant_id_token(parser, &parser->previous); + + int depth; + if ((depth = pm_parser_local_depth_constant_id(parser, constant_id)) == -1) { + pm_parser_local_add(parser, constant_id); } - return (pm_node_t *) pm_local_variable_target_node_create_depth(parser, &name, (uint32_t) depth); + + parse_pattern_capture(parser, captures, constant_id, &PM_LOCATION_TOKEN_VALUE(&parser->previous)); + return (pm_node_t *) pm_local_variable_target_node_create( + parser, + &PM_LOCATION_TOKEN_VALUE(&parser->previous), + constant_id, + (uint32_t) (depth == -1 ? 0 : depth) + ); } case PM_TOKEN_BRACKET_LEFT_ARRAY: { pm_token_t opening = parser->current; @@ -14937,15 +14953,14 @@ parse_pattern_primitive(pm_parser_t *parser, pm_diagnostic_id_t diag_id) { if (accept1(parser, PM_TOKEN_BRACKET_RIGHT)) { // If we have an empty array pattern, then we'll just return a new // array pattern node. - return (pm_node_t *)pm_array_pattern_node_empty_create(parser, &opening, &parser->previous); + return (pm_node_t *) pm_array_pattern_node_empty_create(parser, &opening, &parser->previous); } // Otherwise, we'll parse the inner pattern, then deal with it depending // on the type it returns. - pm_node_t *inner = parse_pattern(parser, true, PM_ERR_PATTERN_EXPRESSION_AFTER_BRACKET); + pm_node_t *inner = parse_pattern(parser, captures, true, PM_ERR_PATTERN_EXPRESSION_AFTER_BRACKET); accept1(parser, PM_TOKEN_NEWLINE); - expect1(parser, PM_TOKEN_BRACKET_RIGHT, PM_ERR_PATTERN_TERM_BRACKET); pm_token_t closing = parser->previous; @@ -15007,7 +15022,7 @@ parse_pattern_primitive(pm_parser_t *parser, pm_diagnostic_id_t diag_id) { first_node = (pm_node_t *) pm_symbol_node_label_create(parser, &parser->previous); break; case PM_TOKEN_USTAR_STAR: - first_node = parse_pattern_keyword_rest(parser); + first_node = parse_pattern_keyword_rest(parser, captures); break; case PM_TOKEN_STRING_BEGIN: first_node = parse_expression(parser, PM_BINDING_POWER_MAX, false, PM_ERR_PATTERN_HASH_KEY); @@ -15021,7 +15036,7 @@ parse_pattern_primitive(pm_parser_t *parser, pm_diagnostic_id_t diag_id) { } } - node = parse_pattern_hash(parser, first_node); + node = parse_pattern_hash(parser, captures, first_node); accept1(parser, PM_TOKEN_NEWLINE); expect1(parser, PM_TOKEN_BRACE_RIGHT, PM_ERR_PATTERN_TERM_BRACE); @@ -15168,14 +15183,14 @@ parse_pattern_primitive(pm_parser_t *parser, pm_diagnostic_id_t diag_id) { pm_node_t *child = (pm_node_t *) pm_constant_read_node_create(parser, &parser->previous); pm_constant_path_node_t *node = pm_constant_path_node_create(parser, NULL, &delimiter, child); - return parse_pattern_constant_path(parser, (pm_node_t *)node); + return parse_pattern_constant_path(parser, captures, (pm_node_t *) node); } case PM_TOKEN_CONSTANT: { pm_token_t constant = parser->current; parser_lex(parser); pm_node_t *node = (pm_node_t *) pm_constant_read_node_create(parser, &constant); - return parse_pattern_constant_path(parser, node); + return parse_pattern_constant_path(parser, captures, node); } default: pm_parser_err_current(parser, diag_id); @@ -15188,7 +15203,7 @@ parse_pattern_primitive(pm_parser_t *parser, pm_diagnostic_id_t diag_id) { * assignment. */ static pm_node_t * -parse_pattern_primitives(pm_parser_t *parser, pm_diagnostic_id_t diag_id) { +parse_pattern_primitives(pm_parser_t *parser, pm_constant_id_list_t *captures, pm_diagnostic_id_t diag_id) { pm_node_t *node = NULL; do { @@ -15205,9 +15220,9 @@ parse_pattern_primitives(pm_parser_t *parser, pm_diagnostic_id_t diag_id) { case PM_TOKEN_UDOT_DOT_DOT: case PM_CASE_PRIMITIVE: { if (node == NULL) { - node = parse_pattern_primitive(parser, diag_id); + node = parse_pattern_primitive(parser, captures, diag_id); } else { - pm_node_t *right = parse_pattern_primitive(parser, PM_ERR_PATTERN_EXPRESSION_AFTER_PIPE); + pm_node_t *right = parse_pattern_primitive(parser, captures, PM_ERR_PATTERN_EXPRESSION_AFTER_PIPE); node = (pm_node_t *) pm_alternation_pattern_node_create(parser, node, right, &operator); } @@ -15217,7 +15232,7 @@ parse_pattern_primitives(pm_parser_t *parser, pm_diagnostic_id_t diag_id) { pm_token_t opening = parser->current; parser_lex(parser); - pm_node_t *body = parse_pattern(parser, false, PM_ERR_PATTERN_EXPRESSION_AFTER_PAREN); + pm_node_t *body = parse_pattern(parser, captures, false, PM_ERR_PATTERN_EXPRESSION_AFTER_PAREN); accept1(parser, PM_TOKEN_NEWLINE); expect1(parser, PM_TOKEN_PARENTHESIS_RIGHT, PM_ERR_PATTERN_TERM_PAREN); pm_node_t *right = (pm_node_t *) pm_parentheses_node_create(parser, &opening, body, &parser->previous); @@ -15249,16 +15264,23 @@ parse_pattern_primitives(pm_parser_t *parser, pm_diagnostic_id_t diag_id) { // In this case we should create an assignment node. while (accept1(parser, PM_TOKEN_EQUAL_GREATER)) { pm_token_t operator = parser->previous; - expect1(parser, PM_TOKEN_IDENTIFIER, PM_ERR_PATTERN_IDENT_AFTER_HROCKET); - pm_token_t identifier = parser->previous; - int depth = pm_parser_local_depth(parser, &identifier); - if (depth < 0) { - depth = 0; - pm_parser_local_add_token(parser, &identifier); + + pm_constant_id_t constant_id = pm_parser_constant_id_token(parser, &parser->previous); + int depth; + + if ((depth = pm_parser_local_depth_constant_id(parser, constant_id)) == -1) { + pm_parser_local_add(parser, constant_id); } - pm_node_t *target = (pm_node_t *) pm_local_variable_target_node_create_depth(parser, &identifier, (uint32_t) depth); + parse_pattern_capture(parser, captures, constant_id, &PM_LOCATION_TOKEN_VALUE(&parser->previous)); + pm_node_t *target = (pm_node_t *) pm_local_variable_target_node_create( + parser, + &PM_LOCATION_TOKEN_VALUE(&parser->previous), + constant_id, + (uint32_t) (depth == -1 ? 0 : depth) + ); + node = (pm_node_t *) pm_capture_pattern_node_create(parser, node, target, &operator); } @@ -15269,7 +15291,7 @@ parse_pattern_primitives(pm_parser_t *parser, pm_diagnostic_id_t diag_id) { * Parse a pattern matching expression. */ static pm_node_t * -parse_pattern(pm_parser_t *parser, bool top_pattern, pm_diagnostic_id_t diag_id) { +parse_pattern(pm_parser_t *parser, pm_constant_id_list_t *captures, bool top_pattern, pm_diagnostic_id_t diag_id) { pm_node_t *node = NULL; bool leading_rest = false; @@ -15279,30 +15301,30 @@ parse_pattern(pm_parser_t *parser, bool top_pattern, pm_diagnostic_id_t diag_id) case PM_TOKEN_LABEL: { parser_lex(parser); pm_node_t *key = (pm_node_t *) pm_symbol_node_label_create(parser, &parser->previous); - return (pm_node_t *) parse_pattern_hash(parser, key); + return (pm_node_t *) parse_pattern_hash(parser, captures, key); } case PM_TOKEN_USTAR_STAR: { - node = parse_pattern_keyword_rest(parser); - return (pm_node_t *) parse_pattern_hash(parser, node); + node = parse_pattern_keyword_rest(parser, captures); + return (pm_node_t *) parse_pattern_hash(parser, captures, node); } case PM_TOKEN_USTAR: { if (top_pattern) { parser_lex(parser); - node = (pm_node_t *) parse_pattern_rest(parser); + node = (pm_node_t *) parse_pattern_rest(parser, captures); leading_rest = true; break; } } /* fallthrough */ default: - node = parse_pattern_primitives(parser, diag_id); + node = parse_pattern_primitives(parser, captures, diag_id); break; } // If we got a dynamic label symbol, then we need to treat it like the // beginning of a hash pattern. if (pm_symbol_node_label_p(node)) { - return (pm_node_t *) parse_pattern_hash(parser, node); + return (pm_node_t *) parse_pattern_hash(parser, captures, node); } if (top_pattern && match1(parser, PM_TOKEN_COMMA)) { @@ -15322,7 +15344,7 @@ parse_pattern(pm_parser_t *parser, bool top_pattern, pm_diagnostic_id_t diag_id) } if (accept1(parser, PM_TOKEN_USTAR)) { - node = (pm_node_t *) parse_pattern_rest(parser); + node = (pm_node_t *) parse_pattern_rest(parser, captures); // If we have already parsed a splat pattern, then this is an error. We // will continue to parse the rest of the patterns, but we will indicate @@ -15333,7 +15355,7 @@ parse_pattern(pm_parser_t *parser, bool top_pattern, pm_diagnostic_id_t diag_id) trailing_rest = true; } else { - node = parse_pattern_primitives(parser, PM_ERR_PATTERN_EXPRESSION_AFTER_COMMA); + node = parse_pattern_primitives(parser, captures, PM_ERR_PATTERN_EXPRESSION_AFTER_COMMA); } pm_node_list_append(&nodes, node); @@ -16300,8 +16322,8 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b return (pm_node_t *) pm_case_node_create(parser, &case_keyword, predicate, &parser->previous); } - // At this point we can create a case node, though we don't yet know if it - // is a case-in or case-when node. + // At this point we can create a case node, though we don't yet know + // if it is a case-in or case-when node. pm_token_t end_keyword = not_provided(parser); pm_node_t *node; @@ -16379,8 +16401,9 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_parser_err_token(parser, &case_keyword, PM_ERR_CASE_MATCH_MISSING_PREDICATE); } - // At this point we expect that we're parsing a case-in node. We will - // continue to parse the in nodes until we hit the end of the list. + // At this point we expect that we're parsing a case-in node. We + // will continue to parse the in nodes until we hit the end of + // the list. while (match1(parser, PM_TOKEN_KEYWORD_IN)) { bool previous_pattern_matching_newlines = parser->pattern_matching_newlines; parser->pattern_matching_newlines = true; @@ -16390,11 +16413,16 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b parser_lex(parser); pm_token_t in_keyword = parser->previous; - pm_node_t *pattern = parse_pattern(parser, true, PM_ERR_PATTERN_EXPRESSION_AFTER_IN); + + pm_constant_id_list_t captures = { 0 }; + pm_node_t *pattern = parse_pattern(parser, &captures, true, PM_ERR_PATTERN_EXPRESSION_AFTER_IN); + parser->pattern_matching_newlines = previous_pattern_matching_newlines; + pm_constant_id_list_free(&captures); - // Since we're in the top-level of the case-in node we need to check - // for guard clauses in the form of `if` or `unless` statements. + // Since we're in the top-level of the case-in node we need + // to check for guard clauses in the form of `if` or + // `unless` statements. if (accept1(parser, PM_TOKEN_KEYWORD_IF_MODIFIER)) { pm_token_t keyword = parser->previous; pm_node_t *predicate = parse_value_expression(parser, PM_BINDING_POWER_COMPOSITION, true, PM_ERR_CONDITIONAL_IF_PREDICATE); @@ -16405,9 +16433,9 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pattern = (pm_node_t *) pm_unless_node_modifier_create(parser, pattern, &keyword, predicate); } - // Now we need to check for the terminator of the in node's pattern. - // It can be a newline or semicolon optionally followed by a `then` - // keyword. + // Now we need to check for the terminator of the in node's + // pattern. It can be a newline or semicolon optionally + // followed by a `then` keyword. pm_token_t then_keyword; if (accept2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON)) { if (accept1(parser, PM_TOKEN_KEYWORD_THEN)) { @@ -16420,8 +16448,8 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b then_keyword = parser->previous; } - // Now we can actually parse the statements associated with the in - // node. + // Now we can actually parse the statements associated with + // the in node. pm_statements_node_t *statements; if (match3(parser, PM_TOKEN_KEYWORD_IN, PM_TOKEN_KEYWORD_ELSE, PM_TOKEN_KEYWORD_END)) { statements = NULL; @@ -16429,8 +16457,8 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b statements = parse_statements(parser, PM_CONTEXT_CASE_IN); } - // Now that we have the full pattern and statements, we can create the - // node and attach it to the case node. + // Now that we have the full pattern and statements, we can + // create the node and attach it to the case node. pm_node_t *condition = (pm_node_t *) pm_in_node_create(parser, pattern, statements, &in_keyword, &then_keyword); pm_case_match_node_condition_append(case_node, condition); } @@ -18152,7 +18180,6 @@ parse_regular_expression_named_captures(pm_parser_t *parser, const pm_string_t * // copy the names directly. The pointers will line up. location = (pm_location_t) { .start = source, .end = source + length }; name = pm_parser_constant_id_location(parser, location.start, location.end); - pm_refute_numbered_parameter(parser, source, source + length); } else { // Otherwise, the name is a slice of the malloc-ed owned string, // in which case we need to copy it out into a new string. @@ -18163,11 +18190,6 @@ parse_regular_expression_named_captures(pm_parser_t *parser, const pm_string_t * memcpy(memory, source, length); name = pm_parser_constant_id_owned(parser, (uint8_t *) memory, length); - - if (pm_token_is_numbered_parameter(source, source + length)) { - const pm_location_t *location = &call->receiver->location; - PM_PARSER_ERR_LOCATION_FORMAT(parser, location, PM_ERR_PARAMETER_NUMBERED_RESERVED, location->start); - } } if (name != 0) { @@ -18191,7 +18213,7 @@ parse_regular_expression_named_captures(pm_parser_t *parser, const pm_string_t * // Next, create the local variable target and add it to the // list of targets for the match. - pm_node_t *target = (pm_node_t *) pm_local_variable_target_node_create_values(parser, &location, name, depth == -1 ? 0 : (uint32_t) depth); + pm_node_t *target = (pm_node_t *) pm_local_variable_target_node_create(parser, &location, name, depth == -1 ? 0 : (uint32_t) depth); pm_node_list_append(&match->targets, target); } } @@ -18963,11 +18985,13 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t pm_token_t operator = parser->current; parser->command_start = false; lex_state_set(parser, PM_LEX_STATE_BEG | PM_LEX_STATE_LABEL); - parser_lex(parser); - pm_node_t *pattern = parse_pattern(parser, true, PM_ERR_PATTERN_EXPRESSION_AFTER_IN); + pm_constant_id_list_t captures = { 0 }; + pm_node_t *pattern = parse_pattern(parser, &captures, true, PM_ERR_PATTERN_EXPRESSION_AFTER_IN); + parser->pattern_matching_newlines = previous_pattern_matching_newlines; + pm_constant_id_list_free(&captures); return (pm_node_t *) pm_match_predicate_node_create(parser, node, pattern, &operator); } @@ -18978,11 +19002,13 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t pm_token_t operator = parser->current; parser->command_start = false; lex_state_set(parser, PM_LEX_STATE_BEG | PM_LEX_STATE_LABEL); - parser_lex(parser); - pm_node_t *pattern = parse_pattern(parser, true, PM_ERR_PATTERN_EXPRESSION_AFTER_HROCKET); + pm_constant_id_list_t captures = { 0 }; + pm_node_t *pattern = parse_pattern(parser, &captures, true, PM_ERR_PATTERN_EXPRESSION_AFTER_HROCKET); + parser->pattern_matching_newlines = previous_pattern_matching_newlines; + pm_constant_id_list_free(&captures); return (pm_node_t *) pm_match_required_node_create(parser, node, pattern, &operator); } diff --git a/prism/templates/src/diagnostic.c.erb b/prism/templates/src/diagnostic.c.erb index b608c44f01afce..b403eeaa8c87dd 100644 --- a/prism/templates/src/diagnostic.c.erb +++ b/prism/templates/src/diagnostic.c.erb @@ -258,6 +258,7 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_ERR_PARAMETER_STAR] = { "unexpected parameter `*`", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_PARAMETER_UNEXPECTED_FWD] = { "unexpected `...` in parameters", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_PARAMETER_WILD_LOOSE_COMMA] = { "unexpected `,` in parameters", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PATTERN_CAPTURE_DUPLICATE] = { "duplicated variable name", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_PATTERN_EXPRESSION_AFTER_BRACKET] = { "expected a pattern expression after the `[` operator", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_PATTERN_EXPRESSION_AFTER_COMMA] = { "expected a pattern expression after `,`", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_PATTERN_EXPRESSION_AFTER_HROCKET] = { "expected a pattern expression after `=>`", PM_ERROR_LEVEL_SYNTAX }, diff --git a/test/prism/errors_test.rb b/test/prism/errors_test.rb index 6cc71f9647172e..b3ae1c8ec59b46 100644 --- a/test/prism/errors_test.rb +++ b/test/prism/errors_test.rb @@ -2157,6 +2157,22 @@ def test_assignment_to_literal_in_conditionals ] * source.lines.count end + def test_duplicate_pattern_capture + source = <<~RUBY + () => [a, a] + () => [a, *a] + () => {a:, a:} + () => {a: a, a: a} + () => {a: a, **a} + () => [a, {a:}] + () => [a, {a: {a: {a: [a]}}}] + () => a => a + () => [A => a, {a: b => a}] + RUBY + + assert_error_messages source, Array.new(source.lines.length, "duplicated variable name"), compare_ripper: false + end + private def assert_errors(expected, source, errors, compare_ripper: RUBY_ENGINE == "ruby") From c2735c48a1465b88cf5a8c67e5d08ec521230828 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 1 Apr 2024 13:33:16 -0400 Subject: [PATCH 193/211] [ruby/prism] Track duplicate hash keys for pattern matching https://github.com/ruby/prism/commit/71ea82f299 --- prism/config.yml | 1 + prism/prism.c | 12 ++++++++++++ prism/templates/src/diagnostic.c.erb | 1 + test/prism/errors_test.rb | 14 ++++++++++++-- 4 files changed, 26 insertions(+), 2 deletions(-) diff --git a/prism/config.yml b/prism/config.yml index 658492a488b930..34cd388f485cb0 100644 --- a/prism/config.yml +++ b/prism/config.yml @@ -187,6 +187,7 @@ errors: - PATTERN_EXPRESSION_AFTER_RANGE - PATTERN_EXPRESSION_AFTER_REST - PATTERN_HASH_KEY + - PATTERN_HASH_KEY_DUPLICATE - PATTERN_HASH_KEY_LABEL - PATTERN_IDENT_AFTER_HROCKET - PATTERN_LABEL_AFTER_COMMA diff --git a/prism/prism.c b/prism/prism.c index 6d46e06c97483a..15c7e4568d5dc2 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -14828,12 +14828,20 @@ parse_pattern_hash_implicit_value(pm_parser_t *parser, pm_constant_id_list_t *ca return (pm_node_t *) pm_implicit_node_create(parser, (pm_node_t *) target); } +static void +parse_pattern_hash_key(pm_parser_t *parser, pm_static_literals_t *keys, pm_node_t *node) { + if (pm_static_literals_add(parser, keys, node) != NULL) { + pm_parser_err_node(parser, node, PM_ERR_PATTERN_HASH_KEY_DUPLICATE); + } +} + /** * Parse a hash pattern. */ static pm_hash_pattern_node_t * parse_pattern_hash(pm_parser_t *parser, pm_constant_id_list_t *captures, pm_node_t *first_node) { pm_node_list_t assocs = { 0 }; + pm_static_literals_t keys = { 0 }; pm_node_t *rest = NULL; switch (PM_NODE_TYPE(first_node)) { @@ -14843,6 +14851,7 @@ parse_pattern_hash(pm_parser_t *parser, pm_constant_id_list_t *captures, pm_node break; case PM_SYMBOL_NODE: { if (pm_symbol_node_label_p(first_node)) { + parse_pattern_hash_key(parser, &keys, first_node); pm_node_t *value; if (match7(parser, PM_TOKEN_COMMA, PM_TOKEN_KEYWORD_THEN, PM_TOKEN_BRACE_RIGHT, PM_TOKEN_BRACKET_RIGHT, PM_TOKEN_PARENTHESIS_RIGHT, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON)) { @@ -14897,6 +14906,8 @@ parse_pattern_hash(pm_parser_t *parser, pm_constant_id_list_t *captures, pm_node } else { expect1(parser, PM_TOKEN_LABEL, PM_ERR_PATTERN_LABEL_AFTER_COMMA); pm_node_t *key = (pm_node_t *) pm_symbol_node_label_create(parser, &parser->previous); + + parse_pattern_hash_key(parser, &keys, key); pm_node_t *value = NULL; if (match7(parser, PM_TOKEN_COMMA, PM_TOKEN_KEYWORD_THEN, PM_TOKEN_BRACE_RIGHT, PM_TOKEN_BRACKET_RIGHT, PM_TOKEN_PARENTHESIS_RIGHT, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON)) { @@ -14919,6 +14930,7 @@ parse_pattern_hash(pm_parser_t *parser, pm_constant_id_list_t *captures, pm_node pm_hash_pattern_node_t *node = pm_hash_pattern_node_node_list_create(parser, &assocs, rest); xfree(assocs.nodes); + pm_static_literals_free(&keys); return node; } diff --git a/prism/templates/src/diagnostic.c.erb b/prism/templates/src/diagnostic.c.erb index b403eeaa8c87dd..301f98134f592c 100644 --- a/prism/templates/src/diagnostic.c.erb +++ b/prism/templates/src/diagnostic.c.erb @@ -270,6 +270,7 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_ERR_PATTERN_EXPRESSION_AFTER_RANGE] = { "expected a pattern expression after the range operator", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_PATTERN_EXPRESSION_AFTER_REST] = { "unexpected pattern expression after the `**` expression", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_PATTERN_HASH_KEY] = { "expected a key in the hash pattern", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PATTERN_HASH_KEY_DUPLICATE] = { "duplicated key name", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_PATTERN_HASH_KEY_LABEL] = { "expected a label as the key in the hash pattern", PM_ERROR_LEVEL_SYNTAX }, // TODO // THIS // AND // ABOVE // IS WEIRD [PM_ERR_PATTERN_IDENT_AFTER_HROCKET] = { "expected an identifier after the `=>` operator", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_PATTERN_LABEL_AFTER_COMMA] = { "expected a label after the `,` in the hash pattern", PM_ERROR_LEVEL_SYNTAX }, diff --git a/test/prism/errors_test.rb b/test/prism/errors_test.rb index b3ae1c8ec59b46..d9dedc88e1ea64 100644 --- a/test/prism/errors_test.rb +++ b/test/prism/errors_test.rb @@ -2161,8 +2161,7 @@ def test_duplicate_pattern_capture source = <<~RUBY () => [a, a] () => [a, *a] - () => {a:, a:} - () => {a: a, a: a} + () => {a: a, b: a} () => {a: a, **a} () => [a, {a:}] () => [a, {a: {a: {a: [a]}}}] @@ -2173,6 +2172,12 @@ def test_duplicate_pattern_capture assert_error_messages source, Array.new(source.lines.length, "duplicated variable name"), compare_ripper: false end + def test_duplicate_pattern_hash_key + assert_error_messages "() => {a:, a:}", ["duplicated key name", "duplicated variable name"] + assert_error_messages "() => {a:1, a:2}", ["duplicated key name"] + refute_error_messages "() => [{a:1}, {a:2}]" + end + private def assert_errors(expected, source, errors, compare_ripper: RUBY_ENGINE == "ruby") @@ -2192,6 +2197,11 @@ def assert_error_messages(source, errors, compare_ripper: RUBY_ENGINE == "ruby") assert_equal(errors, result.errors.map(&:message)) end + def refute_error_messages(source, compare_ripper: RUBY_ENGINE == "ruby") + refute_nil Ripper.sexp_raw(source) if compare_ripper + assert Prism.parse_success?(source) + end + def assert_warning_messages(source, warnings) result = Prism.parse(source) assert_equal(warnings, result.warnings.map(&:message)) From ec879e7eac89dd7ef4f67eb737b3a633a96e280f Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 1 Apr 2024 13:48:09 -0400 Subject: [PATCH 194/211] [ruby/prism] Use RubyVM::InstructionSequence instead of Ripper for validity check https://github.com/ruby/prism/commit/ddec1c163d --- test/prism/errors_test.rb | 138 +++++++++++++++++++++++--------------- 1 file changed, 83 insertions(+), 55 deletions(-) diff --git a/test/prism/errors_test.rb b/test/prism/errors_test.rb index d9dedc88e1ea64..856e46fb76944a 100644 --- a/test/prism/errors_test.rb +++ b/test/prism/errors_test.rb @@ -1130,28 +1130,24 @@ def test_dont_allow_setting_to_back_and_nth_reference end def test_duplicated_parameter_names - # For some reason, Ripper reports no error for Ruby 3.0 when you have - # duplicated parameter names for positional parameters. - unless RUBY_VERSION < "3.1.0" - expected = DefNode( - :foo, - Location(), - nil, - ParametersNode([RequiredParameterNode(0, :a), RequiredParameterNode(0, :b), RequiredParameterNode(ParameterFlags::REPEATED_PARAMETER, :a)], [], nil, [], [], nil, nil), - nil, - [:a, :b], - Location(), - nil, - Location(), - Location(), - nil, - Location() - ) + expected = DefNode( + :foo, + Location(), + nil, + ParametersNode([RequiredParameterNode(0, :a), RequiredParameterNode(0, :b), RequiredParameterNode(ParameterFlags::REPEATED_PARAMETER, :a)], [], nil, [], [], nil, nil), + nil, + [:a, :b], + Location(), + nil, + Location(), + Location(), + nil, + Location() + ) - assert_errors expected, "def foo(a,b,a);end", [ - ["duplicated argument name", 12..13] - ] - end + assert_errors expected, "def foo(a,b,a);end", [ + ["duplicated argument name", 12..13] + ] expected = DefNode( :foo, @@ -1370,7 +1366,7 @@ def test_double_scope_numbered_parameters source = "-> { _1 + -> { _2 } }" errors = [["numbered parameter is already used in outer scope", 15..17]] - assert_errors expression(source), source, errors, compare_ripper: false + assert_errors expression(source), source, errors end def test_invalid_number_underscores @@ -1436,7 +1432,7 @@ def test_parameter_name_ending_with_bang_or_question_mark ["unexpected name for a parameter", 8..10], ["unexpected name for a parameter", 11..13] ] - assert_errors expression(source), source, errors, compare_ripper: false + assert_errors expression(source), source, errors end def test_class_name @@ -1488,11 +1484,10 @@ def test_shadow_args_in_block def test_repeated_parameter_name_in_destructured_params source = "def f(a, (b, (a))); end" - # In Ruby 3.0.x, `Ripper.sexp_raw` does not return `nil` for this case. - compare_ripper = RUBY_ENGINE == "ruby" && (RUBY_VERSION.split('.').map { |x| x.to_i } <=> [3, 1]) >= 1 + assert_errors expression(source), source, [ ["duplicated argument name", 14..15], - ], compare_ripper: compare_ripper + ] end def test_assign_to_numbered_parameter @@ -1574,6 +1569,7 @@ def test_check_value_expression 1 => ^(if 1; (return) else (return) end) 1 => ^(unless 1; (return) else (return) end) RUBY + message = 'unexpected void value expression' assert_errors expression(source), source, [ [message, 7..13], @@ -1584,7 +1580,7 @@ def test_check_value_expression [message, 97..103], [message, 123..129], [message, 168..174], - ], compare_ripper: false # Ripper does not check 'void value expression'. + ] end def test_void_value_expression_in_statement @@ -1607,6 +1603,7 @@ class << (return) for x in (return) end RUBY + message = 'unexpected void value expression' assert_errors expression(source), source, [ [message, 4..10], @@ -1617,7 +1614,7 @@ class << (return) [message, 110..116], [message, 132..138], [message, 154..160], - ], compare_ripper: false # Ripper does not check 'void value expression'. + ] end def test_void_value_expression_in_def @@ -1629,12 +1626,13 @@ def x(a = return) def x(a: return) end RUBY + message = 'unexpected void value expression' assert_errors expression(source), source, [ [message, 5..11], [message, 29..35], [message, 50..56], - ], compare_ripper: false # Ripper does not check 'void value expression'. + ] end def test_void_value_expression_in_assignment @@ -1644,13 +1642,14 @@ def test_void_value_expression_in_assignment a, b = return, 1 a, b = 1, *return RUBY + message = 'unexpected void value expression' assert_errors expression(source), source, [ [message, 4..10], [message, 18..24], [message, 32..38], [message, 53..59], - ], compare_ripper: false # Ripper does not check 'void value expression'. + ] end def test_void_value_expression_in_modifier @@ -1662,6 +1661,7 @@ def test_void_value_expression_in_modifier (return) => a (return) in a RUBY + message = 'unexpected void value expression' assert_errors expression(source), source, [ [message, 6..12], @@ -1670,7 +1670,7 @@ def test_void_value_expression_in_modifier [message, 58..64], [message, 67..73], [message, 81..87] - ], compare_ripper: false # Ripper does not check 'void value expression'. + ] end def test_void_value_expression_in_expression @@ -1685,6 +1685,7 @@ def test_void_value_expression_in_expression ((return)..) ((return)...) RUBY + message = 'unexpected void value expression' assert_errors expression(source), source, [ [message, 1..7], @@ -1696,7 +1697,7 @@ def test_void_value_expression_in_expression [message, 85..91], [message, 96..102], [message, 109..115] - ], compare_ripper: false # Ripper does not check 'void value expression'. + ] end def test_void_value_expression_in_array @@ -1709,6 +1710,7 @@ def test_void_value_expression_in_array [ *return ] [ **return ] RUBY + message = 'unexpected void value expression' assert_errors expression(source), source, [ [message, 1..7], @@ -1718,7 +1720,7 @@ def test_void_value_expression_in_array [message, 58..64], [message, 70..76], [message, 83..89], - ], compare_ripper: false # Ripper does not check 'void value expression'. + ] end def test_void_value_expression_in_hash @@ -1728,13 +1730,14 @@ def test_void_value_expression_in_hash { a: return } { **return } RUBY + message = 'unexpected void value expression' assert_errors expression(source), source, [ [message, 2..8], [message, 23..29], [message, 37..43], [message, 50..56], - ], compare_ripper: false # Ripper does not check 'void value expression'. + ] end def test_void_value_expression_in_call @@ -1745,6 +1748,7 @@ def test_void_value_expression_in_call (return)[1] = 2 (return)::foo RUBY + message = 'unexpected void value expression' assert_errors expression(source), source, [ [message, 1..7], @@ -1752,7 +1756,7 @@ def test_void_value_expression_in_call [message, 27..33], [message, 39..45], [message, 55..61], - ], compare_ripper: false # Ripper does not check 'void value expression'. + ] end def test_void_value_expression_in_constant_path @@ -1760,11 +1764,12 @@ def test_void_value_expression_in_constant_path (return)::A class (return)::A; end RUBY + message = 'unexpected void value expression' assert_errors expression(source), source, [ [message, 1..7], [message, 19..25], - ], compare_ripper: false # Ripper does not check 'void value expression'. + ] end def test_void_value_expression_in_arguments @@ -1778,6 +1783,7 @@ def test_void_value_expression_in_arguments foo(:a => return) foo(a: return) RUBY + message = 'unexpected void value expression' assert_errors expression(source), source, [ [message, 4..10], @@ -1788,7 +1794,7 @@ def test_void_value_expression_in_arguments [message, 71..77], [message, 94..100], [message, 109..115], - ], compare_ripper: false # Ripper does not check 'void value expression'. + ] end def test_void_value_expression_in_unary_call @@ -1796,11 +1802,12 @@ def test_void_value_expression_in_unary_call +(return) not return RUBY + message = 'unexpected void value expression' assert_errors expression(source), source, [ [message, 2..8], [message, 14..20], - ], compare_ripper: false # Ripper does not check 'void value expression'. + ] end def test_void_value_expression_in_binary_call @@ -1812,13 +1819,14 @@ def test_void_value_expression_in_binary_call 1 or (return) (return) or 1 RUBY + message = 'unexpected void value expression' assert_errors expression(source), source, [ [message, 5..11], [message, 14..20], [message, 42..48], [message, 71..77], - ], compare_ripper: false # Ripper does not check 'void value expression'. + ] end def test_trailing_comma_in_calls @@ -1934,13 +1942,14 @@ def foo(bar: bar) = 42 proc { |foo = foo| } proc { |foo: foo| } RUBY + message = 'parameter default value references itself' assert_errors expression(source), source, [ [message, 14..17], [message, 37..40], [message, 61..64], [message, 81..84], - ], compare_ripper: false # Ripper does not check 'circular reference'. + ] end def test_command_calls @@ -1976,8 +1985,9 @@ def a = b rescue c d begin; rescue a b; end begin; rescue a b => c; end RUBY + sources.each do |source| - assert_nil Ripper.sexp_raw(source) + refute_valid_syntax(source) assert_false(Prism.parse(source).success?) end end @@ -1993,8 +2003,9 @@ def test_range_and_bin_op 1.. % 2 1.. ** 2 RUBY + sources.each do |source| - assert_nil Ripper.sexp_raw(source) + refute_valid_syntax(source) assert_false(Prism.parse(source).success?) end end @@ -2050,21 +2061,21 @@ def test_block_arg_and_block source = 'foo(&1) { }' assert_errors expression(source), source, [ ['multiple block arguments; only one block is allowed', 8..11] - ], compare_ripper: false # Ripper does not check 'both block arg and actual block given'. + ] end def test_forwarding_arg_and_block source = 'def foo(...) = foo(...) { }' assert_errors expression(source), source, [ ['both a block argument and a forwarding argument; only one block is allowed', 24..27] - ], compare_ripper: false # Ripper does not check 'both block arg and actual block given'. + ] end def test_it_with_ordinary_parameter source = "proc { || it }" errors = [["`it` is not allowed when an ordinary parameter is defined", 10..12]] - assert_errors expression(source), source, errors, compare_ripper: false + assert_errors expression(source), source, errors end def test_regular_expression_with_unknown_regexp_options @@ -2124,7 +2135,7 @@ def ([1]).foo; end ["cannot define singleton method for literals", 380..388], ["cannot define singleton method for literals", 404..407] ] - assert_errors expression(source), source, errors, compare_ripper: false + assert_errors expression(source), source, errors end def test_assignment_to_literal_in_conditionals @@ -2169,20 +2180,37 @@ def test_duplicate_pattern_capture () => [A => a, {a: b => a}] RUBY - assert_error_messages source, Array.new(source.lines.length, "duplicated variable name"), compare_ripper: false + assert_error_messages source, Array.new(source.lines.length, "duplicated variable name") end def test_duplicate_pattern_hash_key assert_error_messages "() => {a:, a:}", ["duplicated key name", "duplicated variable name"] assert_error_messages "() => {a:1, a:2}", ["duplicated key name"] - refute_error_messages "() => [{a:1}, {a:2}]" + refute_error_messages "case (); in [{a:1}, {a:2}]; end" end private - def assert_errors(expected, source, errors, compare_ripper: RUBY_ENGINE == "ruby") - # Ripper behaves differently on JRuby/TruffleRuby, so only check this on CRuby - assert_nil Ripper.sexp_raw(source) if compare_ripper + def check_syntax(source) + $VERBOSE, previous = nil, $VERBOSE + + begin + RubyVM::InstructionSequence.compile(source) + ensure + $VERBOSE = previous + end + end + + def assert_valid_syntax(source) + check_syntax(source) + end + + def refute_valid_syntax(source) + assert_raise(SyntaxError) { check_syntax(source) } + end + + def assert_errors(expected, source, errors) + refute_valid_syntax(source) if RUBY_ENGINE == "ruby" result = Prism.parse(source) node = result.value.statements.body.last @@ -2191,14 +2219,14 @@ def assert_errors(expected, source, errors, compare_ripper: RUBY_ENGINE == "ruby assert_equal(errors, result.errors.map { |e| [e.message, e.location.start_offset..e.location.end_offset] }) end - def assert_error_messages(source, errors, compare_ripper: RUBY_ENGINE == "ruby") - assert_nil Ripper.sexp_raw(source) if compare_ripper + def assert_error_messages(source, errors) + refute_valid_syntax(source) if RUBY_ENGINE == "ruby" result = Prism.parse(source) assert_equal(errors, result.errors.map(&:message)) end - def refute_error_messages(source, compare_ripper: RUBY_ENGINE == "ruby") - refute_nil Ripper.sexp_raw(source) if compare_ripper + def refute_error_messages(source) + assert_valid_syntax(source) if RUBY_ENGINE == "ruby" assert Prism.parse_success?(source) end From a9658b64094db1371550a5b8ca47308cdd60db78 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 1 Apr 2024 13:57:38 -0400 Subject: [PATCH 195/211] [ruby/prism] Do not track locals starting with _ https://github.com/ruby/prism/commit/0d5a6d936a --- prism/prism.c | 7 +++++ test/prism/errors_test.rb | 63 ++++++++++++++++++++++----------------- 2 files changed, 43 insertions(+), 27 deletions(-) diff --git a/prism/prism.c b/prism/prism.c index 15c7e4568d5dc2..2a288cae934db0 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -14605,6 +14605,9 @@ parse_pattern(pm_parser_t *parser, pm_constant_id_list_t *captures, bool top_pat */ static void parse_pattern_capture(pm_parser_t *parser, pm_constant_id_list_t *captures, pm_constant_id_t capture, const pm_location_t *location) { + // Skip this capture if it starts with an underscore. + if (*location->start == '_') return; + if (pm_constant_id_list_includes(captures, capture)) { pm_parser_err(parser, location->start, location->end, PM_ERR_PATTERN_CAPTURE_DUPLICATE); } else { @@ -14828,6 +14831,10 @@ parse_pattern_hash_implicit_value(pm_parser_t *parser, pm_constant_id_list_t *ca return (pm_node_t *) pm_implicit_node_create(parser, (pm_node_t *) target); } +/** + * Add a node to the list of keys for a hash pattern, and if it is a duplicate + * then add an error to the parser. + */ static void parse_pattern_hash_key(pm_parser_t *parser, pm_static_literals_t *keys, pm_node_t *node) { if (pm_static_literals_add(parser, keys, node) != NULL) { diff --git a/test/prism/errors_test.rb b/test/prism/errors_test.rb index 856e46fb76944a..cfe4ea41bec742 100644 --- a/test/prism/errors_test.rb +++ b/test/prism/errors_test.rb @@ -2075,7 +2075,7 @@ def test_it_with_ordinary_parameter source = "proc { || it }" errors = [["`it` is not allowed when an ordinary parameter is defined", 10..12]] - assert_errors expression(source), source, errors + assert_errors expression(source), source, errors, check_valid_syntax: RUBY_VERSION >= "3.4.0" end def test_regular_expression_with_unknown_regexp_options @@ -2170,47 +2170,56 @@ def test_assignment_to_literal_in_conditionals def test_duplicate_pattern_capture source = <<~RUBY - () => [a, a] - () => [a, *a] - () => {a: a, b: a} - () => {a: a, **a} - () => [a, {a:}] - () => [a, {a: {a: {a: [a]}}}] - () => a => a - () => [A => a, {a: b => a}] + case (); in [a, a]; end + case (); in [a, *a]; end + case (); in {a: a, b: a}; end + case (); in {a: a, **a}; end + case (); in [a, {a:}]; end + case (); in [a, {a: {a: {a: [a]}}}]; end + case (); in a => a; end + case (); in [A => a, {a: b => a}]; end RUBY assert_error_messages source, Array.new(source.lines.length, "duplicated variable name") + refute_error_messages "case (); in [_a, _a]; end" end def test_duplicate_pattern_hash_key - assert_error_messages "() => {a:, a:}", ["duplicated key name", "duplicated variable name"] - assert_error_messages "() => {a:1, a:2}", ["duplicated key name"] + assert_error_messages "case (); in {a:, a:}; end", ["duplicated key name", "duplicated variable name"] + assert_error_messages "case (); in {a:1, a:2}; end", ["duplicated key name"] refute_error_messages "case (); in [{a:1}, {a:2}]; end" end private - def check_syntax(source) - $VERBOSE, previous = nil, $VERBOSE + if RUBY_ENGINE == "ruby" + def check_syntax(source) + $VERBOSE, previous = nil, $VERBOSE - begin - RubyVM::InstructionSequence.compile(source) - ensure - $VERBOSE = previous + begin + RubyVM::InstructionSequence.compile(source) + ensure + $VERBOSE = previous + end end - end - def assert_valid_syntax(source) - check_syntax(source) - end + def assert_valid_syntax(source) + check_syntax(source) + end - def refute_valid_syntax(source) - assert_raise(SyntaxError) { check_syntax(source) } + def refute_valid_syntax(source) + assert_raise(SyntaxError) { check_syntax(source) } + end + else + def assert_valid_syntax(source) + end + + def refute_valid_syntax(source) + end end - def assert_errors(expected, source, errors) - refute_valid_syntax(source) if RUBY_ENGINE == "ruby" + def assert_errors(expected, source, errors, check_valid_syntax: true) + refute_valid_syntax(source) if check_valid_syntax result = Prism.parse(source) node = result.value.statements.body.last @@ -2220,13 +2229,13 @@ def assert_errors(expected, source, errors) end def assert_error_messages(source, errors) - refute_valid_syntax(source) if RUBY_ENGINE == "ruby" + refute_valid_syntax(source) result = Prism.parse(source) assert_equal(errors, result.errors.map(&:message)) end def refute_error_messages(source) - assert_valid_syntax(source) if RUBY_ENGINE == "ruby" + assert_valid_syntax(source) assert Prism.parse_success?(source) end From cc6668c65620a117b7af891beb1bee09d9f460fb Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 1 Apr 2024 14:22:41 -0400 Subject: [PATCH 196/211] [PRISM] Enable passing pattern matching tests --- spec/prism.mspec | 2 -- test/.excludes-prism/TestPatternMatching.rb | 5 +---- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/spec/prism.mspec b/spec/prism.mspec index 1d4d6f3f21adb1..9f5e2b2a5fdaa5 100644 --- a/spec/prism.mspec +++ b/spec/prism.mspec @@ -9,8 +9,6 @@ MSpec.register(:exclude, "Hash literal merges multiple nested '**obj' in Hash li MSpec.register(:exclude, "Hash literal raises a SyntaxError at parse time when Symbol key with invalid bytes") MSpec.register(:exclude, "Hash literal raises a SyntaxError at parse time when Symbol key with invalid bytes and 'key: value' syntax used") MSpec.register(:exclude, "The next statement in a method is invalid and raises a SyntaxError") -MSpec.register(:exclude, "Pattern matching variable pattern does not support using variable name (except _) several times") -MSpec.register(:exclude, "Pattern matching Hash pattern raise SyntaxError when keys duplicate in pattern") MSpec.register(:exclude, "Regexps with encoding modifiers supports /e (EUC encoding) with interpolation") MSpec.register(:exclude, "Regexps with encoding modifiers supports /e (EUC encoding) with interpolation /o") MSpec.register(:exclude, "Regexps with encoding modifiers preserves EUC-JP as /e encoding through interpolation") diff --git a/test/.excludes-prism/TestPatternMatching.rb b/test/.excludes-prism/TestPatternMatching.rb index 40d9d6d99cbde0..cfd0c6bed97058 100644 --- a/test/.excludes-prism/TestPatternMatching.rb +++ b/test/.excludes-prism/TestPatternMatching.rb @@ -1,5 +1,2 @@ -exclude(:test_array_pattern, "duplicated variable error missing") -exclude(:test_find_pattern, "duplicated variable error missing") exclude(:test_hash_pattern, "useless literal warning missing") -exclude(:test_invalid_syntax, "duplicated variable error missing") -exclude(:test_var_pattern, "duplicated variable error missing") +exclude(:test_invalid_syntax, "[a:] is disallowed") From 1e737ec97760fa7dd5436e9e41fefb5f71566344 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 1 Apr 2024 14:36:38 -0400 Subject: [PATCH 197/211] [ruby/prism] Fix up error message for invalid numbered reference alias https://github.com/ruby/prism/commit/74bff9e834 --- prism/config.yml | 1 + prism/prism.c | 2 +- prism/templates/src/diagnostic.c.erb | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/prism/config.yml b/prism/config.yml index 34cd388f485cb0..5d2c2dbd4971bf 100644 --- a/prism/config.yml +++ b/prism/config.yml @@ -1,5 +1,6 @@ errors: - ALIAS_ARGUMENT + - ALIAS_ARGUMENT_NUMBERED_REFERENCE - AMPAMPEQ_MULTI_ASSIGN - ARGUMENT_AFTER_BLOCK - ARGUMENT_AFTER_FORWARDING_ELLIPSES diff --git a/prism/prism.c b/prism/prism.c index 2a288cae934db0..7527e3c55cf50e 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -16300,7 +16300,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b case PM_GLOBAL_VARIABLE_READ_NODE: { if (PM_NODE_TYPE_P(old_name, PM_BACK_REFERENCE_READ_NODE) || PM_NODE_TYPE_P(old_name, PM_NUMBERED_REFERENCE_READ_NODE) || PM_NODE_TYPE_P(old_name, PM_GLOBAL_VARIABLE_READ_NODE)) { if (PM_NODE_TYPE_P(old_name, PM_NUMBERED_REFERENCE_READ_NODE)) { - pm_parser_err_node(parser, old_name, PM_ERR_ALIAS_ARGUMENT); + pm_parser_err_node(parser, old_name, PM_ERR_ALIAS_ARGUMENT_NUMBERED_REFERENCE); } } else { pm_parser_err_node(parser, old_name, PM_ERR_ALIAS_ARGUMENT); diff --git a/prism/templates/src/diagnostic.c.erb b/prism/templates/src/diagnostic.c.erb index 301f98134f592c..a742c62e8e3355 100644 --- a/prism/templates/src/diagnostic.c.erb +++ b/prism/templates/src/diagnostic.c.erb @@ -86,6 +86,7 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { // Errors that should raise syntax errors [PM_ERR_ALIAS_ARGUMENT] = { "invalid argument being passed to `alias`; expected a bare word, symbol, constant, or global variable", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ALIAS_ARGUMENT_NUMBERED_REFERENCE] = { "invalid argument being passed to `alias`; can't make alias for the number variables", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_AMPAMPEQ_MULTI_ASSIGN] = { "unexpected `&&=` in a multiple assignment", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ARGUMENT_AFTER_BLOCK] = { "unexpected argument after a block argument", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ARGUMENT_AFTER_FORWARDING_ELLIPSES] = { "unexpected argument after `...`", PM_ERROR_LEVEL_SYNTAX }, From fee70c1ed71dd0db4ce9a2226dc5f4845e0a0c48 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 1 Apr 2024 14:41:45 -0400 Subject: [PATCH 198/211] [ruby/prism] Add better error messages for invalid block-locals https://github.com/ruby/prism/commit/27ad452436 --- prism/prism.c | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/prism/prism.c b/prism/prism.c index 7527e3c55cf50e..01b920b2d0803e 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -1988,7 +1988,6 @@ pm_block_parameters_node_closing_set(pm_block_parameters_node_t *node, const pm_ */ static pm_block_local_variable_node_t * pm_block_local_variable_node_create(pm_parser_t *parser, const pm_token_t *name) { - assert(name->type == PM_TOKEN_IDENTIFIER || name->type == PM_TOKEN_MISSING); pm_block_local_variable_node_t *node = PM_ALLOC_NODE(parser, pm_block_local_variable_node_t); *node = (pm_block_local_variable_node_t) { @@ -13587,7 +13586,28 @@ parse_block_parameters( if (accept1(parser, PM_TOKEN_SEMICOLON)) { do { - expect1(parser, PM_TOKEN_IDENTIFIER, PM_ERR_BLOCK_PARAM_LOCAL_VARIABLE); + switch (parser->current.type) { + case PM_TOKEN_CONSTANT: + pm_parser_err_current(parser, PM_ERR_ARGUMENT_FORMAL_CONSTANT); + parser_lex(parser); + break; + case PM_TOKEN_INSTANCE_VARIABLE: + pm_parser_err_current(parser, PM_ERR_ARGUMENT_FORMAL_IVAR); + parser_lex(parser); + break; + case PM_TOKEN_GLOBAL_VARIABLE: + pm_parser_err_current(parser, PM_ERR_ARGUMENT_FORMAL_GLOBAL); + parser_lex(parser); + break; + case PM_TOKEN_CLASS_VARIABLE: + pm_parser_err_current(parser, PM_ERR_ARGUMENT_FORMAL_CLASS); + parser_lex(parser); + break; + default: + expect1(parser, PM_TOKEN_IDENTIFIER, PM_ERR_BLOCK_PARAM_LOCAL_VARIABLE); + break; + } + bool repeated = pm_parser_parameter_name_check(parser, &parser->previous); pm_parser_local_add_token(parser, &parser->previous); From a885d597d514693c10b12cbacbd7cc9d8ecf4986 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 1 Apr 2024 14:44:20 -0400 Subject: [PATCH 199/211] [ruby/prism] Match error message for multiple blocks given https://github.com/ruby/prism/commit/6b594d9d42 --- prism/templates/src/diagnostic.c.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prism/templates/src/diagnostic.c.erb b/prism/templates/src/diagnostic.c.erb index a742c62e8e3355..7a73187325467a 100644 --- a/prism/templates/src/diagnostic.c.erb +++ b/prism/templates/src/diagnostic.c.erb @@ -92,7 +92,7 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_ERR_ARGUMENT_AFTER_FORWARDING_ELLIPSES] = { "unexpected argument after `...`", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ARGUMENT_BARE_HASH] = { "unexpected bare hash argument", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ARGUMENT_BLOCK_FORWARDING] = { "both a block argument and a forwarding argument; only one block is allowed", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_ARGUMENT_BLOCK_MULTI] = { "multiple block arguments; only one block is allowed", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARGUMENT_BLOCK_MULTI] = { "both block arg and actual block given; only one block is allowed", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ARGUMENT_FORMAL_CLASS] = { "invalid formal argument; formal argument cannot be a class variable", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ARGUMENT_FORMAL_CONSTANT] = { "invalid formal argument; formal argument cannot be a constant", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_ARGUMENT_FORMAL_GLOBAL] = { "invalid formal argument; formal argument cannot be a global variable", PM_ERROR_LEVEL_SYNTAX }, From 05904c3b729e12eaf19187e00c6be85e7110ec6b Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 1 Apr 2024 14:46:31 -0400 Subject: [PATCH 200/211] [ruby/prism] Match error message for invalid class/module definition https://github.com/ruby/prism/commit/1879a9d22e --- prism/templates/src/diagnostic.c.erb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/prism/templates/src/diagnostic.c.erb b/prism/templates/src/diagnostic.c.erb index 7a73187325467a..81123a8e6cdced 100644 --- a/prism/templates/src/diagnostic.c.erb +++ b/prism/templates/src/diagnostic.c.erb @@ -127,7 +127,7 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_ERR_CASE_MATCH_MISSING_PREDICATE] = { "expected a predicate for a case matching statement", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_CASE_MISSING_CONDITIONS] = { "expected a `when` or `in` clause after `case`", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_CASE_TERM] = { "expected an `end` to close the `case` statement", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_CLASS_IN_METHOD] = { "unexpected class definition in a method definition", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_CLASS_IN_METHOD] = { "unexpected class definition in method body", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_CLASS_NAME] = { "expected a constant name after `class`", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_CLASS_SUPERCLASS] = { "expected a superclass after `<`", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_CLASS_TERM] = { "expected an `end` to close the `class` statement", PM_ERROR_LEVEL_SYNTAX }, @@ -232,7 +232,7 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_ERR_LIST_W_UPPER_TERM] = { "expected a closing delimiter for the `%W` list", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_MALLOC_FAILED] = { "failed to allocate memory", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_MIXED_ENCODING] = { "UTF-8 mixed within %s source", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_MODULE_IN_METHOD] = { "unexpected module definition in a method definition", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_MODULE_IN_METHOD] = { "unexpected module definition in method body", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_MODULE_NAME] = { "expected a constant name after `module`", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_MODULE_TERM] = { "expected an `end` to close the `module` statement", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_MULTI_ASSIGN_MULTI_SPLATS] = { "multiple splats in multiple assignment", PM_ERROR_LEVEL_SYNTAX }, From 67bd5b33f906e3eec50195b8180c3f474d002bd1 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 1 Apr 2024 14:48:23 -0400 Subject: [PATCH 201/211] [ruby/prism] Match error message for invalid class/module name https://github.com/ruby/prism/commit/f00ae59070 --- prism/templates/src/diagnostic.c.erb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/prism/templates/src/diagnostic.c.erb b/prism/templates/src/diagnostic.c.erb index 81123a8e6cdced..15b7513154a53a 100644 --- a/prism/templates/src/diagnostic.c.erb +++ b/prism/templates/src/diagnostic.c.erb @@ -128,7 +128,7 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_ERR_CASE_MISSING_CONDITIONS] = { "expected a `when` or `in` clause after `case`", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_CASE_TERM] = { "expected an `end` to close the `case` statement", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_CLASS_IN_METHOD] = { "unexpected class definition in method body", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_CLASS_NAME] = { "expected a constant name after `class`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_CLASS_NAME] = { "unexpected constant path after `class`; class/module name must be CONSTANT", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_CLASS_SUPERCLASS] = { "expected a superclass after `<`", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_CLASS_TERM] = { "expected an `end` to close the `class` statement", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_CLASS_UNEXPECTED_END] = { "unexpected `end`, expecting ';' or '\\n'", PM_ERROR_LEVEL_SYNTAX }, @@ -233,7 +233,7 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_ERR_MALLOC_FAILED] = { "failed to allocate memory", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_MIXED_ENCODING] = { "UTF-8 mixed within %s source", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_MODULE_IN_METHOD] = { "unexpected module definition in method body", PM_ERROR_LEVEL_SYNTAX }, - [PM_ERR_MODULE_NAME] = { "expected a constant name after `module`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_MODULE_NAME] = { "unexpected constant path after `module`; class/module name must be CONSTANT", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_MODULE_TERM] = { "expected an `end` to close the `module` statement", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_MULTI_ASSIGN_MULTI_SPLATS] = { "multiple splats in multiple assignment", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_MULTI_ASSIGN_UNEXPECTED_REST] = { "unexpected '%.*s' resulting in multiple splats in multiple assignment", PM_ERROR_LEVEL_SYNTAX }, From d898f00fe16f598474aeea06d6000ac10740236a Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 1 Apr 2024 14:59:41 -0400 Subject: [PATCH 202/211] [ruby/prism] Match error messages for invalid instance/class variables https://github.com/ruby/prism/commit/82fd0599ed --- prism/config.yml | 2 ++ prism/prism.c | 5 ++++- prism/templates/src/diagnostic.c.erb | 2 ++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/prism/config.yml b/prism/config.yml index 5d2c2dbd4971bf..8b9bb911e577fb 100644 --- a/prism/config.yml +++ b/prism/config.yml @@ -47,6 +47,7 @@ errors: - CLASS_SUPERCLASS - CLASS_TERM - CLASS_UNEXPECTED_END + - CLASS_VARIABLE_BARE - CONDITIONAL_ELSIF_PREDICATE - CONDITIONAL_IF_PREDICATE - CONDITIONAL_PREDICATE_TERM @@ -119,6 +120,7 @@ errors: - INCOMPLETE_VARIABLE_CLASS_3_3_0 - INCOMPLETE_VARIABLE_INSTANCE - INCOMPLETE_VARIABLE_INSTANCE_3_3_0 + - INSTANCE_VARIABLE_BARE - INVALID_CHARACTER - INVALID_ENCODING_MAGIC_COMMENT - INVALID_FLOAT_EXPONENT diff --git a/prism/prism.c b/prism/prism.c index 01b920b2d0803e..abd1914f4ee2f5 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -9022,7 +9022,7 @@ lex_at_variable(pm_parser_t *parser) { while (parser->current.end < parser->end && (width = char_is_identifier(parser, parser->current.end)) > 0) { parser->current.end += width; } - } else { + } else if (parser->current.end < parser->end && pm_char_is_decimal_digit(*parser->current.end)) { pm_diagnostic_id_t diag_id = (type == PM_TOKEN_CLASS_VARIABLE) ? PM_ERR_INCOMPLETE_VARIABLE_CLASS : PM_ERR_INCOMPLETE_VARIABLE_INSTANCE; if (parser->version == PM_OPTIONS_VERSION_CRUBY_3_3_0) { diag_id = (type == PM_TOKEN_CLASS_VARIABLE) ? PM_ERR_INCOMPLETE_VARIABLE_CLASS_3_3_0 : PM_ERR_INCOMPLETE_VARIABLE_INSTANCE_3_3_0; @@ -9030,6 +9030,9 @@ lex_at_variable(pm_parser_t *parser) { size_t width = parser->encoding->char_width(parser->current.end, parser->end - parser->current.end); PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, diag_id, (int) ((parser->current.end + width) - parser->current.start), (const char *) parser->current.start); + } else { + pm_diagnostic_id_t diag_id = (type == PM_TOKEN_CLASS_VARIABLE) ? PM_ERR_CLASS_VARIABLE_BARE : PM_ERR_INSTANCE_VARIABLE_BARE; + pm_parser_err_token(parser, &parser->current, diag_id); } // If we're lexing an embedded variable, then we need to pop back into the diff --git a/prism/templates/src/diagnostic.c.erb b/prism/templates/src/diagnostic.c.erb index 15b7513154a53a..b91bd973d5f31c 100644 --- a/prism/templates/src/diagnostic.c.erb +++ b/prism/templates/src/diagnostic.c.erb @@ -132,6 +132,7 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_ERR_CLASS_SUPERCLASS] = { "expected a superclass after `<`", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_CLASS_TERM] = { "expected an `end` to close the `class` statement", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_CLASS_UNEXPECTED_END] = { "unexpected `end`, expecting ';' or '\\n'", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_CLASS_VARIABLE_BARE] = { "'@@' without identifiers is not allowed as a class variable name", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_CONDITIONAL_ELSIF_PREDICATE] = { "expected a predicate expression for the `elsif` statement", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_CONDITIONAL_IF_PREDICATE] = { "expected a predicate expression for the `if` statement", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_CONDITIONAL_PREDICATE_TERM] = { "expected `then` or `;` or '\\n'", PM_ERROR_LEVEL_SYNTAX }, @@ -203,6 +204,7 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_ERR_INCOMPLETE_VARIABLE_CLASS] = { "'%.*s' is not allowed as a class variable name", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_INCOMPLETE_VARIABLE_INSTANCE_3_3_0] = { "`%.*s' is not allowed as an instance variable name", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_INCOMPLETE_VARIABLE_INSTANCE] = { "'%.*s' is not allowed as an instance variable name", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_INSTANCE_VARIABLE_BARE] = { "'@' without identifiers is not allowed as an instance variable name", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_INVALID_FLOAT_EXPONENT] = { "invalid exponent", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_INVALID_NUMBER_BINARY] = { "invalid binary number", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_INVALID_NUMBER_DECIMAL] = { "invalid decimal number", PM_ERROR_LEVEL_SYNTAX }, From d6c1cc5532e5d31e5f5abe171fd759e3f4a099f7 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 1 Apr 2024 15:03:41 -0400 Subject: [PATCH 203/211] [ruby/prism] Fix up error messages for empty global variable https://github.com/ruby/prism/commit/fa7559d40b --- prism/config.yml | 1 + prism/prism.c | 10 +++++----- prism/templates/src/diagnostic.c.erb | 1 + 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/prism/config.yml b/prism/config.yml index 8b9bb911e577fb..c813b02d9f15b2 100644 --- a/prism/config.yml +++ b/prism/config.yml @@ -109,6 +109,7 @@ errors: - FOR_IN - FOR_INDEX - FOR_TERM + - GLOBAL_VARIABLE_BARE - HASH_EXPRESSION_AFTER_LABEL - HASH_KEY - HASH_ROCKET diff --git a/prism/prism.c b/prism/prism.c index abd1914f4ee2f5..6631ef71ae75f7 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -7994,8 +7994,7 @@ lex_numeric(pm_parser_t *parser) { static pm_token_type_t lex_global_variable(pm_parser_t *parser) { if (parser->current.end >= parser->end) { - pm_diagnostic_id_t diag_id = parser->version == PM_OPTIONS_VERSION_CRUBY_3_3_0 ? PM_ERR_INVALID_VARIABLE_GLOBAL_3_3_0 : PM_ERR_INVALID_VARIABLE_GLOBAL; - PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, parser->current, diag_id); + pm_parser_err_token(parser, &parser->current, PM_ERR_GLOBAL_VARIABLE_BARE); return PM_TOKEN_GLOBAL_VARIABLE; } @@ -8066,10 +8065,11 @@ lex_global_variable(pm_parser_t *parser) { parser->current.end += width; } while (parser->current.end < parser->end && (width = char_is_identifier(parser, parser->current.end)) > 0); } else { - // If we get here, then we have a $ followed by something that isn't - // recognized as a global variable. + // If we get here, then we have a $ followed by something that + // isn't recognized as a global variable. pm_diagnostic_id_t diag_id = parser->version == PM_OPTIONS_VERSION_CRUBY_3_3_0 ? PM_ERR_INVALID_VARIABLE_GLOBAL_3_3_0 : PM_ERR_INVALID_VARIABLE_GLOBAL; - PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, parser->current, diag_id); + size_t width = parser->encoding->char_width(parser->current.end, parser->end - parser->current.end); + PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, diag_id, (int) ((parser->current.end + width) - parser->current.start), (const char *) parser->current.start); } return PM_TOKEN_GLOBAL_VARIABLE; diff --git a/prism/templates/src/diagnostic.c.erb b/prism/templates/src/diagnostic.c.erb index b91bd973d5f31c..3d5073147a9ffb 100644 --- a/prism/templates/src/diagnostic.c.erb +++ b/prism/templates/src/diagnostic.c.erb @@ -193,6 +193,7 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_ERR_FOR_INDEX] = { "expected an index after `for`", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_FOR_IN] = { "expected an `in` after the index in a `for` statement", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_FOR_TERM] = { "expected an `end` to close the `for` loop", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_GLOBAL_VARIABLE_BARE] = { "'$' without identifiers is not allowed as a global variable name", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_HASH_EXPRESSION_AFTER_LABEL] = { "expected an expression after the label in a hash", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_HASH_KEY] = { "unexpected %s, expecting '}' or a key in the hash literal", PM_ERROR_LEVEL_SYNTAX }, [PM_ERR_HASH_ROCKET] = { "expected a `=>` between the hash key and value", PM_ERROR_LEVEL_SYNTAX }, From b7597dac932c6fa3add9146c82af7a47c9059dfb Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 1 Apr 2024 15:10:59 -0400 Subject: [PATCH 204/211] [ruby/prism] Match more error messages https://github.com/ruby/prism/commit/0cc3a9d63a --- test/prism/errors_test.rb | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/test/prism/errors_test.rb b/test/prism/errors_test.rb index cfe4ea41bec742..dbc4a8924d8565 100644 --- a/test/prism/errors_test.rb +++ b/test/prism/errors_test.rb @@ -26,7 +26,7 @@ def test_module_name_recoverable ) assert_errors expected, "module Parent module end", [ - ["expected a constant name after `module`", 14..20], + ["unexpected constant path after `module`; class/module name must be CONSTANT", 14..20], ["unexpected 'end', assuming it is closing the parent module definition", 21..24] ] end @@ -177,7 +177,7 @@ def test_unterminated_empty_string def test_incomplete_instance_var_string assert_errors expression('%@#@@#'), '%@#@@#', [ - ["'@#' is not allowed as an instance variable name", 4..5], + ["'@' without identifiers is not allowed as an instance variable name", 4..5], ["unexpected instance variable, expecting end-of-input", 4..5] ] end @@ -325,7 +325,7 @@ def test_aliasing_non_global_variable_with_global_variable def test_aliasing_global_variable_with_global_number_variable assert_errors expression("alias $a $1"), "alias $a $1", [ - ["invalid argument being passed to `alias`; expected a bare word, symbol, constant, or global variable", 9..11] + ["invalid argument being passed to `alias`; can't make alias for the number variables", 9..11] ] end @@ -452,7 +452,7 @@ def test_module_definition_in_method_body ) assert_errors expected, "def foo;module A;end;end", [ - ["unexpected module definition in a method definition", 8..14] + ["unexpected module definition in method body", 8..14] ] end @@ -490,7 +490,7 @@ def test_module_definition_in_method_body_within_block Location() ) - assert_errors expected, <<~RUBY, [["unexpected module definition in a method definition", 21..27]] + assert_errors expected, <<~RUBY, [["unexpected module definition in method body", 21..27]] def foo bar do module Foo;end @@ -505,7 +505,7 @@ def foo(bar = module A;end);end def foo;rescue;module A;end;end def foo;ensure;module A;end;end RUBY - message = "unexpected module definition in a method definition" + message = "unexpected module definition in method body" assert_errors expression(source), source, [ [message, 14..20], [message, 47..53], @@ -541,7 +541,7 @@ def test_class_definition_in_method_body ) assert_errors expected, "def foo;class A;end;end", [ - ["unexpected class definition in a method definition", 8..13] + ["unexpected class definition in method body", 8..13] ] end @@ -551,7 +551,7 @@ def foo(bar = class A;end);end def foo;rescue;class A;end;end def foo;ensure;class A;end;end RUBY - message = "unexpected class definition in a method definition" + message = "unexpected class definition in method body" assert_errors expression(source), source, [ [message, 14..19], [message, 46..51], @@ -1254,7 +1254,7 @@ def test_invalid_operator_write_dot def test_unterminated_global_variable assert_errors expression("$"), "$", [ - ["'$' is not allowed as a global variable name", 0..1] + ["'$' without identifiers is not allowed as a global variable name", 0..1] ] end @@ -1438,7 +1438,7 @@ def test_parameter_name_ending_with_bang_or_question_mark def test_class_name source = "class 0.X end" assert_errors expression(source), source, [ - ["expected a constant name after `class`", 6..9], + ["unexpected constant path after `class`; class/module name must be CONSTANT", 6..9], ] end @@ -2060,7 +2060,7 @@ def test_non_assoc_equality def test_block_arg_and_block source = 'foo(&1) { }' assert_errors expression(source), source, [ - ['multiple block arguments; only one block is allowed', 8..11] + ["both block arg and actual block given; only one block is allowed", 8..11] ] end From b25282e61844334c70def2d678c19c6105646ab3 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 1 Apr 2024 15:28:31 -0400 Subject: [PATCH 205/211] [ruby/prism] Replace . with decimal point for strtod https://github.com/ruby/prism/commit/578a4f983e --- prism/prism.c | 9 +++++++++ prism/prism.h | 1 + 2 files changed, 10 insertions(+) diff --git a/prism/prism.c b/prism/prism.c index 6631ef71ae75f7..d03a11583f7a4d 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -3424,6 +3424,15 @@ pm_double_parse(pm_parser_t *parser, const pm_token_t *token) { char *buffer = xmalloc(sizeof(char) * (length + 1)); memcpy((void *) buffer, token->start, length); + // Next, determine if we need to replace the decimal point because of + // locale-specific options, and then normalize them if we have to. + char decimal_point = *localeconv()->decimal_point; + if (decimal_point != '.') { + for (size_t index = 0; index < length; index++) { + if (buffer[index] == '.') buffer[index] = decimal_point; + } + } + // Next, handle underscores by removing them from the buffer. for (size_t index = 0; index < length; index++) { if (buffer[index] == '_') { diff --git a/prism/prism.h b/prism/prism.h index 70e1e9df4960a7..59067c3021a70a 100644 --- a/prism/prism.h +++ b/prism/prism.h @@ -26,6 +26,7 @@ #include #include +#include #include #include #include From 8066e3ea6e9462f510e5d0da887be94b18cce50b Mon Sep 17 00:00:00 2001 From: yui-knk Date: Mon, 1 Apr 2024 15:46:58 +0900 Subject: [PATCH 206/211] Remove VALUE from `struct rb_strterm_struct` In the past, it was imemo. However a075c55 changed it. Therefore no need to use `VALUE` for the first field. --- internal/parse.h | 4 +--- parse.y | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/internal/parse.h b/internal/parse.h index fc78836e5c48e4..d7fc6ddad4bfdb 100644 --- a/internal/parse.h +++ b/internal/parse.h @@ -18,8 +18,6 @@ struct rb_iseq_struct; /* in vm_core.h */ -#define STRTERM_HEREDOC IMEMO_FL_USER0 - /* structs for managing terminator of string literal and heredocment */ typedef struct rb_strterm_literal_struct { long nest; @@ -40,7 +38,7 @@ typedef struct rb_strterm_heredoc_struct { #define HERETERM_LENGTH_MAX UINT_MAX typedef struct rb_strterm_struct { - VALUE flags; + bool heredoc; union { rb_strterm_literal_t literal; rb_strterm_heredoc_t heredoc; diff --git a/parse.y b/parse.y index 55619273b8e317..ddcc2746b9b0a8 100644 --- a/parse.y +++ b/parse.y @@ -7923,7 +7923,7 @@ parser_str_new(struct parser_params *p, const char *ptr, long len, rb_encoding * static int strterm_is_heredoc(rb_strterm_t *strterm) { - return strterm->flags & STRTERM_HEREDOC; + return strterm->heredoc; } static rb_strterm_t * @@ -7940,7 +7940,7 @@ static rb_strterm_t * new_heredoc(struct parser_params *p) { rb_strterm_t *strterm = ZALLOC(rb_strterm_t); - strterm->flags |= STRTERM_HEREDOC; + strterm->heredoc = true; return strterm; } From 799e854897856e431c03a5122252358e2c57aff2 Mon Sep 17 00:00:00 2001 From: yui-knk Date: Sun, 24 Mar 2024 08:39:27 +0900 Subject: [PATCH 207/211] [Feature #20331] Simplify parser warnings for hash keys duplication and when clause duplication This commit simplifies warnings for hash keys duplication and when clause duplication, based on the discussion of https://bugs.ruby-lang.org/issues/20331. Warnings are reported only when strings are same to ohters. --- ext/ripper/ripper_init.c.tmpl | 6 +- internal/parse.h | 1 + parse.y | 183 +++++++++++++++------------------- ruby_parser.c | 29 ------ rubyparser.h | 14 ++- test/ruby/test_literal.rb | 3 - test/ruby/test_syntax.rb | 4 +- universal_parser.c | 7 +- 8 files changed, 96 insertions(+), 151 deletions(-) diff --git a/ext/ripper/ripper_init.c.tmpl b/ext/ripper/ripper_init.c.tmpl index 894cbc0b447b51..bc1b2128f771fb 100644 --- a/ext/ripper/ripper_init.c.tmpl +++ b/ext/ripper/ripper_init.c.tmpl @@ -3,13 +3,13 @@ #include "ruby/encoding.h" #include "internal.h" #include "internal/imemo.h" /* needed by ruby_parser.h */ +#include "rubyparser.h" +#define YYSTYPE_IS_DECLARED +#include "parse.h" #include "internal/parse.h" #include "internal/ruby_parser.h" #include "node.h" -#include "rubyparser.h" #include "eventids1.h" -#define YYSTYPE_IS_DECLARED -#include "parse.h" #include "eventids2.h" #include "ripper_init.h" diff --git a/internal/parse.h b/internal/parse.h index d7fc6ddad4bfdb..f4f2f0c0f521ca 100644 --- a/internal/parse.h +++ b/internal/parse.h @@ -65,6 +65,7 @@ int rb_ruby_parser_end_seen_p(rb_parser_t *p); int rb_ruby_parser_set_yydebug(rb_parser_t *p, int flag); rb_parser_string_t *rb_str_to_parser_string(rb_parser_t *p, VALUE str); +void rb_parser_check_literal_when(struct parser_params *p, NODE *arg, const YYLTYPE *loc); void rb_parser_warn_duplicate_keys(struct parser_params *p, NODE *hash); int rb_parser_dvar_defined_ref(struct parser_params*, ID, ID**); ID rb_parser_internal_id(struct parser_params*); diff --git a/parse.y b/parse.y index ddcc2746b9b0a8..5784cc4b20dfd0 100644 --- a/parse.y +++ b/parse.y @@ -19,8 +19,6 @@ #define YYDEBUG 1 #define YYERROR_VERBOSE 1 #define YYSTACK_USE_ALLOCA 0 -#define YYLTYPE rb_code_location_t -#define YYLTYPE_IS_DECLARED 1 /* For Ripper */ #ifdef RUBY_EXTCONF_H @@ -73,6 +71,22 @@ #include "ruby/ractor.h" #include "symbol.h" +#ifndef RIPPER +static VALUE +syntax_error_new(void) +{ + return rb_class_new_instance(0, 0, rb_eSyntaxError); +} +#endif + +static NODE *reg_named_capture_assign(struct parser_params* p, VALUE regexp, const YYLTYPE *loc); + +#define compile_callback rb_suppress_tracing +VALUE rb_io_gets_internal(VALUE io); + +VALUE rb_node_case_when_optimizable_literal(const NODE *const node); +#endif /* !UNIVERSAL_PARSER */ + #ifndef RIPPER static int rb_parser_string_hash_cmp(rb_parser_string_t *str1, rb_parser_string_t *str2); @@ -117,15 +131,8 @@ rb_parser_regx_hash_cmp(rb_node_regx_t *n1, rb_node_regx_t *n2) rb_parser_string_hash_cmp(n1->string, n2->string)); } -static int -node_integer_line_cmp(const NODE *node_i, const NODE *line) -{ - VALUE num = rb_node_integer_literal_val(node_i); - - return !(FIXNUM_P(num) && line->nd_loc.beg_pos.lineno == FIX2INT(num)); -} - static st_index_t rb_parser_str_hash(rb_parser_string_t *str); +static st_index_t rb_char_p_hash(const char *c); static int literal_cmp(st_data_t val, st_data_t lit) @@ -137,22 +144,6 @@ literal_cmp(st_data_t val, st_data_t lit) enum node_type type_val = nd_type(node_val); enum node_type type_lit = nd_type(node_lit); - /* Special case for Integer and __LINE__ */ - if (type_val == NODE_INTEGER && type_lit == NODE_LINE) { - return node_integer_line_cmp(node_val, node_lit); - } - if (type_lit == NODE_INTEGER && type_val == NODE_LINE) { - return node_integer_line_cmp(node_lit, node_val); - } - - /* Special case for String and __FILE__ */ - if (type_val == NODE_STR && type_lit == NODE_FILE) { - return rb_parser_string_hash_cmp(RNODE_STR(node_val)->string, RNODE_FILE(node_lit)->path); - } - if (type_lit == NODE_STR && type_val == NODE_FILE) { - return rb_parser_string_hash_cmp(RNODE_STR(node_lit)->string, RNODE_FILE(node_val)->path); - } - if (type_val != type_lit) { return -1; } @@ -179,7 +170,11 @@ literal_cmp(st_data_t val, st_data_t lit) case NODE_ENCODING: return RNODE_ENCODING(node_val)->enc != RNODE_ENCODING(node_lit)->enc; default: +#ifdef UNIVERSAL_PARSER + abort(); +#else rb_bug("unexpected node: %s, %s", ruby_node_name(type_val), ruby_node_name(type_lit)); +#endif } } @@ -187,23 +182,17 @@ static st_index_t literal_hash(st_data_t a) { NODE *node = (NODE *)a; - VALUE val; enum node_type type = nd_type(node); switch (type) { case NODE_INTEGER: - val = rb_node_integer_literal_val(node); - if (!FIXNUM_P(val)) val = rb_big_hash(val); - return FIX2LONG(val); + return rb_char_p_hash(RNODE_INTEGER(node)->val); case NODE_FLOAT: - val = rb_node_float_literal_val(node); - return rb_dbl_long_hash(RFLOAT_VALUE(val)); + return rb_char_p_hash(RNODE_FLOAT(node)->val); case NODE_RATIONAL: - val = rb_node_rational_literal_val(node); - return rb_rational_hash(val); + return rb_char_p_hash(RNODE_RATIONAL(node)->val); case NODE_IMAGINARY: - val = rb_node_imaginary_literal_val(node); - return rb_complex_hash(val); + return rb_char_p_hash(RNODE_IMAGINARY(node)->val); case NODE_STR: return rb_parser_str_hash(RNODE_STR(node)->string); case NODE_SYM: @@ -211,32 +200,20 @@ literal_hash(st_data_t a) case NODE_REGX: return rb_parser_str_hash(RNODE_REGX(node)->string); case NODE_LINE: - /* Same with NODE_INTEGER FIXNUM case */ return (st_index_t)node->nd_loc.beg_pos.lineno; case NODE_FILE: - /* Same with NODE_STR */ return rb_parser_str_hash(RNODE_FILE(node)->path); case NODE_ENCODING: return (st_index_t)RNODE_ENCODING(node)->enc; default: +#ifdef UNIVERSAL_PARSER + abort(); +#else rb_bug("unexpected node: %s", ruby_node_name(type)); +#endif } } - -static VALUE -syntax_error_new(void) -{ - return rb_class_new_instance(0, 0, rb_eSyntaxError); -} -#endif - -static NODE *reg_named_capture_assign(struct parser_params* p, VALUE regexp, const YYLTYPE *loc); - -#define compile_callback rb_suppress_tracing -VALUE rb_io_gets_internal(VALUE io); - -VALUE rb_node_case_when_optimizable_literal(const NODE *const node); -#endif /* !UNIVERSAL_PARSER */ +#endif /* !RIPPER */ static inline int parse_isascii(int c) @@ -567,7 +544,7 @@ struct parser_params { VALUE ruby_sourcefile_string; rb_encoding *enc; token_info *token_info; - VALUE case_labels; + st_table *case_labels; rb_node_exits_t *exits; VALUE debug_buffer; @@ -1532,8 +1509,6 @@ int reg_fragment_check(struct parser_params*, rb_parser_string_t*, int); static int literal_concat0(struct parser_params *p, rb_parser_string_t *head, rb_parser_string_t *tail); static NODE *heredoc_dedent(struct parser_params*,NODE*); -static void check_literal_when(struct parser_params *p, NODE *args, const YYLTYPE *loc); - #ifdef RIPPER static VALUE var_field(struct parser_params *p, VALUE a); #define get_value(idx) (rb_ary_entry(p->s_value_stack, idx)) @@ -1615,6 +1590,9 @@ static void numparam_pop(struct parser_params *p, NODE *prev_inner); #define RE_OPTION_MASK 0xff #define RE_OPTION_ARG_ENCODING_NONE 32 +#define CHECK_LITERAL_WHEN (st_table *)1 +#define CASE_LABELS_ENABLED_P(case_labels) (case_labels && case_labels != CHECK_LITERAL_WHEN) + #define yytnamerr(yyres, yystr) (YYSIZE_T)rb_yytnamerr(p, yyres, yystr) size_t rb_yytnamerr(struct parser_params *p, char *yyres, const char *yystr); @@ -2079,7 +2057,6 @@ get_nd_args(struct parser_params *p, NODE *node) } #ifndef RIPPER -#ifndef UNIVERSAL_PARSER static st_index_t djb2(const uint8_t *str, size_t len) { @@ -2098,7 +2075,6 @@ parser_memhash(const void *ptr, long len) return djb2(ptr, len); } #endif -#endif #define PARSER_STRING_PTR(str) (str->ptr) #define PARSER_STRING_LEN(str) (str->len) @@ -2163,13 +2139,17 @@ rb_parser_string_free(rb_parser_t *p, rb_parser_string_t *str) } #ifndef RIPPER -#ifndef UNIVERSAL_PARSER static st_index_t rb_parser_str_hash(rb_parser_string_t *str) { return parser_memhash((const void *)PARSER_STRING_PTR(str), PARSER_STRING_LEN(str)); } -#endif + +static st_index_t +rb_char_p_hash(const char *c) +{ + return parser_memhash((const void *)c, strlen(c)); +} #endif static size_t @@ -2567,7 +2547,6 @@ rb_parser_str_resize(struct parser_params *p, rb_parser_string_t *str, long len) } #ifndef RIPPER -#ifndef UNIVERSAL_PARSER # define PARSER_ENC_STRING_GETMEM(str, ptrvar, lenvar, encvar) \ ((ptrvar) = str->ptr, \ (lenvar) = str->len, \ @@ -2587,7 +2566,6 @@ rb_parser_string_hash_cmp(rb_parser_string_t *str1, rb_parser_string_t *str2) enc1 != enc2 || memcmp(ptr1, ptr2, len1) != 0); } -#endif static void rb_parser_ary_extend(rb_parser_t *p, rb_parser_ary_t *ary, long len) @@ -2694,6 +2672,10 @@ rb_parser_tokens_free(rb_parser_t *p, rb_parser_ary_t *tokens) rb_parser_printf(p, "$%c", (int)RNODE_BACK_REF($$)->nd_nth); } tBACK_REF +%destructor { + if (CASE_LABELS_ENABLED_P($$)) st_free_table($$); +} + %lex-param {struct parser_params *p} %parse-param {struct parser_params *p} %initial-action @@ -2707,7 +2689,6 @@ rb_parser_tokens_free(rb_parser_t *p, rb_parser_ary_t *tokens) %after-pop-stack after_pop_stack %union { - VALUE val; NODE *node; rb_node_fcall_t *node_fcall; rb_node_args_t *node_args; @@ -2721,6 +2702,7 @@ rb_parser_tokens_free(rb_parser_t *p, rb_parser_ary_t *tokens) ID id; int num; st_table *tbl; + st_table *labels; const struct vtable *vars; struct rb_strterm_struct *strterm; struct lex_context ctxt; @@ -4536,28 +4518,28 @@ primary : literal } | k_case expr_value terms? { - $$ = p->case_labels; - p->case_labels = Qnil; - } + $$ = p->case_labels; + p->case_labels = CHECK_LITERAL_WHEN; + } case_body k_end { - if (RTEST(p->case_labels)) rb_hash_clear(p->case_labels); - p->case_labels = $4; + if (CASE_LABELS_ENABLED_P(p->case_labels)) st_free_table(p->case_labels); + p->case_labels = $4; $$ = NEW_CASE($2, $5, &@$); fixpos($$, $2); /*% ripper: case!($:2, $:5) %*/ } | k_case terms? { - $$ = p->case_labels; + $$ = p->case_labels; p->case_labels = 0; - } + } case_body k_end { - if (RTEST(p->case_labels)) rb_hash_clear(p->case_labels); - p->case_labels = $3; + if (p->case_labels) st_free_table(p->case_labels); + p->case_labels = $3; $$ = NEW_CASE2($4, &@$); /*% ripper: case!(Qnil, $:4) %*/ } @@ -5415,7 +5397,7 @@ do_body : { case_args : arg_value { - check_literal_when(p, $1, &@1); + rb_parser_check_literal_when(p, $1, &@1); $$ = NEW_LIST($1, &@$); /*% ripper: args_add!(args_new!, $:1) %*/ } @@ -5426,7 +5408,7 @@ case_args : arg_value } | case_args ',' arg_value { - check_literal_when(p, $3, &@3); + rb_parser_check_literal_when(p, $3, &@3); $$ = last_arg_append(p, $1, $3, &@$); /*% ripper: args_add!($:1, $:3) %*/ } @@ -11568,7 +11550,7 @@ yylex(YYSTYPE *lval, YYLTYPE *yylloc, struct parser_params *p) enum yytokentype t; p->lval = lval; - lval->val = Qundef; + lval->node = 0; p->yylloc = yylloc; t = parser_yylex(p); @@ -13487,36 +13469,35 @@ new_xstring(struct parser_params *p, NODE *node, const YYLTYPE *loc) } #ifndef RIPPER -VALUE -rb_parser_node_case_when_optimizable_literal(struct parser_params *p, const NODE *const node) -{ - return rb_node_case_when_optimizable_literal(node); -} -#endif +static const +struct st_hash_type literal_type = { + literal_cmp, + literal_hash, +}; -static void -check_literal_when(struct parser_params *p, NODE *arg, const YYLTYPE *loc) -{ - VALUE lit; +static int nd_type_st_key_enable_p(NODE *node); +void +rb_parser_check_literal_when(struct parser_params *p, NODE *arg, const YYLTYPE *loc) +{ + /* See https://bugs.ruby-lang.org/issues/20331 for discussion about what is warned. */ if (!arg || !p->case_labels) return; + if (!nd_type_st_key_enable_p(arg)) return; - lit = rb_parser_node_case_when_optimizable_literal(p, arg); - if (UNDEF_P(lit)) return; - - if (NIL_P(p->case_labels)) { - p->case_labels = rb_obj_hide(rb_hash_new()); + if (p->case_labels == CHECK_LITERAL_WHEN) { + p->case_labels = st_init_table(&literal_type); } else { - VALUE line = rb_hash_lookup(p->case_labels, lit); - if (!NIL_P(line)) { + st_data_t line; + if (st_lookup(p->case_labels, (st_data_t)arg, &line)) { rb_warning1("duplicated 'when' clause with line %d is ignored", - WARN_IVAL(line)); + WARN_IVAL(INT2NUM((int)line))); return; } } - rb_hash_aset(p->case_labels, lit, INT2NUM(p->ruby_sourceline)); + st_insert(p->case_labels, (st_data_t)arg, (st_data_t)p->ruby_sourceline); } +#endif #ifdef RIPPER static int @@ -15229,14 +15210,7 @@ nd_value(struct parser_params *p, NODE *node) void rb_parser_warn_duplicate_keys(struct parser_params *p, NODE *hash) { -#ifndef UNIVERSAL_PARSER - static const -#endif - struct st_hash_type literal_type = { - literal_cmp, - literal_hash, - }; - + /* See https://bugs.ruby-lang.org/issues/20331 for discussion about what is warned. */ st_table *literal_keys = st_init_table_with_size(&literal_type, RNODE_LIST(hash)->as.nd_alen / 2); while (hash && RNODE_LIST(hash)->nd_next) { NODE *head = RNODE_LIST(hash)->nd_head; @@ -16159,7 +16133,6 @@ rb_ruby_parser_mark(void *ptr) rb_gc_mark(p->lex.input); rb_gc_mark(p->ruby_sourcefile_string); rb_gc_mark((VALUE)p->ast); - rb_gc_mark(p->case_labels); rb_gc_mark(p->delayed.token); #ifndef RIPPER rb_gc_mark(p->debug_lines); @@ -16213,6 +16186,10 @@ rb_ruby_parser_free(void *ptr) st_free_table(p->pvtbl); } + if (CASE_LABELS_ENABLED_P(p->case_labels)) { + st_free_table(p->case_labels); + } + xfree(ptr); } diff --git a/ruby_parser.c b/ruby_parser.c index 83539612e882e4..90ee4504a241e6 100644 --- a/ruby_parser.c +++ b/ruby_parser.c @@ -93,32 +93,6 @@ dvar_defined(ID id, const void *p) return rb_dvar_defined(id, (const rb_iseq_t *)p); } -static bool -hash_literal_key_p(VALUE k) -{ - switch (OBJ_BUILTIN_TYPE(k)) { - case T_NODE: - return false; - default: - return true; - } -} - -static int -literal_cmp(VALUE val, VALUE lit) -{ - if (val == lit) return 0; - if (!hash_literal_key_p(val) || !hash_literal_key_p(lit)) return -1; - return rb_iseq_cdhash_cmp(val, lit); -} - -static st_index_t -literal_hash(VALUE a) -{ - if (!hash_literal_key_p(a)) return (st_index_t)a; - return rb_iseq_cdhash_hash(a); -} - static int is_usascii_enc(void *enc) { @@ -611,9 +585,6 @@ static const rb_parser_config_t rb_global_parser_config = { .local_defined = local_defined, .dvar_defined = dvar_defined, - .literal_cmp = literal_cmp, - .literal_hash = literal_hash, - .syntax_error_append = syntax_error_append, .raise = rb_raise, .syntax_error_new = syntax_error_new, diff --git a/rubyparser.h b/rubyparser.h index b0bef9d56feb6c..28e675c39fa8c0 100644 --- a/rubyparser.h +++ b/rubyparser.h @@ -200,6 +200,8 @@ typedef struct rb_code_location_struct { rb_code_position_t beg_pos; rb_code_position_t end_pos; } rb_code_location_t; +#define YYLTYPE rb_code_location_t +#define YYLTYPE_IS_DECLARED 1 typedef struct rb_parser_ast_token { int id; @@ -673,7 +675,7 @@ typedef struct RNode_LIT { typedef struct RNode_INTEGER { NODE node; - char* val; + char *val; int minus; int base; } rb_node_integer_t; @@ -681,14 +683,14 @@ typedef struct RNode_INTEGER { typedef struct RNode_FLOAT { NODE node; - char* val; + char *val; int minus; } rb_node_float_t; typedef struct RNode_RATIONAL { NODE node; - char* val; + char *val; int minus; int base; int seen_point; @@ -703,7 +705,7 @@ enum rb_numeric_type { typedef struct RNode_IMAGINARY { NODE node; - char* val; + char *val; int minus; int base; int seen_point; @@ -1378,10 +1380,6 @@ typedef struct rb_parser_config_struct { // int rb_dvar_defined(ID id, const rb_iseq_t *iseq); int (*dvar_defined)(ID, const void*); - /* Compile (parse.y) */ - int (*literal_cmp)(VALUE val, VALUE lit); - parser_st_index_t (*literal_hash)(VALUE a); - /* Error (Exception) */ RBIMPL_ATTR_FORMAT(RBIMPL_PRINTF_FORMAT, 6, 0) VALUE (*syntax_error_append)(VALUE, VALUE, int, int, rb_encoding*, const char*, va_list); diff --git a/test/ruby/test_literal.rb b/test/ruby/test_literal.rb index 61e1e096df8d08..c6154af1f61161 100644 --- a/test/ruby/test_literal.rb +++ b/test/ruby/test_literal.rb @@ -509,9 +509,6 @@ def test_hash_duplicated_key ) do |key| assert_warning(/key #{Regexp.quote(eval(key).inspect)} is duplicated/) { eval("{#{key} => :bar, #{key} => :foo}") } end - - assert_warning(/key 1 is duplicated/) { eval("{__LINE__ => :bar, 1 => :foo}") } - assert_warning(/key \"FILENAME\" is duplicated/) { eval("{__FILE__ => :bar, 'FILENAME' => :foo}", binding, "FILENAME") } end def test_hash_frozen_key_id diff --git a/test/ruby/test_syntax.rb b/test/ruby/test_syntax.rb index 371c41fe37beda..7cc5e542a7b7bd 100644 --- a/test/ruby/test_syntax.rb +++ b/test/ruby/test_syntax.rb @@ -734,7 +734,7 @@ def test_duplicated_when end } } - assert_warning(/3: #{w}.+4: #{w}.+4: #{w}.+5: #{w}.+5: #{w}/m) { + assert_warning(/3: #{w}/m) { eval %q{ case 1 when __LINE__, __LINE__ @@ -743,7 +743,7 @@ def test_duplicated_when end } } - assert_warning(/3: #{w}.+4: #{w}.+4: #{w}.+5: #{w}.+5: #{w}/m) { + assert_warning(/3: #{w}/m) { eval %q{ case 1 when __FILE__, __FILE__ diff --git a/universal_parser.c b/universal_parser.c index 4a02675eec303d..dfb02eaa4c1db7 100644 --- a/universal_parser.c +++ b/universal_parser.c @@ -54,6 +54,10 @@ #define st_delete rb_parser_st_delete #undef st_is_member #define st_is_member parser_st_is_member +#undef st_init_table +#define st_init_table rb_parser_st_init_table +#undef st_lookup +#define st_lookup rb_parser_st_lookup #define rb_encoding void @@ -227,9 +231,6 @@ struct rb_imemo_tmpbuf_struct { #define rb_local_defined p->config->local_defined #define rb_dvar_defined p->config->dvar_defined -#define literal_cmp p->config->literal_cmp -#define literal_hash p->config->literal_hash - #define rb_syntax_error_append p->config->syntax_error_append #define rb_raise p->config->raise #define syntax_error_new p->config->syntax_error_new From 4db7c8a24ad66e15ef6bce053c4d5d90b84cb855 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Tue, 2 Apr 2024 10:12:48 +0900 Subject: [PATCH 208/211] Warn ostruct for Ruby 3.5 --- lib/bundled_gems.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/bundled_gems.rb b/lib/bundled_gems.rb index a8ea9fe211c2ee..15ce9000ee8e27 100644 --- a/lib/bundled_gems.rb +++ b/lib/bundled_gems.rb @@ -26,6 +26,7 @@ module Gem::BUNDLED_GEMS "resolv-replace" => "3.4.0", "rinda" => "3.4.0", "syslog" => "3.4.0", + "ostruct" => "3.5.0", }.freeze EXACT = { @@ -41,6 +42,7 @@ module Gem::BUNDLED_GEMS "resolv-replace" => true, "rinda" => true, "syslog" => true, + "ostruct" => true, }.freeze PREFIXED = { From 94f56098b12552c447bda764c7e1bdac7e84efb3 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Tue, 2 Apr 2024 10:13:33 +0900 Subject: [PATCH 209/211] Use fixed version of rake with ostruct --- gems/bundled_gems | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gems/bundled_gems b/gems/bundled_gems index 408b5426d4a450..38c97c77a01754 100644 --- a/gems/bundled_gems +++ b/gems/bundled_gems @@ -8,7 +8,7 @@ minitest 5.22.3 https://github.com/minitest/minitest 287b35d60c8e19c11ba090efc6eeb225325a8520 power_assert 2.0.3 https://github.com/ruby/power_assert 84e85124c5014a139af39161d484156cfe87a9ed -rake 13.1.0 https://github.com/ruby/rake +rake 13.2.0 https://github.com/ruby/rake test-unit 3.6.2 https://github.com/test-unit/test-unit rexml 3.2.6 https://github.com/ruby/rexml rss 0.3.0 https://github.com/ruby/rss From a65d49ce77af76b29ee17ec64c15b7e981818c10 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Tue, 2 Apr 2024 10:48:39 +0900 Subject: [PATCH 210/211] Use Rake 13.2.0 --- spec/bundler/support/builders.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/bundler/support/builders.rb b/spec/bundler/support/builders.rb index 46931e9ca530ba..91e6e6e574b931 100644 --- a/spec/bundler/support/builders.rb +++ b/spec/bundler/support/builders.rb @@ -18,7 +18,7 @@ def pl(platform) end def rake_version - "13.1.0" + "13.2.0" end def build_repo1 From 457a30df85a1fbf3b7a7b71064dc3d11466d8da3 Mon Sep 17 00:00:00 2001 From: git Date: Tue, 2 Apr 2024 02:22:51 +0000 Subject: [PATCH 211/211] Update bundled gems list at a65d49ce77af76b29ee17ec64c15b7 [ci skip] --- NEWS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS.md b/NEWS.md index 874794bff9dbd6..77e48d8ca0ef38 100644 --- a/NEWS.md +++ b/NEWS.md @@ -54,6 +54,7 @@ The following default gems are updated. The following bundled gems are updated. * minitest 5.22.3 +* rake 13.2.0 * test-unit 3.6.2 * net-ftp 0.3.4 * net-imap 0.4.10