Compatibility with IPython 8 beta

[frenzymadness]

Hello.

I've started testing IPython 8 beta release and I see that there is one incompatibility in tests, the problem is caused by this change: https://github.com/ipython/ipython/blob/master/docs/source/whatsnew/version8.rst#traceback-improvements which makes this regex

nbclient/nbclient/tests/test_client.py

Line 30 in 612a9e8

ipython_input_pat = re.compile(r'<ipython-input-\d+-[0-9a-f]+>')

not compatible with the new output.

The whole output is kinda big but only three tests are failing and all of them for the same reason:

=========================== short test summary info ============================
FAILED nbclient/tests/test_client.py::test_run_all_notebooks[Interrupt.ipynb-opts6]
FAILED nbclient/tests/test_client.py::test_run_all_notebooks[Skip Exceptions with Cell Tags.ipynb-opts8]
FAILED nbclient/tests/test_client.py::test_run_all_notebooks[Skip Exceptions.ipynb-opts9]

Would it make sense to have a different regex for IPython 8+?

[davidbrochart]

Hi @frenzymadness, thanks for reporting the issue.

Would it make sense to have a different regex for IPython 8+?

Yes, probably. Something like input In [2], is much less randomized, but I don't think we have some code that looks like this in our tests.
Do you want to open a PR?

[frenzymadness]

Will do it tomorrow.

[bnavigator]

Still getting this with IPython 8.0.0, although #190 is in nbclient 0.5.10:

[  169s] ________________ test_run_all_notebooks[Interrupt.ipynb-opts6] _________________
[  169s] 
[  169s] input_name = 'Interrupt.ipynb'
[  169s] opts = {'allow_errors': True, 'interrupt_on_timeout': True, 'kernel_name': 'python', 'timeout': 1}
[  169s] 
[  169s]     @pytest.mark.parametrize(
[  169s]         ["input_name", "opts"],
[  169s]         [
[  169s]             ("Other Comms.ipynb", dict(kernel_name="python")),
[  169s]             ("Clear Output.ipynb", dict(kernel_name="python")),
[  169s]             ("Empty Cell.ipynb", dict(kernel_name="python")),
[  169s]             ("Factorials.ipynb", dict(kernel_name="python")),
[  169s]             ("HelloWorld.ipynb", dict(kernel_name="python")),
[  169s]             ("Inline Image.ipynb", dict(kernel_name="python")),
[  169s]             (
[  169s]                 "Interrupt.ipynb",
[  169s]                 dict(kernel_name="python", timeout=1, interrupt_on_timeout=True, allow_errors=True),
[  169s]             ),
[  169s]             ("JupyterWidgets.ipynb", dict(kernel_name="python")),
[  169s]             ("Skip Exceptions with Cell Tags.ipynb", dict(kernel_name="python")),
[  169s]             ("Skip Exceptions.ipynb", dict(kernel_name="python", allow_errors=True)),
[  169s]             ("Skip Execution with Cell Tag.ipynb", dict(kernel_name="python")),
[  169s]             ("SVG.ipynb", dict(kernel_name="python")),
[  169s]             ("Unicode.ipynb", dict(kernel_name="python")),
[  169s]             ("UnicodePy3.ipynb", dict(kernel_name="python")),
[  169s]             ("update-display-id.ipynb", dict(kernel_name="python")),
[  169s]             ("Check History in Memory.ipynb", dict(kernel_name="python")),
[  169s]         ],
[  169s]     )
[  169s]     def test_run_all_notebooks(input_name, opts):
[  169s]         """Runs a series of test notebooks and compares them to their actual output"""
[  169s]         input_file = os.path.join(current_dir, 'files', input_name)
[  169s]         input_nb, output_nb = run_notebook(input_file, opts, notebook_resources())
[  169s] >       assert_notebooks_equal(input_nb, output_nb)
[  169s] 
[  169s] nbclient/tests/test_client.py:284: 
[  169s] _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
[  169s] 
[  169s] expected = {'cells': [{'cell_type': 'code', 'execution_count': 1, 'metadata': {'collapsed': False}, 'outputs': [{'ename': 'Keyboa...t_type': 'stream', 'text': 'done\n'}], 'source': 'print("done")'}], 'metadata': {}, 'nbformat': 4, 'nbformat_minor': 0}
[  169s] actual = {'cells': [{'cell_type': 'code', 'metadata': {'collapsed': False, 'execution': {'iopub.status.busy': '2022-01-15T21:59...ents_lexer': 'ipython3', 'nbconvert_exporter': 'python', 'file_extension': '.py'}}, 'nbformat': 4, 'nbformat_minor': 0}
[  169s] 
[  169s]     def assert_notebooks_equal(expected, actual):
[  169s]         expected_cells = expected['cells']
[  169s]         actual_cells = actual['cells']
[  169s]         assert len(expected_cells) == len(actual_cells)
[  169s]     
[  169s]         for expected_cell, actual_cell in zip(expected_cells, actual_cells):
[  169s]             # Uncomment these to help debug test failures better
[  169s]             # from pprint import pprint
[  169s]             # pprint(expected_cell)
[  169s]             # pprint(actual_cell)
[  169s]             expected_outputs = expected_cell.get('outputs', [])
[  169s]             actual_outputs = actual_cell.get('outputs', [])
[  169s]             normalized_expected_outputs = list(map(normalize_output, expected_outputs))
[  169s]             normalized_actual_outputs = list(map(normalize_output, actual_outputs))
[  169s] >           assert normalized_expected_outputs == normalized_actual_outputs
[  169s] E           AssertionError: assert [{'ename': 'KeyboardInterrupt',\n  'evalue': '',\n  'output_type': 'error',\n  'traceback': ['---------------------------------------------------------------------------',\n                'KeyboardInterrupt                         Traceback (most '\n                'recent call last)',\n                '<IPY-INPUT> in <module>\n----> 1 while True: continue\n',\n                'KeyboardInterrupt: ']}] == [{'name': 'stderr', 'output_type': 'stream', 'text': '\nKeyboardInterrupt\n\n'}]
[  169s] E             At index 0 diff: {'ename': 'KeyboardInterrupt', 'evalue': '', 'output_type': 'error', 'traceback': ['---------------------------------------------------------------------------', 'KeyboardInterrupt                         Traceback (most recent call last)', '<IPY-INPUT> in <module>\n----> 1 while True: continue\n', 'KeyboardInterrupt: ']} != {'output_type': 'stream', 'name': 'stderr', 'text': '\nKeyboardInterrupt\n\n'}
[  169s] E             Full diff:
[  169s] E               [
[  169s] E             -  {'name': 'stderr',
[  169s] E             +  {'ename': 'KeyboardInterrupt',
[  169s] E             +   'evalue': '',
[  169s] E             -   'output_type': 'stream',
[  169s] E             ?                   ^^ ^^^
[  169s] E             +   'output_type': 'error',
[  169s] E             ?                   ^ ^^^
[  169s] E             -   'text': '\n'
[  169s] E             +   'traceback': ['---------------------------------------------------------------------------',
[  169s] E             +                 'KeyboardInterrupt                         Traceback (most '
[  169s] E             +                 'recent call last)',
[  169s] E             +                 '<IPY-INPUT> in <module>\n'
[  169s] E             +                 '----> 1 while True: continue\n',
[  169s] E             -           'KeyboardInterrupt\n'
[  169s] E             ?                             ^^
[  169s] E             +                 'KeyboardInterrupt: ']},
[  169s] E             ? ++++++                            ^^ +++
[  169s] E             -           '\n'},
[  169s] E               ]
[  169s] 
[  169s] nbclient/tests/test_client.py:231: AssertionError

[davidbrochart]

I'm not sure it comes from IPython, but maybe from ipykernel. I opened ipython/ipykernel#845.

[bnavigator]

You're right it's random. All I can say is it did not fail in https://build.opensuse.org/package/show/devel:languages:python:jupyter/python-nbclient when building with latest ipykernel but IPython 7.30.1 and started failing after the update to IPython 8.0