From ad9f4e74cd864f3c2afaa0e5a9a5f7471dd44745 Mon Sep 17 00:00:00 2001 From: Keewis Date: Sat, 10 Sep 2022 00:25:19 +0200 Subject: [PATCH 01/18] ignore TokenErrors in favor of `black`'s error messages --- blackdoc/formats/doctest.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/blackdoc/formats/doctest.py b/blackdoc/formats/doctest.py index a73b74c..c6029a8 100644 --- a/blackdoc/formats/doctest.py +++ b/blackdoc/formats/doctest.py @@ -1,5 +1,7 @@ import itertools import re +import tokenize +from tokenize import TokenError import more_itertools @@ -53,16 +55,28 @@ def detection_func(lines): return line_range, name, "\n".join(lines) -def tokenize(code): +def suppress(iterable, errors): + iter_ = iter(iterable) + while True: + try: + yield next(iter_) + except errors: + yield None + except StopIteration: + break + + +def extract_string_tokens(code): import io - import tokenize readline = io.StringIO(code).readline + tokens = tokenize.generate_tokens(readline) + # suppress invalid code errors: `black` will raise with a better error message return ( token - for token in tokenize.generate_tokens(readline) - if token.type == tokenize.STRING + for token in suppress(tokens, TokenError) + if token is not None and token.type == tokenize.STRING ) From 3de0b688376471c9c64709a7b787ee3dd60a8281 Mon Sep 17 00:00:00 2001 From: Keewis Date: Sat, 10 Sep 2022 00:26:40 +0200 Subject: [PATCH 02/18] improve the triple-quote extraction function --- blackdoc/formats/doctest.py | 16 +++++----------- blackdoc/tests/test_doctest.py | 10 +++++----- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/blackdoc/formats/doctest.py b/blackdoc/formats/doctest.py index c6029a8..db4a639 100644 --- a/blackdoc/formats/doctest.py +++ b/blackdoc/formats/doctest.py @@ -94,17 +94,11 @@ def detect_quotes(string): else: return None - def expand_quotes(quotes, n_lines): - lines = [None] * n_lines - for token, quote in quotes.items(): - token_length = token.end[0] - token.start[0] + 1 - lines[token.start[0] - 1 : token.end[0]] = [quote] * token_length - return lines - - string_tokens = list(tokenize(line)) - quotes = {token: detect_quotes(token.string) for token in string_tokens} - lines = line.split("\n") - return expand_quotes(quotes, len(lines)) + string_tokens = list(extract_string_tokens(line)) + token_quotes = {token: detect_quotes(token.string) for token in string_tokens} + quotes = set(token_quotes.values()) + + return more_itertools.first(quotes, None) def extraction_func(line): diff --git a/blackdoc/tests/test_doctest.py b/blackdoc/tests/test_doctest.py index 4d2711d..da6ce49 100644 --- a/blackdoc/tests/test_doctest.py +++ b/blackdoc/tests/test_doctest.py @@ -11,11 +11,11 @@ @pytest.mark.parametrize( ("string", "expected"), ( - pytest.param("", [None], id="empty string"), - pytest.param("a", [None], id="no quotes"), - pytest.param("'''a'''", ["'''"], id="single quotes"), - pytest.param('"""a"""', ['"""'], id="double quotes"), - pytest.param('"a"""', [None], id="trailing empty string"), + pytest.param("", None, id="empty string"), + pytest.param("a", None, id="no quotes"), + pytest.param("'''a'''", "'''", id="single quotes"), + pytest.param('"""a"""', '"""', id="double quotes"), + pytest.param('"a"""', None, id="trailing empty string"), ), ) def test_detect_docstring_quotes(string, expected): From e28048ccf29fcbad22051b44ea42decd2afbecd0 Mon Sep 17 00:00:00 2001 From: Keewis Date: Sat, 10 Sep 2022 00:27:07 +0200 Subject: [PATCH 03/18] add a few more tests to the docstring detection --- blackdoc/tests/test_doctest.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/blackdoc/tests/test_doctest.py b/blackdoc/tests/test_doctest.py index da6ce49..a4f8340 100644 --- a/blackdoc/tests/test_doctest.py +++ b/blackdoc/tests/test_doctest.py @@ -16,6 +16,28 @@ pytest.param("'''a'''", "'''", id="single quotes"), pytest.param('"""a"""', '"""', id="double quotes"), pytest.param('"a"""', None, id="trailing empty string"), + pytest.param( + textwrap.dedent( + """\ + ''' + multiple lines + ''' + """ + ).rstrip(), + "'''", + id="multiple lines single quotes", + ), + pytest.param( + textwrap.dedent( + '''\ + """ + multiple lines + """ + ''' + ).rstrip(), + '"""', + id="multiple lines double quotes", + ), ), ) def test_detect_docstring_quotes(string, expected): From a6aad606c8ae444d92113651897ea2b773116fa7 Mon Sep 17 00:00:00 2001 From: Keewis Date: Sat, 10 Sep 2022 02:03:43 +0200 Subject: [PATCH 04/18] move tokenizing to a different function --- blackdoc/formats/doctest.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/blackdoc/formats/doctest.py b/blackdoc/formats/doctest.py index db4a639..1b1a584 100644 --- a/blackdoc/formats/doctest.py +++ b/blackdoc/formats/doctest.py @@ -1,3 +1,4 @@ +import io import itertools import re import tokenize @@ -66,12 +67,15 @@ def suppress(iterable, errors): break -def extract_string_tokens(code): - import io - +def tokenize_string(code): readline = io.StringIO(code).readline - tokens = tokenize.generate_tokens(readline) + return tokenize.generate_tokens(readline) + + +def extract_string_tokens(code): + tokens = tokenize_string(code) + # suppress invalid code errors: `black` will raise with a better error message return ( token From b6e4068c07a1eed0a2db9007d56c807b12051f19 Mon Sep 17 00:00:00 2001 From: Keewis Date: Sat, 10 Sep 2022 02:04:49 +0200 Subject: [PATCH 05/18] replace replace_quotes with a new restore_quotes function The advantage is that we're actually using `tokenize` to find the position of the triple-quoted strings before replacing the quotes. --- blackdoc/formats/doctest.py | 76 +++++++++++++++++++++++++------------ 1 file changed, 52 insertions(+), 24 deletions(-) diff --git a/blackdoc/formats/doctest.py b/blackdoc/formats/doctest.py index 1b1a584..3f87fb1 100644 --- a/blackdoc/formats/doctest.py +++ b/blackdoc/formats/doctest.py @@ -139,45 +139,73 @@ def remove_prompt(line): }, extracted_line -def replace_quotes(line, current, saved): - if current is None or saved is None: - return line - elif current == saved: - return line - else: - return line.replace(current, saved) +def restore_quotes(code_unit, original_quotes): + def is_docstring(string): + return (string.startswith("'''") and string.endswith("'''")) or ( + string.startswith('"""') and string.endswith('"""') + ) + + def compute_offset(pos, offsets): + lineno, charno = pos + return offsets[lineno - 1] + charno + 1 + + if original_quotes is None: + return code_unit + + to_replace = "'''" if original_quotes == '"""' else '"""' + + string_tokens = extract_string_tokens(code_unit) + triple_quote_tokens = [ + token + for token in string_tokens + if token.string.startswith(to_replace) and token.string.endswith(to_replace) + ] + + line_lengths = [len(line) for line in code_unit.split("\n")] + offsets = [0] + list(itertools.accumulate(line_lengths[:-1])) + + mutable_string = io.StringIO(code_unit) + for token in triple_quote_tokens: + # reset stream + mutable_string.seek(0) + + # find the offset in the stream + start = compute_offset(token.start, offsets) + end = compute_offset(token.end, offsets) - 3 + mutable_string.seek(start) + mutable_string.write(original_quotes) -def reformatting_func(line, docstring_quotes): + mutable_string.seek(end) + mutable_string.write(original_quotes) + + mutable_string.seek(0) + restored_code_unit = mutable_string.getvalue() + + return restored_code_unit + + +def reformatting_func(code_unit, docstring_quotes): def add_prompt(prompt, line): if not line: return prompt return " ".join([prompt, line]) - lines = line.rstrip().split("\n") + restored_quotes = restore_quotes(code_unit, docstring_quotes) + + lines = restored_quotes.rstrip().split("\n") if block_start_re.match(lines[0]): lines.append("") - lines = iter(lines) - + lines_ = iter(lines) reformatted = list( itertools.chain( more_itertools.always_iterable( - add_prompt(prompt, more_itertools.first(lines)) + add_prompt(prompt, more_itertools.first(lines_)) ), - (add_prompt(continuation_prompt, line) for line in lines), - ) - ) - - # make sure nested docstrings still work - current_quotes = detect_docstring_quotes("\n".join(reformatted)) - restored = "\n".join( - replace_quotes(line, current, saved) - for line, saved, current in itertools.zip_longest( - reformatted, docstring_quotes, current_quotes + (add_prompt(continuation_prompt, line) for line in lines_), ) - if line is not None ) - return restored + return "\n".join(reformatted) From 9d51dc535221fae198776faa52efbe2f9c14e040 Mon Sep 17 00:00:00 2001 From: Keewis Date: Sat, 10 Sep 2022 02:06:22 +0200 Subject: [PATCH 06/18] remove the now unused expand_tokens --- blackdoc/formats/doctest.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/blackdoc/formats/doctest.py b/blackdoc/formats/doctest.py index 3f87fb1..9915765 100644 --- a/blackdoc/formats/doctest.py +++ b/blackdoc/formats/doctest.py @@ -84,11 +84,6 @@ def extract_string_tokens(code): ) -def expand_tokens(token): - length = token.end[0] - token.start[0] + 1 - return [token.string] * length - - def detect_docstring_quotes(line): def detect_quotes(string): if string.startswith("'''"): From ee0961a5ac97c7b5513e28cec744c05a1f0e586f Mon Sep 17 00:00:00 2001 From: Keewis Date: Sat, 10 Sep 2022 02:06:55 +0200 Subject: [PATCH 07/18] adapt the tests to the new situation --- blackdoc/tests/test_doctest.py | 132 ++++++++------------------------- 1 file changed, 31 insertions(+), 101 deletions(-) diff --git a/blackdoc/tests/test_doctest.py b/blackdoc/tests/test_doctest.py index a4f8340..6052b02 100644 --- a/blackdoc/tests/test_doctest.py +++ b/blackdoc/tests/test_doctest.py @@ -109,34 +109,22 @@ def prepare_lines(lines, remove_prompt=False): ( pytest.param( "file", - [None], + None, ">>> file", id="single line", ), pytest.param( "", - [None], + None, ">>>", id="single empty line", ), - pytest.param( - "file", - [None, None, None], - ">>> file", - id="single line with more quotes", - ), pytest.param( '"""docstring"""', - ['"""'], + '"""', '>>> """docstring"""', id="single-line triple-quoted string", ), - pytest.param( - '"""docstring"""', - ['"""', '"""'], - '>>> """docstring"""', - id="single-line triple-quoted string with more quotes", - ), pytest.param( textwrap.dedent( """\ @@ -146,7 +134,7 @@ def prepare_lines(lines, remove_prompt=False): ] """.rstrip() ), - [None, None, None, None], + None, textwrap.dedent( """\ >>> a = [ @@ -157,46 +145,6 @@ def prepare_lines(lines, remove_prompt=False): ), id="multiple lines", ), - pytest.param( - textwrap.dedent( - """\ - a = [ - 1, - 2, - ] - """.rstrip() - ), - [None, None], - textwrap.dedent( - """\ - >>> a = [ - ... 1, - ... 2, - ... ] - """.rstrip() - ), - id="multiple lines with less quotes", - ), - pytest.param( - textwrap.dedent( - """\ - a = [ - 1, - 2, - ] - """.rstrip() - ), - [None, None, None, None, None], - textwrap.dedent( - """\ - >>> a = [ - ... 1, - ... 2, - ... ] - """.rstrip() - ), - id="multiple lines with more quotes", - ), pytest.param( textwrap.dedent( """\ @@ -205,7 +153,7 @@ def prepare_lines(lines, remove_prompt=False): ''' """.rstrip() ), - ["'''", "'''", "'''"], + "'''", textwrap.dedent( """\ >>> ''' @@ -215,42 +163,6 @@ def prepare_lines(lines, remove_prompt=False): ), id="multi-line triple-quoted string", ), - pytest.param( - textwrap.dedent( - """\ - ''' - docstring content - ''' - """.rstrip() - ), - ["'''"], - textwrap.dedent( - """\ - >>> ''' - ... docstring content - ... ''' - """.rstrip() - ), - id="multi-line triple-quoted string with less quotes", - ), - pytest.param( - textwrap.dedent( - """\ - ''' - docstring content - ''' - """.rstrip() - ), - ["'''", "'''", "'''", None, None], - textwrap.dedent( - """\ - >>> ''' - ... docstring content - ... ''' - """.rstrip() - ), - id="multi-line triple-quoted string with more quotes", - ), pytest.param( textwrap.dedent( """\ @@ -260,7 +172,7 @@ def prepare_lines(lines, remove_prompt=False): ''' """.rstrip(), ), - ["'''", "'''", "'''", "'''"], + "'''", textwrap.dedent( """\ >>> ''' arbitrary triple-quoted string @@ -273,7 +185,7 @@ def prepare_lines(lines, remove_prompt=False): ), pytest.param( '"""inverted quotes"""', - ['"""'], + '"""', '>>> """inverted quotes"""', id="triple-quoted string with inverted quotes", ), @@ -284,7 +196,7 @@ def myfunc(arg1, arg2): pass """ ), - [None, None, None], + None, textwrap.dedent( """\ >>> def myfunc(arg1, arg2): @@ -301,13 +213,13 @@ def myfunc(arg1, arg2): """ ), - [None, None], + None, ">>> a = 1", id="trailing newline at the end of a normal line", ), pytest.param( "# this is not a block:", - [None], + None, ">>> # this is not a block:", id="trailing colon at the end of a comment", ), @@ -321,7 +233,7 @@ def f(arg1, arg2): ''' """ ), - [None, "'''", "'''", "'''", "'''", None], + "'''", textwrap.dedent( """\ >>> def f(arg1, arg2): @@ -336,10 +248,28 @@ def f(arg1, arg2): ), pytest.param( "s = '''triple-quoted string'''", - [None, "'''", None], + "'''", ">>> s = '''triple-quoted string'''", id="moved triple-quoted string", - marks=[pytest.mark.skip(reason="to be fixed")], + ), + pytest.param( + textwrap.dedent( + '''\ + def f(arg1, arg2): + """ docstring """ + s = "trailing empty string""" + ''' + ), + "'''", + textwrap.dedent( + """\ + >>> def f(arg1, arg2): + ... ''' docstring ''' + ... s = "trailing empty string\""" + ... + """.rstrip() + ), + id="docstring and trailing empty string", ), ), ) From f1f823d5e75e8044c236d72ad0e3c1addd17ab07 Mon Sep 17 00:00:00 2001 From: Keewis Date: Sat, 10 Sep 2022 02:29:23 +0200 Subject: [PATCH 08/18] one more test case with both a docstring and a triple-quoted string --- blackdoc/tests/test_doctest.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/blackdoc/tests/test_doctest.py b/blackdoc/tests/test_doctest.py index 6052b02..ec66c31 100644 --- a/blackdoc/tests/test_doctest.py +++ b/blackdoc/tests/test_doctest.py @@ -271,6 +271,25 @@ def f(arg1, arg2): ), id="docstring and trailing empty string", ), + pytest.param( + textwrap.dedent( + '''\ + def f(arg1, arg2): + """ docstring """ + s = """triple-quoted string""" + ''' + ), + "'''", + textwrap.dedent( + """\ + >>> def f(arg1, arg2): + ... ''' docstring ''' + ... s = '''triple-quoted string''' + ... + """.rstrip() + ), + id="docstring and triple-quoted string", + ), ), ) def test_reformatting_func(code_unit, docstring_quotes, expected): From 50f41cb60b5c9ff73c814802989964b0b362eb41 Mon Sep 17 00:00:00 2001 From: Keewis Date: Sat, 10 Sep 2022 02:30:05 +0200 Subject: [PATCH 09/18] fix the offset calculation --- blackdoc/formats/doctest.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/blackdoc/formats/doctest.py b/blackdoc/formats/doctest.py index 9915765..7cf5aa3 100644 --- a/blackdoc/formats/doctest.py +++ b/blackdoc/formats/doctest.py @@ -142,7 +142,7 @@ def is_docstring(string): def compute_offset(pos, offsets): lineno, charno = pos - return offsets[lineno - 1] + charno + 1 + return offsets[lineno] + charno + 1 if original_quotes is None: return code_unit @@ -156,8 +156,10 @@ def compute_offset(pos, offsets): if token.string.startswith(to_replace) and token.string.endswith(to_replace) ] - line_lengths = [len(line) for line in code_unit.split("\n")] - offsets = [0] + list(itertools.accumulate(line_lengths[:-1])) + offsets = {1: 0} | { + lineno: m.start() + for lineno, m in enumerate(re.finditer("\n", code_unit), start=2) + } mutable_string = io.StringIO(code_unit) for token in triple_quote_tokens: From dbfb9f13b7a4aaaf1246149b6160f2d3a7dfe7ae Mon Sep 17 00:00:00 2001 From: Keewis Date: Sat, 10 Sep 2022 02:30:47 +0200 Subject: [PATCH 10/18] extend the example doctests --- doc/directory/file.py | 16 ++++++++++++++++ doc/directory/reformatted.py | 11 +++++++++++ 2 files changed, 27 insertions(+) diff --git a/doc/directory/file.py b/doc/directory/file.py index 5ad6095..e26241e 100644 --- a/doc/directory/file.py +++ b/doc/directory/file.py @@ -7,6 +7,22 @@ ... "f", "g",'h', "i", "j", "k",'l', 'm', ... } +>>> s = ( +... "a" +... + "b" +... ) + +>>> def f(): +... '''nested docstring +... +... parameter documentation +... ''' +... +... s = ( +... '''triple-quoted string''' +... ) +... + ipython: In [1]: d= { "a": 0, "b": 1, "c": 2, ...: "d": 3, "e": 4, "f": 5, "g": 6, diff --git a/doc/directory/reformatted.py b/doc/directory/reformatted.py index fa35f81..0c2325b 100644 --- a/doc/directory/reformatted.py +++ b/doc/directory/reformatted.py @@ -21,6 +21,17 @@ ... "m", ... } +>>> s = "a" + "b" + +>>> def f(): +... '''nested docstring +... +... parameter documentation +... ''' +... +... s = '''triple-quoted string''' +... + ipython: In [1]: d = {"a": 0, "b": 1, "c": 2, "d": 3, "e": 4, "f": 5, "g": 6, "h": 7, "i": 8, "j": 9} """ From 66e28b8243826f4a02b85dd2a635b7cb9c6ccd42 Mon Sep 17 00:00:00 2001 From: Keewis Date: Sat, 10 Sep 2022 12:24:48 +0200 Subject: [PATCH 11/18] update changelog --- doc/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/changelog.rst b/doc/changelog.rst index 80a5cb7..625c313 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -2,7 +2,7 @@ Changelog ========= v0.4.0 (*unreleased*) --------------------- - +- replace docstrings by modifying by token (:pull:`144`) v0.3.6 (25 August 2022) ----------------------- From afd48aa692487a10606ea03c82e8d7f235bb0e31 Mon Sep 17 00:00:00 2001 From: Keewis Date: Sat, 10 Sep 2022 12:30:42 +0200 Subject: [PATCH 12/18] move the line offset calculation to a separate function and use compat code --- blackdoc/formats/doctest.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/blackdoc/formats/doctest.py b/blackdoc/formats/doctest.py index 7cf5aa3..4b46cdc 100644 --- a/blackdoc/formats/doctest.py +++ b/blackdoc/formats/doctest.py @@ -140,6 +140,15 @@ def is_docstring(string): string.startswith('"""') and string.endswith('"""') ) + def line_offsets(code_unit): + offsets = { + lineno: m.start() + for lineno, m in enumerate(re.finditer("\n", code_unit), start=2) + } + offsets[1] = 0 + + return offsets + def compute_offset(pos, offsets): lineno, charno = pos return offsets[lineno] + charno + 1 @@ -156,11 +165,7 @@ def compute_offset(pos, offsets): if token.string.startswith(to_replace) and token.string.endswith(to_replace) ] - offsets = {1: 0} | { - lineno: m.start() - for lineno, m in enumerate(re.finditer("\n", code_unit), start=2) - } - + offsets = line_offsets(code_unit) mutable_string = io.StringIO(code_unit) for token in triple_quote_tokens: # reset stream From 8f0ebb3beb071fc7df9f44f0bbe453246ced789d Mon Sep 17 00:00:00 2001 From: Keewis Date: Sat, 10 Sep 2022 12:31:41 +0200 Subject: [PATCH 13/18] remove unnecessary stream resets --- blackdoc/formats/doctest.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/blackdoc/formats/doctest.py b/blackdoc/formats/doctest.py index 4b46cdc..b230053 100644 --- a/blackdoc/formats/doctest.py +++ b/blackdoc/formats/doctest.py @@ -168,9 +168,6 @@ def compute_offset(pos, offsets): offsets = line_offsets(code_unit) mutable_string = io.StringIO(code_unit) for token in triple_quote_tokens: - # reset stream - mutable_string.seek(0) - # find the offset in the stream start = compute_offset(token.start, offsets) end = compute_offset(token.end, offsets) - 3 @@ -181,7 +178,6 @@ def compute_offset(pos, offsets): mutable_string.seek(end) mutable_string.write(original_quotes) - mutable_string.seek(0) restored_code_unit = mutable_string.getvalue() return restored_code_unit From 79789783e7c827e6ac43619c1fd0967e8b80a4c2 Mon Sep 17 00:00:00 2001 From: Keewis Date: Mon, 12 Sep 2022 11:03:24 +0200 Subject: [PATCH 14/18] fix the offset computation --- blackdoc/formats/doctest.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/blackdoc/formats/doctest.py b/blackdoc/formats/doctest.py index b230053..e30b90e 100644 --- a/blackdoc/formats/doctest.py +++ b/blackdoc/formats/doctest.py @@ -141,17 +141,13 @@ def is_docstring(string): ) def line_offsets(code_unit): - offsets = { - lineno: m.start() - for lineno, m in enumerate(re.finditer("\n", code_unit), start=2) - } - offsets[1] = 0 + offsets = [m.end() for m in re.finditer("\n", code_unit)] - return offsets + return {lineno: offset for lineno, offset in enumerate([0] + offsets, start=1)} def compute_offset(pos, offsets): lineno, charno = pos - return offsets[lineno] + charno + 1 + return offsets[lineno] + charno if original_quotes is None: return code_unit From a574e09035c54f69790f2562728fb95697a7b638 Mon Sep 17 00:00:00 2001 From: Keewis Date: Mon, 12 Sep 2022 11:18:38 +0200 Subject: [PATCH 15/18] also test multi-line triple-quoted strings --- blackdoc/tests/test_doctest.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/blackdoc/tests/test_doctest.py b/blackdoc/tests/test_doctest.py index ec66c31..b231e50 100644 --- a/blackdoc/tests/test_doctest.py +++ b/blackdoc/tests/test_doctest.py @@ -250,7 +250,25 @@ def f(arg1, arg2): "s = '''triple-quoted string'''", "'''", ">>> s = '''triple-quoted string'''", - id="moved triple-quoted string", + id="triple-quoted string", + ), + pytest.param( + textwrap.dedent( + """\ + s = ''' + triple-quoted string + ''' + """ + ).rstrip(), + "'''", + textwrap.dedent( + """\ + >>> s = ''' + ... triple-quoted string + ... ''' + """.rstrip(), + ), + id="multi-line triple-quoted string", ), pytest.param( textwrap.dedent( From 0db1c348b0557ff83f491124f7db16ab18d17fea Mon Sep 17 00:00:00 2001 From: Keewis Date: Mon, 12 Sep 2022 11:18:54 +0200 Subject: [PATCH 16/18] remove a unused function --- blackdoc/formats/doctest.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/blackdoc/formats/doctest.py b/blackdoc/formats/doctest.py index e30b90e..4741535 100644 --- a/blackdoc/formats/doctest.py +++ b/blackdoc/formats/doctest.py @@ -135,11 +135,6 @@ def remove_prompt(line): def restore_quotes(code_unit, original_quotes): - def is_docstring(string): - return (string.startswith("'''") and string.endswith("'''")) or ( - string.startswith('"""') and string.endswith('"""') - ) - def line_offsets(code_unit): offsets = [m.end() for m in re.finditer("\n", code_unit)] From c165ac09587bdc7790c503b6cc4de0b74d5f754c Mon Sep 17 00:00:00 2001 From: Keewis Date: Mon, 12 Sep 2022 11:19:55 +0200 Subject: [PATCH 17/18] rewrite the quotes extraction function to also check the end of the string --- blackdoc/formats/doctest.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/blackdoc/formats/doctest.py b/blackdoc/formats/doctest.py index 4741535..deb0d25 100644 --- a/blackdoc/formats/doctest.py +++ b/blackdoc/formats/doctest.py @@ -84,18 +84,18 @@ def extract_string_tokens(code): ) -def detect_docstring_quotes(line): - def detect_quotes(string): - if string.startswith("'''"): +def detect_docstring_quotes(code_unit): + def extract_quotes(string): + if string.startswith("'''") and string.endswith("'''"): return "'''" - elif string.startswith('"""'): + elif string.startswith('"""') and string.endswith('"""'): return '"""' else: return None - string_tokens = list(extract_string_tokens(line)) - token_quotes = {token: detect_quotes(token.string) for token in string_tokens} - quotes = set(token_quotes.values()) + string_tokens = list(extract_string_tokens(code_unit)) + token_quotes = {token: extract_quotes(token.string) for token in string_tokens} + quotes = (quote for quote in token_quotes.values() if quote is not None) return more_itertools.first(quotes, None) From 60827ff5a215e6ce57dc65eb8aef320f2b7ad43c Mon Sep 17 00:00:00 2001 From: Keewis Date: Mon, 12 Sep 2022 11:33:16 +0200 Subject: [PATCH 18/18] make sure the tests actually cover the triple-quoted string changes --- blackdoc/tests/test_doctest.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/blackdoc/tests/test_doctest.py b/blackdoc/tests/test_doctest.py index b231e50..b1a9716 100644 --- a/blackdoc/tests/test_doctest.py +++ b/blackdoc/tests/test_doctest.py @@ -248,8 +248,8 @@ def f(arg1, arg2): ), pytest.param( "s = '''triple-quoted string'''", - "'''", - ">>> s = '''triple-quoted string'''", + '"""', + '>>> s = """triple-quoted string"""', id="triple-quoted string", ), pytest.param( @@ -260,13 +260,13 @@ def f(arg1, arg2): ''' """ ).rstrip(), - "'''", + '"""', textwrap.dedent( - """\ - >>> s = ''' + '''\ + >>> s = """ ... triple-quoted string - ... ''' - """.rstrip(), + ... """ + '''.rstrip(), ), id="multi-line triple-quoted string", ),