From c87360d45b9d9452ead729c28fba68e3ce57c4fd Mon Sep 17 00:00:00 2001 From: Fabian Haase Date: Sun, 28 Oct 2018 13:00:40 +0100 Subject: [PATCH 01/10] [skip ci] Integrate flake8 into validate_docstrings.py Signed-off-by: Fabian Haase --- scripts/validate_docstrings.py | 18 ++++++++++++++++-- setup.cfg | 25 +++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/scripts/validate_docstrings.py b/scripts/validate_docstrings.py index 29d485550be40..c1528508cf123 100755 --- a/scripts/validate_docstrings.py +++ b/scripts/validate_docstrings.py @@ -24,6 +24,9 @@ import inspect import importlib import doctest + +from flake8.api import legacy as flake8 + try: from io import StringIO except ImportError: @@ -40,7 +43,6 @@ from numpydoc.docscrape import NumpyDocString from pandas.io.formats.printing import pprint_thing - PRIVATE_CLASSES = ['NDFrame', 'IndexOpsMixin'] DIRECTIVES = ['versionadded', 'versionchanged', 'deprecated'] @@ -331,6 +333,12 @@ def parameter_mismatches(self): return errs + @property + def pep8_violations(self): + style_guide = flake8.get_style_guide(doctests=True) + report = style_guide.input_file(filename=self.source_file_name) + return report.get_statistics('') + @property def correct_parameters(self): return not bool(self.parameter_mismatches) @@ -446,7 +454,7 @@ def validate_one(func_name): if doc.summary != doc.summary.lstrip(): errs.append('Summary contains heading whitespaces.') elif (doc.is_function_or_method - and doc.summary.split(' ')[0][-1] == 's'): + and doc.summary.split(' ')[0][-1] == 's'): errs.append('Summary must start with infinitive verb, ' 'not third person (e.g. use "Generate" instead of ' '"Generates")') @@ -490,6 +498,12 @@ def validate_one(func_name): for param_err in param_errs: errs.append('\t{}'.format(param_err)) + pep8_errs = doc.pep8_violations + if pep8_errs: + errs.append('Errors in doctest sections') + for pep8_err in pep8_errs: + errs.append('\t{}'.format(pep8_err)) + if doc.is_function_or_method: if not doc.returns and "return" in doc.method_source: errs.append('No Returns section found') diff --git a/setup.cfg b/setup.cfg index e1d0dc84f464a..334cbd00177bb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,6 +28,31 @@ exclude = doc/temp/*.py, .eggs/*.py, versioneer.py + .tox + .git + +doctests = True +#TODO fix doctests +exclude_from_doctest = + ./pandas/_libs + ./pandas/api + ./pandas/compat + ./pandas/computation + ./pandas/core + ./pandas/errors + ./pandas/io + ./pandas/plotting + ./pandas/tests + ./pandas/tools + ./pandas/tseries + ./pandas/types + ./pandas/util + +[flake8-rst] +ignore = + F821, # undefined name + W391, # blank line at end of file [Seems to be a bug (v0.4.1)] + [yapf] based_on_style = pep8 From bc009a48fd612ce2e794661cb51991c36bc47242 Mon Sep 17 00:00:00 2001 From: Fabian Haase Date: Mon, 29 Oct 2018 09:28:30 +0100 Subject: [PATCH 02/10] [skip ci] Run flake8 on tmpfile containing the docstring Signed-off-by: Fabian Haase --- scripts/validate_docstrings.py | 38 +++++++++++++++++++++++++++++++--- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/scripts/validate_docstrings.py b/scripts/validate_docstrings.py index c1528508cf123..9e5352d7c101f 100755 --- a/scripts/validate_docstrings.py +++ b/scripts/validate_docstrings.py @@ -24,6 +24,7 @@ import inspect import importlib import doctest +from contextlib import contextmanager from flake8.api import legacy as flake8 @@ -335,9 +336,40 @@ def parameter_mismatches(self): @property def pep8_violations(self): - style_guide = flake8.get_style_guide(doctests=True) - report = style_guide.input_file(filename=self.source_file_name) - return report.get_statistics('') + with self._file_representation() as filename: + style_guide = flake8.get_style_guide(doctests=True) + report = style_guide.input_file(filename=filename) + return report.get_statistics('') + + @contextmanager + def _file_representation(self): + """ + Creates a tmp file containing the function without the body + + :returns filename of tmp file + """ + create_function = 'def {name}{signature}: # noqa: E501\n' \ + ' """{doc}"""\n' \ + ' pass\n' + + tmp_dir = os.path.join(BASE_PATH, 'build', 'validate_docstring') + os.makedirs(tmp_dir, exist_ok=True) + + filename = os.path.join(tmp_dir, self.name + '.py') + with open(filename, 'w') as f: + name = self.name.split('.')[-1] + sig = str(inspect.signature(self.obj)) + lines = self.raw_doc.split("\n") + indented_lines = [' ' + line if line else '' + for line in lines[1:]] + doc = '\n'.join([lines[0], *indented_lines]) + + f.write(create_function.format(name=name, signature=sig, doc=doc)) + + yield filename + + os.remove(filename) + os.rmdir(tmp_dir) @property def correct_parameters(self): From cb477f8e26b74dbcf03a7a578a61a6d79615bcc4 Mon Sep 17 00:00:00 2001 From: Fabian Haase Date: Tue, 30 Oct 2018 14:24:40 +0100 Subject: [PATCH 03/10] Merge with master * updated folders excluded from flake8 --doctests * fixed failing doctests Signed-off-by: Fabian Haase --- scripts/tests/test_validate_docstrings.py | 2 ++ setup.cfg | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/tests/test_validate_docstrings.py b/scripts/tests/test_validate_docstrings.py index 0e10265a7291d..1f86cb8a33d03 100644 --- a/scripts/tests/test_validate_docstrings.py +++ b/scripts/tests/test_validate_docstrings.py @@ -127,6 +127,7 @@ def head1(self, n=5): Examples -------- + >>> import pandas as pd >>> s = pd.Series(['Ant', 'Bear', 'Cow', 'Dog', 'Falcon']) >>> s.head() 0 Ant @@ -164,6 +165,7 @@ def contains(self, pat, case=True, na=np.nan): Examples -------- + >>> import pandas as pd >>> s = pd.Series(['Antelope', 'Lion', 'Zebra', np.nan]) >>> s.str.contains(pat='a') 0 False diff --git a/setup.cfg b/setup.cfg index 80bef6fdc3584..d323e1d2a891d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,7 +37,6 @@ exclude_from_doctest = ./pandas/_libs ./pandas/api ./pandas/compat - ./pandas/computation ./pandas/core ./pandas/errors ./pandas/io @@ -45,7 +44,6 @@ exclude_from_doctest = ./pandas/tests ./pandas/tools ./pandas/tseries - ./pandas/types ./pandas/util [flake8-rst] From 762de5d7d85e6b40c08c5cef0e13b819e8d6eb91 Mon Sep 17 00:00:00 2001 From: Fabian Haase Date: Tue, 30 Oct 2018 18:06:04 +0100 Subject: [PATCH 04/10] Fixed failing tests. * Examples in `test_validate_docstrings.py` lacked imports * Removed signature from function in generated file as not really needed at this point. * Used `clean_doc` instead of `raw_doc` to get correct indentation. * Added missing try: finally: block in `_file_representation()` Signed-off-by: Fabian Haase --- scripts/tests/test_validate_docstrings.py | 1 + scripts/validate_docstrings.py | 22 +++++++++++----------- setup.cfg | 1 - 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/scripts/tests/test_validate_docstrings.py b/scripts/tests/test_validate_docstrings.py index 1f86cb8a33d03..1276e5742e9ea 100644 --- a/scripts/tests/test_validate_docstrings.py +++ b/scripts/tests/test_validate_docstrings.py @@ -166,6 +166,7 @@ def contains(self, pat, case=True, na=np.nan): Examples -------- >>> import pandas as pd + >>> import numpy as np >>> s = pd.Series(['Antelope', 'Lion', 'Zebra', np.nan]) >>> s.str.contains(pat='a') 0 False diff --git a/scripts/validate_docstrings.py b/scripts/validate_docstrings.py index 9e5352d7c101f..c3645c90fb283 100755 --- a/scripts/validate_docstrings.py +++ b/scripts/validate_docstrings.py @@ -344,11 +344,12 @@ def pep8_violations(self): @contextmanager def _file_representation(self): """ - Creates a tmp file containing the function without the body + Temporarily creates file with current function inside. + The signature and body are **not** included. :returns filename of tmp file """ - create_function = 'def {name}{signature}: # noqa: E501\n' \ + create_function = 'def {name}():\n' \ ' """{doc}"""\n' \ ' pass\n' @@ -358,18 +359,17 @@ def _file_representation(self): filename = os.path.join(tmp_dir, self.name + '.py') with open(filename, 'w') as f: name = self.name.split('.')[-1] - sig = str(inspect.signature(self.obj)) - lines = self.raw_doc.split("\n") - indented_lines = [' ' + line if line else '' + lines = self.clean_doc.split("\n") + indented_lines = [(' ' * 4) + line if line else '' for line in lines[1:]] doc = '\n'.join([lines[0], *indented_lines]) - f.write(create_function.format(name=name, signature=sig, doc=doc)) - - yield filename - - os.remove(filename) - os.rmdir(tmp_dir) + f.write(create_function.format(name=name, doc=doc)) + try: + yield filename + finally: + os.remove(filename) + os.rmdir(tmp_dir) @property def correct_parameters(self): diff --git a/setup.cfg b/setup.cfg index d323e1d2a891d..34597947f69bc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -42,7 +42,6 @@ exclude_from_doctest = ./pandas/io ./pandas/plotting ./pandas/tests - ./pandas/tools ./pandas/tseries ./pandas/util From 9bc7f6518aaeb2b71ad11ebdbec78c7352146d58 Mon Sep 17 00:00:00 2001 From: Fabian Haase Date: Thu, 1 Nov 2018 12:48:14 +0100 Subject: [PATCH 05/10] Added test and slimmed PR * added test * simplified temporary file usage * included flake8 by calling application directly * removed doctests from setup.cfg Signed-off-by: Fabian Haase --- scripts/tests/test_validate_docstrings.py | 30 +++++++++++++- scripts/validate_docstrings.py | 48 +++++++++++------------ setup.cfg | 22 ----------- 3 files changed, 52 insertions(+), 48 deletions(-) diff --git a/scripts/tests/test_validate_docstrings.py b/scripts/tests/test_validate_docstrings.py index 1276e5742e9ea..2047cf9d5f8ff 100644 --- a/scripts/tests/test_validate_docstrings.py +++ b/scripts/tests/test_validate_docstrings.py @@ -587,6 +587,29 @@ def prefix_pandas(self): pass +class BadExamples(object): + + def doctest(self): + """ + Return whether each value contains `pat`. + + Examples + -------- + >>> import pandas as pd + >>> df = pd.DataFrame(np.ones((3, 3)), + ... columns=('a', 'b', 'c')) + >>> df.all(1) + 0 True + 1 True + 2 True + dtype: bool + + >>> df.all(bool_only=True) + Series([], dtype: bool) + """ + pass + + class TestValidator(object): def _import_path(self, klass=None, func=None): @@ -706,10 +729,13 @@ def test_bad_generic_functions(self, func): # See Also tests ('BadSeeAlso', 'prefix_pandas', ('pandas.Series.rename in `See Also` section ' - 'does not need `pandas` prefix',)) + 'does not need `pandas` prefix',)), + # Examples tests + ('BadExamples', 'doctest', + ('1 F821 undefined name \'np\'',)) ]) def test_bad_examples(self, capsys, klass, func, msgs): - result = validate_one(self._import_path(klass=klass, func=func)) # noqa:F821 + result = validate_one(self._import_path(klass=klass, func=func)) for msg in msgs: assert msg in ' '.join(result['errors']) diff --git a/scripts/validate_docstrings.py b/scripts/validate_docstrings.py index c3645c90fb283..db2b4fc25902e 100755 --- a/scripts/validate_docstrings.py +++ b/scripts/validate_docstrings.py @@ -24,9 +24,10 @@ import inspect import importlib import doctest +import tempfile from contextlib import contextmanager -from flake8.api import legacy as flake8 +from flake8.main.application import Application as Flake8 try: from io import StringIO @@ -44,6 +45,7 @@ from numpydoc.docscrape import NumpyDocString from pandas.io.formats.printing import pprint_thing + PRIVATE_CLASSES = ['NDFrame', 'IndexOpsMixin'] DIRECTIVES = ['versionadded', 'versionchanged', 'deprecated'] @@ -336,10 +338,16 @@ def parameter_mismatches(self): @property def pep8_violations(self): - with self._file_representation() as filename: - style_guide = flake8.get_style_guide(doctests=True) - report = style_guide.input_file(filename=filename) - return report.get_statistics('') + with self._file_representation() as file: + application = Flake8() + application.initialize(["--doctests", "--quiet"]) + application.run_checks([file.name]) + application.report() + stats = application.guide.stats + return [ + "{} {} {}".format(s.count, s.error_code, s.message) + for s in stats.statistics_for('') + ] @contextmanager def _file_representation(self): @@ -347,29 +355,21 @@ def _file_representation(self): Temporarily creates file with current function inside. The signature and body are **not** included. - :returns filename of tmp file + :returns file """ create_function = 'def {name}():\n' \ ' """{doc}"""\n' \ ' pass\n' - tmp_dir = os.path.join(BASE_PATH, 'build', 'validate_docstring') - os.makedirs(tmp_dir, exist_ok=True) - - filename = os.path.join(tmp_dir, self.name + '.py') - with open(filename, 'w') as f: - name = self.name.split('.')[-1] - lines = self.clean_doc.split("\n") - indented_lines = [(' ' * 4) + line if line else '' - for line in lines[1:]] - doc = '\n'.join([lines[0], *indented_lines]) - - f.write(create_function.format(name=name, doc=doc)) - try: - yield filename - finally: - os.remove(filename) - os.rmdir(tmp_dir) + name = self.name.split('.')[-1] + lines = self.clean_doc.split("\n") + indented_lines = [(' ' * 4) + line if line else '' + for line in lines[1:]] + doc = '\n'.join([lines[0], *indented_lines]) + with tempfile.NamedTemporaryFile(mode='w', suffix='.py') as file: + file.write(create_function.format(name=name, doc=doc)) + file.flush() + yield file @property def correct_parameters(self): @@ -532,7 +532,7 @@ def validate_one(func_name): pep8_errs = doc.pep8_violations if pep8_errs: - errs.append('Errors in doctest sections') + errs.append('Errors in doctests') for pep8_err in pep8_errs: errs.append('\t{}'.format(pep8_err)) diff --git a/setup.cfg b/setup.cfg index 34597947f69bc..edd3b507cb183 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,28 +28,6 @@ exclude = doc/temp/*.py, .eggs/*.py, versioneer.py - .tox - .git - -doctests = True -#TODO fix doctests -exclude_from_doctest = - ./pandas/_libs - ./pandas/api - ./pandas/compat - ./pandas/core - ./pandas/errors - ./pandas/io - ./pandas/plotting - ./pandas/tests - ./pandas/tseries - ./pandas/util - -[flake8-rst] -ignore = - F821, # undefined name - W391, # blank line at end of file [Seems to be a bug (v0.4.1)] - [yapf] based_on_style = pep8 From 6090b2493f83e25ca799291bda66db08275ad283 Mon Sep 17 00:00:00 2001 From: Fabian Haase Date: Fri, 2 Nov 2018 21:09:45 +0100 Subject: [PATCH 06/10] Fixing review Signed-off-by: Fabian Haase --- scripts/tests/test_validate_docstrings.py | 6 +-- scripts/validate_docstrings.py | 55 +++++++++-------------- 2 files changed, 23 insertions(+), 38 deletions(-) diff --git a/scripts/tests/test_validate_docstrings.py b/scripts/tests/test_validate_docstrings.py index 2047cf9d5f8ff..b377118a972b8 100644 --- a/scripts/tests/test_validate_docstrings.py +++ b/scripts/tests/test_validate_docstrings.py @@ -165,8 +165,8 @@ def contains(self, pat, case=True, na=np.nan): Examples -------- - >>> import pandas as pd >>> import numpy as np + >>> import pandas as pd >>> s = pd.Series(['Antelope', 'Lion', 'Zebra', np.nan]) >>> s.str.contains(pat='a') 0 False @@ -589,7 +589,7 @@ def prefix_pandas(self): class BadExamples(object): - def doctest(self): + def missing_numpy_import(self): """ Return whether each value contains `pat`. @@ -731,7 +731,7 @@ def test_bad_generic_functions(self, func): ('pandas.Series.rename in `See Also` section ' 'does not need `pandas` prefix',)), # Examples tests - ('BadExamples', 'doctest', + ('BadExamples', 'missing_numpy_import', ('1 F821 undefined name \'np\'',)) ]) def test_bad_examples(self, capsys, klass, func, msgs): diff --git a/scripts/validate_docstrings.py b/scripts/validate_docstrings.py index db2b4fc25902e..5ef280e10de68 100755 --- a/scripts/validate_docstrings.py +++ b/scripts/validate_docstrings.py @@ -25,9 +25,8 @@ import importlib import doctest import tempfile -from contextlib import contextmanager -from flake8.main.application import Application as Flake8 +from flake8.main import application as flake8 try: from io import StringIO @@ -336,40 +335,25 @@ def parameter_mismatches(self): return errs - @property - def pep8_violations(self): - with self._file_representation() as file: - application = Flake8() - application.initialize(["--doctests", "--quiet"]) - application.run_checks([file.name]) - application.report() - stats = application.guide.stats - return [ - "{} {} {}".format(s.count, s.error_code, s.message) - for s in stats.statistics_for('') - ] - - @contextmanager - def _file_representation(self): - """ - Temporarily creates file with current function inside. - The signature and body are **not** included. - - :returns file - """ + def validate_pep8(self): create_function = 'def {name}():\n' \ - ' """{doc}"""\n' \ + ' """\n{examples}\n """\n' \ ' pass\n' name = self.name.split('.')[-1] - lines = self.clean_doc.split("\n") - indented_lines = [(' ' * 4) + line if line else '' - for line in lines[1:]] - doc = '\n'.join([lines[0], *indented_lines]) - with tempfile.NamedTemporaryFile(mode='w', suffix='.py') as file: - file.write(create_function.format(name=name, doc=doc)) + lines = (' ' * 4 + line if line else '' for line in self.examples) + examples = '\n'.join(lines) + with tempfile.NamedTemporaryFile(mode='w') as file: + func = create_function.format(name=name, examples=examples) + file.write(func) file.flush() - yield file + + application = flake8.Application() + application.initialize(["--doctests", "--quiet"]) + application.run_checks([file.name]) + application.report() + + yield from application.guide.stats.statistics_for('') @property def correct_parameters(self): @@ -486,7 +470,7 @@ def validate_one(func_name): if doc.summary != doc.summary.lstrip(): errs.append('Summary contains heading whitespaces.') elif (doc.is_function_or_method - and doc.summary.split(' ')[0][-1] == 's'): + and doc.summary.split(' ')[0][-1] == 's'): errs.append('Summary must start with infinitive verb, ' 'not third person (e.g. use "Generate" instead of ' '"Generates")') @@ -530,11 +514,12 @@ def validate_one(func_name): for param_err in param_errs: errs.append('\t{}'.format(param_err)) - pep8_errs = doc.pep8_violations + pep8_errs = [error for error in doc.validate_pep8()] if pep8_errs: errs.append('Errors in doctests') - for pep8_err in pep8_errs: - errs.append('\t{}'.format(pep8_err)) + for err in pep8_errs: + errs.append('\t{} {} {}'.format(err.count, err.error_code, + err.message)) if doc.is_function_or_method: if not doc.returns and "return" in doc.method_source: From 72f99bd426eb72767f7c7d1fdd24b5d472e23f25 Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Sat, 3 Nov 2018 17:45:35 +0100 Subject: [PATCH 07/10] Rephrasing to Linting issues Co-Authored-By: FHaase --- scripts/validate_docstrings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/validate_docstrings.py b/scripts/validate_docstrings.py index 5ef280e10de68..7c8c7b5c65166 100755 --- a/scripts/validate_docstrings.py +++ b/scripts/validate_docstrings.py @@ -516,7 +516,7 @@ def validate_one(func_name): pep8_errs = [error for error in doc.validate_pep8()] if pep8_errs: - errs.append('Errors in doctests') + errs.append('Linting issues in doctests:') for err in pep8_errs: errs.append('\t{} {} {}'.format(err.count, err.error_code, err.message)) From affd8f4747a3eded274106a5e23bf45f9302cd38 Mon Sep 17 00:00:00 2001 From: Fabian Haase Date: Sat, 3 Nov 2018 20:43:06 +0100 Subject: [PATCH 08/10] [skip ci] Simplified validate_pep8 Signed-off-by: Fabian Haase --- scripts/tests/test_validate_docstrings.py | 65 ++++++++++++++++------- scripts/validate_docstrings.py | 44 ++++++++------- 2 files changed, 73 insertions(+), 36 deletions(-) diff --git a/scripts/tests/test_validate_docstrings.py b/scripts/tests/test_validate_docstrings.py index b377118a972b8..fa75d6e556744 100644 --- a/scripts/tests/test_validate_docstrings.py +++ b/scripts/tests/test_validate_docstrings.py @@ -127,7 +127,6 @@ def head1(self, n=5): Examples -------- - >>> import pandas as pd >>> s = pd.Series(['Ant', 'Bear', 'Cow', 'Dog', 'Falcon']) >>> s.head() 0 Ant @@ -165,8 +164,6 @@ def contains(self, pat, case=True, na=np.nan): Examples -------- - >>> import numpy as np - >>> import pandas as pd >>> s = pd.Series(['Antelope', 'Lion', 'Zebra', np.nan]) >>> s.str.contains(pat='a') 0 False @@ -322,8 +319,6 @@ def method(self, foo=None, bar=None): Examples -------- - >>> import numpy as np - >>> import pandas as pd >>> df = pd.DataFrame(np.ones((3, 3)), ... columns=('a', 'b', 'c')) >>> df.all(1) @@ -589,23 +584,47 @@ def prefix_pandas(self): class BadExamples(object): - def missing_numpy_import(self): + def numpy_imported(self): """ - Return whether each value contains `pat`. + Examples + -------- + >>> import numpy as np + >>> df = pd.DataFrame(np.ones((3, 3)), columns=('a', 'b', 'c')) + """ + pass + def pandas_imported(self): + """ Examples -------- >>> import pandas as pd - >>> df = pd.DataFrame(np.ones((3, 3)), - ... columns=('a', 'b', 'c')) - >>> df.all(1) - 0 True - 1 True - 2 True - dtype: bool + >>> s = pd.Series(['Antelope', 'Lion', 'Zebra', np.nan]) + """ + pass - >>> df.all(bool_only=True) - Series([], dtype: bool) + def missing_whitespace_around_arithmetic_operator(self): + """ + Examples + -------- + >>> 2+5 + 7 + """ + pass + + def indentation_is_not_a_multiple_of_four(self): + """ + Examples + -------- + >>> if 2 + 5: + ... pass + """ + pass + + def missing_whitespace_after_comma(self): + """ + Examples + -------- + >>> df = pd.DataFrame(np.ones((3,3)),columns=('a','b', 'c')) """ pass @@ -731,8 +750,18 @@ def test_bad_generic_functions(self, func): ('pandas.Series.rename in `See Also` section ' 'does not need `pandas` prefix',)), # Examples tests - ('BadExamples', 'missing_numpy_import', - ('1 F821 undefined name \'np\'',)) + ('BadExamples', 'numpy_imported', + ('F811 It\'s assumed that pandas and numpy' + ' are imported as pd or np',)), + ('BadExamples', 'pandas_imported', + ('F811 It\'s assumed that pandas and numpy' + ' are imported as pd or np',)), + ('BadExamples', 'indentation_is_not_a_multiple_of_four', + ('E111 indentation is not a multiple of four',)), + ('BadExamples', 'missing_whitespace_around_arithmetic_operator', + ('E226 missing whitespace around arithmetic operator',)), + ('BadExamples', 'missing_whitespace_after_comma', + ('3 E231 missing whitespace after \',\'',)), ]) def test_bad_examples(self, capsys, klass, func, msgs): result = validate_one(self._import_path(klass=klass, func=func)) diff --git a/scripts/validate_docstrings.py b/scripts/validate_docstrings.py index 7c8c7b5c65166..dd6fabb6bc904 100755 --- a/scripts/validate_docstrings.py +++ b/scripts/validate_docstrings.py @@ -13,6 +13,7 @@ $ ./validate_docstrings.py $ ./validate_docstrings.py pandas.DataFrame.head """ +import contextlib import os import sys import json @@ -26,7 +27,7 @@ import doctest import tempfile -from flake8.main import application as flake8 +import flake8.main.application try: from io import StringIO @@ -335,25 +336,32 @@ def parameter_mismatches(self): return errs - def validate_pep8(self): - create_function = 'def {name}():\n' \ - ' """\n{examples}\n """\n' \ - ' pass\n' - - name = self.name.split('.')[-1] - lines = (' ' * 4 + line if line else '' for line in self.examples) - examples = '\n'.join(lines) + @contextlib.contextmanager + def _write_examples_code_to_temp_file(self): + """ + Generate file with source code from examples section. + """ + content = ''.join(('import numpy as np; ' + 'import pandas as pd # noqa: F401,E702\n', + *self.examples_source_code)) with tempfile.NamedTemporaryFile(mode='w') as file: - func = create_function.format(name=name, examples=examples) - file.write(func) + file.write(content) file.flush() + yield file - application = flake8.Application() - application.initialize(["--doctests", "--quiet"]) - application.run_checks([file.name]) - application.report() - - yield from application.guide.stats.statistics_for('') + def validate_pep8(self): + if self.examples: + with self._write_examples_code_to_temp_file() as file: + application = flake8.main.application.Application() + application.initialize(["--quiet"]) + application.run_checks([file.name]) + application.report() + + for statistic in application.guide.stats.statistics_for(''): + if statistic.message.endswith('from line 1'): + statistic.message = "It's assumed that pandas and numpy" \ + " are imported as pd or np" + yield statistic @property def correct_parameters(self): @@ -514,7 +522,7 @@ def validate_one(func_name): for param_err in param_errs: errs.append('\t{}'.format(param_err)) - pep8_errs = [error for error in doc.validate_pep8()] + pep8_errs = list(doc.validate_pep8()) if pep8_errs: errs.append('Linting issues in doctests:') for err in pep8_errs: From 561f3c3f9d9f2f68e104ee2e6b61645ed6aa0009 Mon Sep 17 00:00:00 2001 From: Fabian Haase Date: Sun, 4 Nov 2018 15:15:00 +0100 Subject: [PATCH 09/10] [skip ci] Minor changes Signed-off-by: Fabian Haase --- scripts/tests/test_validate_docstrings.py | 25 ++++----------- scripts/validate_docstrings.py | 38 +++++++++-------------- 2 files changed, 21 insertions(+), 42 deletions(-) diff --git a/scripts/tests/test_validate_docstrings.py b/scripts/tests/test_validate_docstrings.py index fa75d6e556744..4a33a4f7979c4 100644 --- a/scripts/tests/test_validate_docstrings.py +++ b/scripts/tests/test_validate_docstrings.py @@ -584,24 +584,15 @@ def prefix_pandas(self): class BadExamples(object): - def numpy_imported(self): + def unused_import(self): """ Examples -------- - >>> import numpy as np + >>> import pandas as pdf >>> df = pd.DataFrame(np.ones((3, 3)), columns=('a', 'b', 'c')) """ pass - def pandas_imported(self): - """ - Examples - -------- - >>> import pandas as pd - >>> s = pd.Series(['Antelope', 'Lion', 'Zebra', np.nan]) - """ - pass - def missing_whitespace_around_arithmetic_operator(self): """ Examples @@ -750,16 +741,12 @@ def test_bad_generic_functions(self, func): ('pandas.Series.rename in `See Also` section ' 'does not need `pandas` prefix',)), # Examples tests - ('BadExamples', 'numpy_imported', - ('F811 It\'s assumed that pandas and numpy' - ' are imported as pd or np',)), - ('BadExamples', 'pandas_imported', - ('F811 It\'s assumed that pandas and numpy' - ' are imported as pd or np',)), + ('BadExamples', 'unused_import', + ('1 F401 \'pandas as pdf\' imported but unused',)), ('BadExamples', 'indentation_is_not_a_multiple_of_four', - ('E111 indentation is not a multiple of four',)), + ('1 E111 indentation is not a multiple of four',)), ('BadExamples', 'missing_whitespace_around_arithmetic_operator', - ('E226 missing whitespace around arithmetic operator',)), + ('1 E226 missing whitespace around arithmetic operator',)), ('BadExamples', 'missing_whitespace_after_comma', ('3 E231 missing whitespace after \',\'',)), ]) diff --git a/scripts/validate_docstrings.py b/scripts/validate_docstrings.py index dd6fabb6bc904..165b5866ef480 100755 --- a/scripts/validate_docstrings.py +++ b/scripts/validate_docstrings.py @@ -13,7 +13,6 @@ $ ./validate_docstrings.py $ ./validate_docstrings.py pandas.DataFrame.head """ -import contextlib import os import sys import json @@ -173,7 +172,7 @@ def _load_obj(name): @staticmethod def _to_original_callable(obj): """ - Find the Python object that contains the source code ot the object. + Find the Python object that contains the source code of the object. This is useful to find the place in the source code (file and line number) where a docstring is defined. It does not currently work for @@ -336,32 +335,25 @@ def parameter_mismatches(self): return errs - @contextlib.contextmanager - def _write_examples_code_to_temp_file(self): - """ - Generate file with source code from examples section. - """ - content = ''.join(('import numpy as np; ' - 'import pandas as pd # noqa: F401,E702\n', + def validate_pep8(self): + if not self.examples: + return + + content = ''.join(('import numpy as np # noqa: F401\n', + 'import pandas as pd # noqa: F401\n', *self.examples_source_code)) + + application = flake8.main.application.Application() + application.initialize(["--quiet"]) + with tempfile.NamedTemporaryFile(mode='w') as file: file.write(content) file.flush() - yield file + application.run_checks([file.name]) - def validate_pep8(self): - if self.examples: - with self._write_examples_code_to_temp_file() as file: - application = flake8.main.application.Application() - application.initialize(["--quiet"]) - application.run_checks([file.name]) - application.report() - - for statistic in application.guide.stats.statistics_for(''): - if statistic.message.endswith('from line 1'): - statistic.message = "It's assumed that pandas and numpy" \ - " are imported as pd or np" - yield statistic + application.report() + + yield from application.guide.stats.statistics_for('') @property def correct_parameters(self): From 1178caef75e8e0386703ce819ca9bc854be126c9 Mon Sep 17 00:00:00 2001 From: Fabian Haase Date: Sun, 4 Nov 2018 16:37:51 +0100 Subject: [PATCH 10/10] Fix good_imports test Signed-off-by: Fabian Haase --- scripts/tests/test_validate_docstrings.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/tests/test_validate_docstrings.py b/scripts/tests/test_validate_docstrings.py index d78a85b1942e2..a3feee6552178 100644 --- a/scripts/tests/test_validate_docstrings.py +++ b/scripts/tests/test_validate_docstrings.py @@ -225,8 +225,9 @@ def good_imports(self): Examples -------- This example does not import pandas or import numpy. - >>> import time >>> import datetime + >>> datetime.MAXYEAR + 9999 """ pass @@ -672,7 +673,7 @@ def test_good_class(self): @capture_stderr @pytest.mark.parametrize("func", [ 'plot', 'sample', 'random_letters', 'sample_values', 'head', 'head1', - 'contains', 'mode']) + 'contains', 'mode', 'good_imports']) def test_good_functions(self, func): errors = validate_one(self._import_path( klass='GoodDocStrings', func=func))['errors']