From c4acc7070c8271b6afc861c0847cc09e27fcdd84 Mon Sep 17 00:00:00 2001 From: Joost Ellerbroek Date: Thu, 20 Oct 2022 21:29:04 +0200 Subject: [PATCH 1/4] Add test for MultiplexedPath.joinpath with common subdirs --- .../tests/data02/subdirectory/subsubdir/resource.txt | 1 + importlib_resources/tests/test_reader.py | 11 +++++++++++ 2 files changed, 12 insertions(+) create mode 100644 importlib_resources/tests/data02/subdirectory/subsubdir/resource.txt diff --git a/importlib_resources/tests/data02/subdirectory/subsubdir/resource.txt b/importlib_resources/tests/data02/subdirectory/subsubdir/resource.txt new file mode 100644 index 00000000..48f587a2 --- /dev/null +++ b/importlib_resources/tests/data02/subdirectory/subsubdir/resource.txt @@ -0,0 +1 @@ +a resource \ No newline at end of file diff --git a/importlib_resources/tests/test_reader.py b/importlib_resources/tests/test_reader.py index 1c8ebeeb..e2bdf19c 100644 --- a/importlib_resources/tests/test_reader.py +++ b/importlib_resources/tests/test_reader.py @@ -81,6 +81,17 @@ def test_join_path_compound(self): path = MultiplexedPath(self.folder) assert not path.joinpath('imaginary/foo.py').exists() + def test_join_path_common_subdir(self): + prefix = os.path.abspath(os.path.join(__file__, '..')) + data01 = os.path.join(prefix, 'data01') + data02 = os.path.join(prefix, 'data02') + path = MultiplexedPath(data01, data02) + self.assertIsInstance(path.joinpath('subdirectory'), MultiplexedPath) + self.assertEqual( + str(path.joinpath('subdirectory', 'subsubdir'))[len(prefix) + 1 :], + os.path.join('data02', 'subdirectory', 'subsubdir'), + ) + def test_repr(self): self.assertEqual( repr(MultiplexedPath(self.folder)), From 35e7ebcecb27fb650a9568c06ead3fc43eba9e64 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 5 Feb 2023 13:49:47 -0500 Subject: [PATCH 2/4] In MultiplexedPath.iterdir, honor multiple subdirectories of the same name. --- importlib_resources/_itertools.py | 35 ------------------------------- importlib_resources/readers.py | 29 ++++++++++++++++++++++--- 2 files changed, 26 insertions(+), 38 deletions(-) delete mode 100644 importlib_resources/_itertools.py diff --git a/importlib_resources/_itertools.py b/importlib_resources/_itertools.py deleted file mode 100644 index cce05582..00000000 --- a/importlib_resources/_itertools.py +++ /dev/null @@ -1,35 +0,0 @@ -from itertools import filterfalse - -from typing import ( - Callable, - Iterable, - Iterator, - Optional, - Set, - TypeVar, - Union, -) - -# Type and type variable definitions -_T = TypeVar('_T') -_U = TypeVar('_U') - - -def unique_everseen( - iterable: Iterable[_T], key: Optional[Callable[[_T], _U]] = None -) -> Iterator[_T]: - "List unique elements, preserving order. Remember all elements ever seen." - # unique_everseen('AAAABBBCCDAABBB') --> A B C D - # unique_everseen('ABBCcAD', str.lower) --> A B C D - seen: Set[Union[_T, _U]] = set() - seen_add = seen.add - if key is None: - for element in filterfalse(seen.__contains__, iterable): - seen_add(element) - yield element - else: - for element in iterable: - k = key(element) - if k not in seen: - seen_add(k) - yield element diff --git a/importlib_resources/readers.py b/importlib_resources/readers.py index c5d435f4..ab4bbea7 100644 --- a/importlib_resources/readers.py +++ b/importlib_resources/readers.py @@ -1,10 +1,11 @@ import collections +import contextlib +import itertools import pathlib import operator from . import abc -from ._itertools import unique_everseen from ._compat import ZipPath @@ -69,8 +70,10 @@ def __init__(self, *paths): raise NotADirectoryError('MultiplexedPath only supports directories') def iterdir(self): - files = (file for path in self._paths for file in path.iterdir()) - return unique_everseen(files, key=operator.attrgetter('name')) + children = (child for path in self._paths for child in path.iterdir()) + by_name = operator.attrgetter('name') + groups = itertools.groupby(sorted(children, key=by_name), key=by_name) + return map(self._maybe, (locs for name, locs in groups)) def read_bytes(self): raise FileNotFoundError(f'{self} is not a file') @@ -92,6 +95,26 @@ def joinpath(self, *descendants): # Just return something that will not exist. return self._paths[0].joinpath(*descendants) + @classmethod + def _maybe(cls, subdirs): + """ + Construct a MultiplexedPath if needed. + + If subdirs contains a sole element, return it. + Otherwise, return a MultiplexedPath of the items. + """ + saved = list(subdirs) + + try: + result = cls(*saved) + except NotADirectoryError: + return saved[0] + + with contextlib.suppress(ValueError): + (result,) = result._paths + + return result + def open(self, *args, **kwargs): raise FileNotFoundError(f'{self} is not a file') From 438af1586ebb382a4ad3f583a5fcd0c519b2d93a Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 5 Feb 2023 17:21:23 -0500 Subject: [PATCH 3/4] Prefer tee and only. --- importlib_resources/_itertools.py | 38 +++++++++++++++++++++++++++++++ importlib_resources/readers.py | 25 ++++++++++---------- 2 files changed, 50 insertions(+), 13 deletions(-) create mode 100644 importlib_resources/_itertools.py diff --git a/importlib_resources/_itertools.py b/importlib_resources/_itertools.py new file mode 100644 index 00000000..7b775ef5 --- /dev/null +++ b/importlib_resources/_itertools.py @@ -0,0 +1,38 @@ +# from more_itertools 9.0 +def only(iterable, default=None, too_long=None): + """If *iterable* has only one item, return it. + If it has zero items, return *default*. + If it has more than one item, raise the exception given by *too_long*, + which is ``ValueError`` by default. + >>> only([], default='missing') + 'missing' + >>> only([1]) + 1 + >>> only([1, 2]) # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + ValueError: Expected exactly one item in iterable, but got 1, 2, + and perhaps more.' + >>> only([1, 2], too_long=TypeError) # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + TypeError + Note that :func:`only` attempts to advance *iterable* twice to ensure there + is only one item. See :func:`spy` or :func:`peekable` to check + iterable contents less destructively. + """ + it = iter(iterable) + first_value = next(it, default) + + try: + second_value = next(it) + except StopIteration: + pass + else: + msg = ( + 'Expected exactly one item in iterable, but got {!r}, {!r}, ' + 'and perhaps more.'.format(first_value, second_value) + ) + raise too_long or ValueError(msg) + + return first_value diff --git a/importlib_resources/readers.py b/importlib_resources/readers.py index ab4bbea7..51d030a6 100644 --- a/importlib_resources/readers.py +++ b/importlib_resources/readers.py @@ -1,11 +1,11 @@ import collections -import contextlib import itertools import pathlib import operator from . import abc +from ._itertools import only from ._compat import ZipPath @@ -73,7 +73,7 @@ def iterdir(self): children = (child for path in self._paths for child in path.iterdir()) by_name = operator.attrgetter('name') groups = itertools.groupby(sorted(children, key=by_name), key=by_name) - return map(self._maybe, (locs for name, locs in groups)) + return map(self._follow, (locs for name, locs in groups)) def read_bytes(self): raise FileNotFoundError(f'{self} is not a file') @@ -96,24 +96,23 @@ def joinpath(self, *descendants): return self._paths[0].joinpath(*descendants) @classmethod - def _maybe(cls, subdirs): + def _follow(cls, children): """ Construct a MultiplexedPath if needed. - If subdirs contains a sole element, return it. + If children contains a sole element, return it. Otherwise, return a MultiplexedPath of the items. + Unless one of the items is not a Directory, then return the first. """ - saved = list(subdirs) + subdirs, one_dir, one_file = itertools.tee(children, 3) try: - result = cls(*saved) - except NotADirectoryError: - return saved[0] - - with contextlib.suppress(ValueError): - (result,) = result._paths - - return result + return only(one_dir) + except ValueError: + try: + return cls(*subdirs) + except NotADirectoryError: + return next(one_file) def open(self, *args, **kwargs): raise FileNotFoundError(f'{self} is not a file') From 7fb0260f7c7dd068be176207de8b5a875179c221 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 5 Feb 2023 20:16:51 -0500 Subject: [PATCH 4/4] Update changelog. --- CHANGES.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index c688b1c6..6d76f67a 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,9 @@ +v5.11.0 +======= + +* #265: ``MultiplexedPath`` now honors multiple subdirectories + in ``iterdir`` and ``joinpath``. + v5.10.2 =======