Implement new importlib.resources API (closes #977):

All runtime tests passing on arm64

demo/app/build.gradle.kts

@@ -47,7 +47,9 @@ android {
                       verParsed[2] * 10
 
         ndk {
-            abiFilters += listOf("armeabi-v7a", "arm64-v8a", "x86", "x86_64")
+            abiFilters += listOf(
+                "arm64-v8a", "armeabi-v7a", "x86", "x86_64"
+            )
         }
 
         // Remove other languages imported from Android support libraries.
@@ -101,8 +103,10 @@ chaquopy {
 
         // Python unit tests
         pip {
-            // In newer versions, importing murmurhash automatically imports and
-            // extracts murmurhash/mrmr.so, which would complicate the tests.
+            // We use an old version of murmurhash (built from the Chaquopy branch
+            // `murmurhash-0`), because in newer versions, importing murmurhash
+            // automatically imports and extracts murmurhash/mrmr.so, which would
+            // complicate the tests.
             install("murmurhash==0.28.0")  // Requires chaquopy-libcxx
 
             // Because we set pyc.src to false, we must test extractPackages via pip.

product/runtime/src/main/python/java/android/importer.py

@@ -6,8 +6,10 @@
 from inspect import getmodulename
 import io
 import os
-from os.path import basename, dirname, exists, join, normpath, realpath, relpath, split, splitext
-import pathlib
+from os.path import (
+    basename, dirname, exists, isfile, join, normpath, realpath, relpath, split, splitext
+)
+from pathlib import Path
 from pkgutil import get_importer
 import re
 from shutil import copyfileobj, rmtree
@@ -280,6 +282,15 @@ def __init__(self, module):
             super().__init__(module)
             self.finder = self.loader.finder
 
+        # pkg_resources has a mechanism for extracting resources to temporary files, but
+        # we don't currently support it. So this will only work for files which are
+        # already extracted.
+        def get_resource_filename(self, manager, resource_name):
+            path = self._fn(self.module_path, resource_name)
+            if not self._has(path):
+                raise FileNotFoundError(path)
+            return path
+
         def _has(self, path):
             return self.finder.exists(self.finder.zip_path(path))
 
@@ -331,7 +342,7 @@ def find_distributions(cls, context=None):
         pattern = fr"^{name}(-.*)?\.(dist|egg)-info$"
 
         for entry in context.path:
-            path_cls = AssetPath if entry.startswith(ASSET_PREFIX + "/") else pathlib.Path
+            path_cls = AssetPath if entry.startswith(ASSET_PREFIX + "/") else Path
             entry_path = path_cls(entry)
             try:
                 if entry_path.is_dir():
@@ -342,30 +353,60 @@ def find_distributions(cls, context=None):
                 pass  # Inaccessible path entries should be ignored.
 
 
-class AssetPath(pathlib.PosixPath):
-
-    # Derived path objects (e.g. from `joinpath` or `/`) are created using object.__new__,
-    # so we can't initialize them by overriding __new__ or __init__.
-    @property
-    def finder(self):
-        root_dir = str(self)
+# This does not inherit from PosixPath, because that would cause
+# importlib.resources.as_file to return it unchanged, rather than creating a temporary
+# file as it should. However, once our minimum version is Python 3.9, we can inherit
+# from importlib.resources.abc.Traversable, and remove our implementations of read_text,
+# read_bytes, and __truediv__.
+class AssetPath:
+    def __init__(self, path):
+        root_dir = path
         while dirname(root_dir) != ASSET_PREFIX:
             root_dir = dirname(root_dir)
-            assert root_dir, str(self)
-        return get_importer(root_dir)
+            assert root_dir, path
+        self.finder = get_importer(root_dir)
+        self.zip_path = self.finder.zip_path(path)
+
+    def __str__(self):
+        return join(self.finder.extract_root, self.zip_path)
+
+    def __repr__(self):
+        return f"{type(self).__name__}({str(self)!r})"
+
+    def __eq__(self, other):
+        return (type(self) is type(other)) and (str(self) == str(other))
+
+    def __hash(self):
+        return hash(str(self))
 
     @property
-    def zip_path(self):
-        return self.finder.zip_path(str(self))
+    def name(self):
+        return basename(str(self))
+
+    def exists(self):
+        return self.finder.exists(self.zip_path)
 
     def is_dir(self):
         return self.finder.isdir(self.zip_path)
 
+    def is_file(self):
+        return self.exists() and not self.is_dir()
+
     def iterdir(self):
         for name in self.finder.listdir(self.zip_path):
-            yield AssetPath(join(str(self),  name))
+            yield self.joinpath(name)
+
+    def joinpath(self, *segments):
+        child_path = join(str(self), *segments)
+        if isfile(child_path):
+            return Path(child_path)  # For data files created by extract_dir.
+        else:
+            return type(self)(child_path)
+
+    def __truediv__(self, child):
+        return self.joinpath(child)
 
-    def open(self, mode="r", buffering=-1, **kwargs):
+    def open(self, mode="r", buffering="ignored", **kwargs):
         if "r" in mode:
             bio = io.BytesIO(self.finder.get_data(self.zip_path))
             if mode == "r":
@@ -374,9 +415,16 @@ def open(self, mode="r", buffering=-1, **kwargs):
                 return bio
         raise ValueError(f"unsupported mode: {mode!r}")
 
+    def read_bytes(self):
+        with self.open('rb') as strm:
+            return strm.read()
+
+    def read_text(self, encoding=None):
+        with self.open(encoding=encoding) as strm:
+            return strm.read()
 
-class AssetFinder:
 
+class AssetFinder:
     def __init__(self, context, build_json, path):
         if not path.startswith(ASSET_PREFIX + "/"):
             raise ImportError(f"not an asset path: '{path}'")
@@ -526,6 +574,8 @@ def listdir(self, zip_path):
 
     def get_data(self, zip_path):
         for zf in self.zip_files:
+            if zf.isdir(zip_path):
+                raise IsADirectoryError(zip_path)
             try:
                 return zf.read(zip_path)
             except KeyError:
@@ -542,7 +592,8 @@ def zip_path(self, path):
         return path[len(self.extract_root) + 1:]
 
 
-# To create a concrete loader class, inherit this class followed by a FileLoader subclass.
+# To create a concrete loader class, inherit this class followed by a FileLoader
+# subclass, in that order.
 class AssetLoader:
     def __init__(self, finder, fullname, zip_info):
         self.finder = finder
@@ -552,7 +603,8 @@ def __init__(self, finder, fullname, zip_info):
     def __repr__(self):
         return f"{type(self).__name__}({self.name!r}, {self.path!r})"
 
-    # Override to disable the fullname check. This is necessary for module renaming via imp.
+    # Override to disable the fullname check. This is only necessary for module renaming
+    # via `imp`, so it can be removed one our minimum version is Python 3.12.
     def get_filename(self, fullname):
         return self.path
 
@@ -568,31 +620,53 @@ def exec_module(self, mod):
         super().exec_module(mod)
         exec_module_trigger(mod)
 
-    def get_resource_reader(self, mod_name):
-        return self if self.is_package(mod_name) else None
+    # The importlib.resources.abc documentation says "If the module specified by
+    # fullname is not a package, this method should return None", but that's no longer
+    # true as of Python 3.12, because importlib.resources.files can accept a module as
+    # well as a package.
+    def get_resource_reader(self, fullname):
+        assert fullname == self.name, (fullname, self.name)
+        return AssetResourceReader(self.finder, dirname(self.path))
 
-    def open_resource(self, name):
-        return io.BytesIO(self.get_data(self.res_abs_path(name)))
 
-    def resource_path(self, name):
-        path = self.res_abs_path(name)
-        if exists(path):
-            # For __pycache__ directories created by SourceAssetLoader, and data files created
-            # by extract_dir.
-            return path
+class AssetResourceReader:
+    def __init__(self, finder, path):
+        assert finder.isdir(finder.zip_path(path)), path
+        self.asset_path = AssetPath(path)
+
+    def __repr__(self):
+        return f"<{type(self).__name__} {str(self.asset_path)!r}>"
+
+    # Implementation of importlib.resources.abc.TraversableResources
+    def files(self):
+        return self.asset_path
+
+    # The remaining methods are an implementation of
+    # importlib.resources.abc.ResourceReader. In Python 3.11, the old
+    # importlib.resources API is entirely implemented in terms of the new API, so once
+    # that's our minimum version, we can remove these methods and inherit them from
+    # TraversableResources instead.
+
+    def open_resource(self, resource):
+        return self.files().joinpath(resource).open('rb')
+
+    def resource_path(self, resource):
+        path = self.files().joinpath(resource)
+        if isinstance(path, Path):
+            return path  # For data files created by extract_dir.
         else:
             # importlib.resources.path will call open_resource and create a temporary file.
             raise FileNotFoundError()
 
+    # The documentation says this should raise FileNotFoundError if the name doesn't
+    # exist, but that would cause inconsistent behavior of the public is_resource
+    # function, which forwards directly to this method before Python 3.11, but uses
+    # files().iterdir() after Python 3.11.
     def is_resource(self, name):
-        zip_path = self.finder.zip_path(self.res_abs_path(name))
-        return self.finder.exists(zip_path) and not self.finder.isdir(zip_path)
+        return self.files().joinpath(name).is_file()
 
     def contents(self):
-        return self.finder.listdir(self.finder.zip_path(dirname(self.path)))
-
-    def res_abs_path(self, name):
-        return join(dirname(self.path), name)
+        return (item.name for item in self.files().iterdir())
 
 
 def add_import_trigger(name, trigger):