feat: Support importlib.resources in edittable installs

.github/workflows/ci.yml

@@ -73,6 +73,9 @@ jobs:
           - python-version: "3.9"
             runs-on: ubuntu-latest
             cmake-version: "3.20.x"
+          - python-version: "3.9"
+            runs-on: macos-latest
+            cmake-version: "3.18.x"
           - python-version: "3.10"
             runs-on: ubuntu-latest
             cmake-version: "3.22.x"

pyproject.toml

@@ -271,9 +271,10 @@ exclude = []
 [tool.ruff.per-file-ignores]
 "tests/**" = ["T20"]
 "noxfile.py" = ["T20", "TID251"]
-"src/scikit_build_core/resources/*.py" = ["PTH", "ARG002", "FBT"]
+"src/scikit_build_core/resources/*.py" = ["PTH", "ARG002", "FBT", "TID251"]
 "src/scikit_build_core/_compat/**.py" = ["TID251"]
 "tests/conftest.py" = ["TID251"]
+"tests/packages/**.py" = ["TID251"]
 "docs/conf.py" = ["TID251"]
 
 

src/scikit_build_core/resources/_editable_redirect.py

@@ -1,12 +1,27 @@
 from __future__ import annotations
 
 import importlib.abc
-import importlib.machinery
 import importlib.util
 import os
 import subprocess
 import sys
 
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    import importlib.machinery
+
+    if sys.version_info < (3, 8):
+        from typing_extensions import TypedDict
+    else:
+        from typing import TypedDict
+
+    class KWDict_1(TypedDict, total=False):
+        submodule_search_locations: list[str]
+
+else:
+    KWDict_1 = dict
+
+
 DIR = os.path.abspath(os.path.dirname(__file__))
 MARKER = "SKBUILD_EDITABLE_SKIP"
 VERBOSE = "SKBUILD_EDITABLE_VERBOSE"
@@ -36,24 +51,65 @@ def __init__(
         self.verbose = verbose
         self.build_options = build_options
         self.install_options = install_options
+        # Construct the __path__ of all resource files
+        # I.e. the paths of all package-like objects
+        submodule_search_locations: dict[str, set[str]] = {}
+        pkgs: list[str] = []
+        # Loop over both python native source files and cmake installed ones
+        for tree in (known_source_files, known_wheel_files):
+            for module, file in tree.items():
+                # Strip the last element of the module
+                parent = ".".join(module.split(".")[:-1])
+                # Check if it is a package
+                if "__init__.py" in file:
+                    parent = module
+                    pkgs.append(parent)
+                # Skip if it's a root module (there are no search paths for these)
+                if not parent:
+                    continue
+                # Initialize the tree element if needed
+                submodule_search_locations.setdefault(parent, set())
+                # Add the parent path to the dictionary values
+                parent_path, _ = os.path.split(file)
+                if not parent_path:
+                    # root modules are skipped so all files should be in a parent package
+                    msg = f"Unexpected path to source file: {file} [{module}]"
+                    raise ImportError(msg)
+                if not os.path.isabs(parent_path):
+                    parent_path = os.path.join(str(DIR), parent_path)
+                submodule_search_locations[parent].add(parent_path)
+        self.submodule_search_locations = submodule_search_locations
+        self.pkgs = pkgs
 
     def find_spec(
         self,
         fullname: str,
         path: object = None,
         target: object = None,
     ) -> importlib.machinery.ModuleSpec | None:
+        # If current item is a know package use its search locations, otherwise if it's a module use the parent's
+        parent = (
+            fullname if fullname in self.pkgs else ".".join(fullname.split(".")[:-1])
+        )
+        # If no known submodule_search_locations is found, it means it is a root module. Do not populate its kwargs
+        # in that case
+        kwargs = KWDict_1()
+        if parent in self.submodule_search_locations:
+            kwargs["submodule_search_locations"] = list(
+                self.submodule_search_locations[parent]
+            )
         if fullname in self.known_wheel_files:
             redir = self.known_wheel_files[fullname]
             if self.rebuild_flag:
                 self.rebuild()
             return importlib.util.spec_from_file_location(
-                fullname, os.path.join(DIR, redir)
+                fullname,
+                os.path.join(DIR, redir),
+                **kwargs,
             )
         if fullname in self.known_source_files:
             redir = self.known_source_files[fullname]
-            return importlib.util.spec_from_file_location(fullname, redir)
-
+            return importlib.util.spec_from_file_location(fullname, redir, **kwargs)
         return None
 
     def rebuild(self) -> None:

tests/conftest.py

@@ -92,6 +92,12 @@ def __init__(self, env_dir: Path, *, wheelhouse: Path | None = None) -> None:
         self.wheelhouse = wheelhouse
         self.executable = Path(result.creator.exe)
         self.env_dir = env_dir.resolve()
+        self.platlib = Path(
+            self.execute("import sysconfig; print(sysconfig.get_path('platlib'))")
+        )
+        self.purelib = Path(
+            self.execute("import sysconfig; print(sysconfig.get_path('purelib'))")
+        )
 
     @overload
     def run(self, *args: str, capture: Literal[True]) -> str:
@@ -279,6 +285,15 @@ def package_simplest_c(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Packa
     return package
 
 
+@pytest.fixture()
+def navigate_editable(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> PackageInfo:
+    package = PackageInfo(
+        "navigate_editable",
+    )
+    process_package(package, tmp_path, monkeypatch)
+    return package
+
+
 @pytest.fixture()
 def package_sdist_config(
     tmp_path: Path, monkeypatch: pytest.MonkeyPatch

tests/packages/navigate_editable/CMakeLists.txt

@@ -0,0 +1,21 @@
+cmake_minimum_required(VERSION 3.15...3.26)
+
+project(
+  ${SKBUILD_PROJECT_NAME}
+  LANGUAGES C
+  VERSION ${SKBUILD_PROJECT_VERSION})
+
+find_package(Python COMPONENTS Interpreter Development.Module)
+
+python_add_library(c_module MODULE src/shared_pkg/c_module.c WITH_SOABI)
+
+set(CMakeVar "Some_value_C")
+configure_file(src/shared_pkg/data/generated.txt.in
+               shared_pkg/data/c_generated.txt)
+
+install(
+  TARGETS c_module
+  DESTINATION shared_pkg/
+  COMPONENT PythonModule)
+install(FILES ${PROJECT_BINARY_DIR}/shared_pkg/data/c_generated.txt
+        DESTINATION shared_pkg/data/)

tests/packages/navigate_editable/pyproject.toml

@@ -0,0 +1,13 @@
+[build-system]
+requires = ["scikit-build-core"]
+build-backend = "scikit_build_core.build"
+
+[project]
+name = "navigate_editable"
+version = "0.0.1"
+dependencies = [
+    "importlib-resources; python_version<'3.9'"
+]
+
+[tool.scikit-build]
+wheel.packages = ["python/shared_pkg"]

tests/packages/navigate_editable/python/shared_pkg/__init__.py

@@ -0,0 +1,9 @@
+from .c_module import call_py_method
+from .py_module import call_c_method, read_c_generated_txt, read_py_data_txt
+
+__all__ = [
+    "call_py_method",
+    "call_c_method",
+    "read_py_data_txt",
+    "read_c_generated_txt",
+]

tests/packages/navigate_editable/python/shared_pkg/c_module.pyi

@@ -0,0 +1,2 @@
+def c_method() -> str: ...
+def call_py_method() -> None: ...

tests/packages/navigate_editable/python/shared_pkg/data/py_data.txt

@@ -0,0 +1 @@
+Some_value_Py

tests/packages/navigate_editable/python/shared_pkg/py_module.py

@@ -0,0 +1,28 @@
+import sys
+
+if sys.version_info < (3, 9):
+    from importlib_resources import files
+else:
+    from importlib.resources import files
+
+from .c_module import c_method
+
+
+def call_c_method():
+    print(c_method())
+
+
+def py_method():
+    print("py_method")
+
+
+def read_py_data_txt():
+    root = files("shared_pkg.data")
+    py_data = root / "py_data.txt"
+    print(py_data.read_text())
+
+
+def read_c_generated_txt():
+    root = files("shared_pkg.data")
+    c_generated_txt = root / "c_generated.txt"
+    print(c_generated_txt.read_text())

tests/packages/navigate_editable/src/shared_pkg/c_module.c

@@ -0,0 +1,54 @@
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+const char* c_method() { return "c_method"; }
+
+static PyObject *c_method_wrapper(PyObject *self, PyObject *args) {
+  return PyUnicode_FromString(c_method());
+}
+
+static PyObject *py_method_wrapper(PyObject *self, PyObject *args) {
+  PyObject *py_module = PyImport_ImportModule("shared_pkg.py_module");
+  if (py_module == NULL) {
+    PyErr_Print();
+    fprintf(stderr, "Failed to load shared_pkg.py_module\n");
+    exit(1);
+  }
+  PyObject *py_method = PyObject_GetAttrString(py_module,(char*)"py_method");
+  if (py_method == NULL) {
+    PyErr_Print();
+    fprintf(stderr, "Failed to load shared_pkg.py_module.py_method\n");
+    exit(1);
+  }
+
+#if PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION > 8
+  PyObject *res = PyObject_CallNoArgs(py_method);
+#else
+  PyObject *res = PyObject_CallObject(py_method, NULL);
+#endif
+
+  if (res == NULL) {
+    PyErr_Print();
+    fprintf(stderr, "Failed to execute shared_pkg.py_module.py_method\n");
+    exit(1);
+  }
+
+  Py_DECREF(py_module);
+  Py_DECREF(py_method);
+  Py_DECREF(res);
+  Py_RETURN_NONE;
+}
+
+static PyMethodDef c_module_methods[] = {
+    {"c_method", c_method_wrapper, METH_NOARGS, "C native method"},
+    {"call_py_method", py_method_wrapper, METH_NOARGS, "Call python native method"},
+    {NULL, NULL, 0, NULL}};
+
+static struct PyModuleDef c_module = {PyModuleDef_HEAD_INIT, "c_module",
+                                             NULL, -1, c_module_methods};
+
+PyMODINIT_FUNC PyInit_c_module(void) {
+  return PyModule_Create(&c_module);
+}

tests/packages/navigate_editable/src/shared_pkg/data/generated.txt.in

@@ -0,0 +1 @@
+@CMakeVar@

tests/test_editable.py

@@ -0,0 +1,50 @@
+import sys
+from pathlib import Path
+
+import pytest
+
+
+@pytest.mark.compile()
+@pytest.mark.configure()
+@pytest.mark.integration()
+@pytest.mark.parametrize("isolate", [True, False], ids=["isolated", "notisolated"])
+@pytest.mark.parametrize(
+    "package",
+    [
+        pytest.param(
+            True,
+            id="package",
+            marks=[pytest.mark.xfail(reason="Only data folders supported currently")],
+        ),
+        pytest.param(False, id="datafolder"),
+    ],
+)
+@pytest.mark.usefixtures("navigate_editable")
+@pytest.mark.xfail(
+    sys.version_info[:2] == (3, 9), reason="Python 3.9 not supported yet"
+)
+def test_navigate_editable(isolated, isolate, package):
+    isolate_args = ["--no-build-isolation"] if not isolate else []
+    isolated.install("pip>=23")
+    if not isolate:
+        isolated.install("scikit-build-core[pyproject]")
+
+    if package:
+        init_py = Path("python/shared_pkg/data/__init__.py")
+        init_py.touch()
+
+    isolated.install(
+        "-v", "--config-settings=build-dir=build/{wheel_tag}", *isolate_args, "-e", "."
+    )
+
+    value = isolated.execute("import shared_pkg; shared_pkg.call_c_method()")
+    assert value == "c_method"
+
+    value = isolated.execute("import shared_pkg; shared_pkg.call_py_method()")
+    assert value == "py_method"
+
+    value = isolated.execute("import shared_pkg; shared_pkg.read_py_data_txt()")
+    assert value == "Some_value_Py"
+
+    value = isolated.execute("import shared_pkg; shared_pkg.read_c_generated_txt()")
+    assert value == "Some_value_C"

tests/test_pyproject_pep660.py

@@ -1,4 +1,5 @@
 import sys
+import sysconfig
 import zipfile
 from pathlib import Path
 
@@ -57,8 +58,40 @@ def test_pep660_pip_isolated(isolated, isolate):
     value = isolated.execute("import simplest; print(simplest.square(2))")
     assert value == "4.0"
 
-    location = isolated.execute("import simplest; print(*simplest.__path__)")
-    assert location == str(Path.cwd() / "src/simplest")
+    location_str = isolated.execute(
+        "import simplest; print(*simplest.__path__, sep=';')"
+    )
+    locations = [Path(s).resolve() for s in location_str.split(";")]
+
+    # First path is from the python source
+    python_source = Path("src/simplest").resolve()
+    assert any(x.samefile(python_source) for x in locations)
+
+    # Second path is from the CMake install
+    cmake_install = isolated.platlib.joinpath("simplest").resolve()
+    assert any(x.samefile(cmake_install) for x in locations)
 
     location = isolated.execute("import simplest; print(simplest.__file__)")
-    assert location == str(Path.cwd() / "src/simplest/__init__.py")
+    # The package file is defined in the python source and __file__ must point to it
+    assert Path("src/simplest/__init__.py").resolve().samefile(Path(location).resolve())
+
+    location = isolated.execute(
+        "import simplest._module; print(simplest._module.__file__)"
+    )
+    if sys.version_info < (3, 8, 7):
+        import distutils.sysconfig  # pylint: disable=deprecated-module
+
+        ext_suffix = distutils.sysconfig.get_config_var("EXT_SUFFIX")
+    else:
+        ext_suffix = sysconfig.get_config_var("EXT_SUFFIX")
+
+    module_file = cmake_install / f"_module{ext_suffix}"
+    # Windows FindPython may produce the wrong extension
+    if (
+        sys.version_info < (3, 8, 7)
+        and sys.platform.startswith("win")
+        and not module_file.is_file()
+    ):
+        module_file = cmake_install / "_module.pyd"
+
+    assert module_file.samefile(Path(location).resolve())