gh-102615: Fix type vars substitution of collections.abc.Callable and custom generics with ParamSpec

Lib/_collections_abc.py

@@ -498,6 +498,19 @@ def __getitem__(self, item):
             t_result = new_args[-1]
             t_args = new_args[:-1]
             new_args = (t_args, t_result)
+
+        # This happens in cases like `Callable[P, T][[P, str], bool][int]`,
+        # we need to flatten the result.
+        if (len(new_args) > 2
+                and self.__parameters__
+                and _is_param_expr(self.__parameters__[0])):
+            res = []
+            for new_arg in new_args:
+                if isinstance(new_arg, tuple):
+                    res.extend(new_arg)
+                else:
+                    res.append(new_arg)
+            new_args = (res[:-1], res[-1])
         return _CallableGenericAlias(Callable, tuple(new_args))
 
 def _is_param_expr(obj):

Lib/test/test_typing.py

@@ -848,7 +848,6 @@ class C(Generic[*Ts]): pass
                         )
 
 
-
 class UnpackTests(BaseTestCase):
 
     def test_accepts_single_type(self):
@@ -1997,6 +1996,16 @@ def test_paramspec(self):
         self.assertEqual(repr(C2), f"{fullname}[~P, int]")
         self.assertEqual(repr(C2[int, str]), f"{fullname}[[int, str], int]")
 
+        # gh-102615:
+        C3 = C1[[P, str], bool]
+        self.assertEqual(C3.__parameters__, (P,))
+        self.assertEqual(C3.__args__, (P, str, bool))
+
+        self.assertEqual(C3[int].__args__, (int, str, bool))
+        self.assertEqual(C3[[int, complex]].__args__, (int, complex, str, bool))
+        self.assertEqual(C3[int, complex].__args__, (int, complex, str, bool))
+        self.assertEqual(C3[[]].__args__, (str, bool))
+
     def test_concatenate(self):
         Callable = self.Callable
         fullname = f"{Callable.__module__}.Callable"
@@ -7516,16 +7525,18 @@ class Z(Generic[P]):
     def test_multiple_paramspecs_in_user_generics(self):
         P = ParamSpec("P")
         P2 = ParamSpec("P2")
+        T = TypeVar("T")
 
-        class X(Generic[P, P2]):
+        class X(Generic[P, P2, T]):
             f: Callable[P, int]
             g: Callable[P2, str]
+            t: T
 
-        G1 = X[[int, str], [bytes]]
-        G2 = X[[int], [str, bytes]]
+        G1 = X[[int, str], [bytes], bool]
+        G2 = X[[int], [str, bytes], bool]
         self.assertNotEqual(G1, G2)
-        self.assertEqual(G1.__args__, ((int, str), (bytes,)))
-        self.assertEqual(G2.__args__, ((int,), (str, bytes)))
+        self.assertEqual(G1.__args__, ((int, str), (bytes,), bool))
+        self.assertEqual(G2.__args__, ((int,), (str, bytes), bool))
 
     def test_typevartuple_and_paramspecs_in_user_generics(self):
         Ts = TypeVarTuple("Ts")
@@ -7561,6 +7572,54 @@ class Y(Generic[P, *Ts]):
         with self.assertRaises(TypeError):
             Y[()]
 
+    def test_paramspec_subst(self):
+        # See: https://github.com/python/cpython/issues/102615
+        P = ParamSpec("P")
+        T = TypeVar("T")
+
+        class MyCallable(Generic[P, T]):
+            pass
+
+        G = MyCallable[P, T]
+        self.assertEqual(G.__parameters__, (P, T))
+        self.assertEqual(G.__args__, (P, T))
+
+        C = G[[P, str], bool]
+        self.assertEqual(C.__parameters__, (P,))
+        self.assertEqual(C.__args__, ((P, str), bool))
+
+        self.assertEqual(C[int].__parameters__, ())
+        self.assertEqual(C[int].__args__, ((int, str), bool))
+        self.assertEqual(C[[int, complex]].__args__, ((int, complex, str), bool))
+        self.assertEqual(C[[]].__args__, ((str,), bool))
+
+        Q = G[[int, str], T]
+        self.assertEqual(Q.__parameters__, (T,))
+        self.assertEqual(Q[bool].__parameters__, ())
+        self.assertEqual(Q[bool].__args__, ((int, str), bool))
+
+        # Reversed order:
+        class MyCallable2(Generic[T, P]):
+            pass
+
+        G2 = MyCallable[T, P]
+        self.assertEqual(G2.__parameters__, (T, P))
+        self.assertEqual(G2.__args__, (T, P))
+
+        C2 = G2[bool, [P, str]]
+        self.assertEqual(C2.__parameters__, (P,))
+        self.assertEqual(C2.__args__, (bool, (P, str)))
+
+        self.assertEqual(C2[int].__parameters__, ())
+        self.assertEqual(C2[int].__args__, (bool, (int, str)))
+        self.assertEqual(C2[[int, complex]].__args__, (bool, (int, complex, str)))
+        self.assertEqual(C2[[]].__args__, (bool, (str,)))
+
+        Q2 = G2[T, [int, str]]
+        self.assertEqual(Q2.__parameters__, (T,))
+        self.assertEqual(Q2[bool].__parameters__, ())
+        self.assertEqual(Q2[bool].__args__, (bool, (int, str)))
+
     def test_typevartuple_and_paramspecs_in_generic_aliases(self):
         P = ParamSpec('P')
         T = TypeVar('T')

Lib/typing.py

@@ -254,6 +254,8 @@ def _collect_parameters(args):
         # We don't want __parameters__ descriptor of a bare Python class.
         if isinstance(t, type):
             continue
+        if isinstance(t, tuple):
+            parameters.extend(_collect_parameters(t))
         if hasattr(t, '__typing_subst__'):
             if t not in parameters:
                 parameters.append(t)
@@ -1440,54 +1442,85 @@ def _determine_new_args(self, args):
 
         new_args = []
         for old_arg in self.__args__:
+            if isinstance(old_arg, tuple):
+                self._substitute_tuple_args(old_arg, new_args, new_arg_by_param)
+            else:
+                self._substitute_arg(old_arg, new_args, new_arg_by_param)
+        return tuple(new_args)
+
+    def _substitute_tuple_args(self, old_arg, new_args, new_arg_by_param):
+        # This method required to make this case correct:
+        #
+        # P = ParamSpec("P")
+        # T = TypeVar("T")
+        # class MyCallable(Generic[P, T]): ...
+        #
+        # MyCallable[P, T][[P, str], bool][int]
+        #
+        # Which must be equal to:
+        # MyCallable[[int, str], bool]
+        sub_args = []
+        for sub_old_arg in old_arg:
+            if _is_param_expr(sub_old_arg):
+                self._substitute_arg(sub_old_arg, sub_args, new_arg_by_param)
+            else:
+                sub_args.append(sub_old_arg)
 
-            if isinstance(old_arg, type):
-                new_args.append(old_arg)
+        # Now, unflatten the result:
+        res = []
+        for sub_arg in sub_args:
+            if isinstance(sub_arg, tuple):
+                res.extend(sub_arg)
                 continue
+            res.append(sub_arg)
+        new_args.append(tuple(res))
 
-            substfunc = getattr(old_arg, '__typing_subst__', None)
-            if substfunc:
-                new_arg = substfunc(new_arg_by_param[old_arg])
-            else:
-                subparams = getattr(old_arg, '__parameters__', ())
-                if not subparams:
-                    new_arg = old_arg
-                else:
-                    subargs = []
-                    for x in subparams:
-                        if isinstance(x, TypeVarTuple):
-                            subargs.extend(new_arg_by_param[x])
-                        else:
-                            subargs.append(new_arg_by_param[x])
-                    new_arg = old_arg[tuple(subargs)]
-
-            if self.__origin__ == collections.abc.Callable and isinstance(new_arg, tuple):
-                # Consider the following `Callable`.
-                #   C = Callable[[int], str]
-                # Here, `C.__args__` should be (int, str) - NOT ([int], str).
-                # That means that if we had something like...
-                #   P = ParamSpec('P')
-                #   T = TypeVar('T')
-                #   C = Callable[P, T]
-                #   D = C[[int, str], float]
-                # ...we need to be careful; `new_args` should end up as
-                # `(int, str, float)` rather than `([int, str], float)`.
-                new_args.extend(new_arg)
-            elif _is_unpacked_typevartuple(old_arg):
-                # Consider the following `_GenericAlias`, `B`:
-                #   class A(Generic[*Ts]): ...
-                #   B = A[T, *Ts]
-                # If we then do:
-                #   B[float, int, str]
-                # The `new_arg` corresponding to `T` will be `float`, and the
-                # `new_arg` corresponding to `*Ts` will be `(int, str)`. We
-                # should join all these types together in a flat list
-                # `(float, int, str)` - so again, we should `extend`.
-                new_args.extend(new_arg)
-            else:
-                new_args.append(new_arg)
+    def _substitute_arg(self, old_arg, new_args, new_arg_by_param):
+        if isinstance(old_arg, type):
+            new_args.append(old_arg)
+            return
 
-        return tuple(new_args)
+        substfunc = getattr(old_arg, '__typing_subst__', None)
+        if substfunc:
+            new_arg = substfunc(new_arg_by_param[old_arg])
+        else:
+            subparams = getattr(old_arg, '__parameters__', ())
+            if not subparams:
+                new_arg = old_arg
+            else:
+                subargs = []
+                for x in subparams:
+                    if isinstance(x, TypeVarTuple):
+                        subargs.extend(new_arg_by_param[x])
+                    else:
+                        subargs.append(new_arg_by_param[x])
+                new_arg = old_arg[tuple(subargs)]
+
+        if self.__origin__ == collections.abc.Callable and isinstance(new_arg, tuple):
+            # Consider the following `Callable`.
+            #   C = Callable[[int], str]
+            # Here, `C.__args__` should be (int, str) - NOT ([int], str).
+            # That means that if we had something like...
+            #   P = ParamSpec('P')
+            #   T = TypeVar('T')
+            #   C = Callable[P, T]
+            #   D = C[[int, str], float]
+            # ...we need to be careful; `new_args` should end up as
+            # `(int, str, float)` rather than `([int, str], float)`.
+            new_args.extend(new_arg)
+        elif _is_unpacked_typevartuple(old_arg):
+            # Consider the following `_GenericAlias`, `B`:
+            #   class A(Generic[*Ts]): ...
+            #   B = A[T, *Ts]
+            # If we then do:
+            #   B[float, int, str]
+            # The `new_arg` corresponding to `T` will be `float`, and the
+            # `new_arg` corresponding to `*Ts` will be `(int, str)`. We
+            # should join all these types together in a flat list
+            # `(float, int, str)` - so again, we should `extend`.
+            new_args.extend(new_arg)
+        else:
+            new_args.append(new_arg)
 
     def copy_with(self, args):
         return self.__class__(self.__origin__, args, name=self._name, inst=self._inst,

Misc/NEWS.d/next/Library/2023-03-14-10-11-46.gh-issue-102615.hgTYdd.rst

@@ -0,0 +1,2 @@
+Fix type variables substitution of :class:`collections.abc.Callable` and
+custom generics with ``ParamSpec``.