[Async]ContextDecorator.__call__ return type should depend on the underlying context manager's _ExitT_co

[bswck]

Note

This is a fairly rare scenario. It's not top priority.

This is how ContextDecorator.__call__ is implemented in the Python standard library:

def __call__(self, func):
    @wraps(func)
    def inner(*args, **kwds):
        with self._recreate_cm():
            return func(*args, **kwds)
    return inner

In typeshed, ContextDecorator.__call__ currently has a return type of the original callable (func).
However, if the return value of self._recreate_cm().__call__ is truthy, the return type of ContextDecorator.__call__ can as well be None, like in the silly example below:

from collections.abc import Generator
from contextlib import contextmanager, suppress
from typing import reveal_type


@contextmanager
def zero_division_to_none() -> Generator[None]:
    with suppress(ZeroDivisionError):
        yield


@zero_division_to_none()
def div(a: int, b: int) -> float:
    return a / b


reveal_type(div(1, 0))
# mypy, pyright: Revealed type is "builtins.float"
# Runtime type is 'NoneType'

My first shot was to patch the proper __call__ functions to "extend" the return type with _ThrottledExcReturnT (a new type parameter of ContextDecorator), constrained by Never and None, with the default type Never for backward compatibility. To me, there's no way to declare the relationship symbolically, so it would have to be typed manually in classes implementing the context manager protocol.

However, that approach couldn't have worked:

diff --git a/stdlib/contextlib.pyi b/stdlib/contextlib.pyi
index f57e7fa67..1d65a7c12 100644
--- a/stdlib/contextlib.pyi
+++ b/stdlib/contextlib.pyi
@@ -4,7 +4,7 @@ from _typeshed import FileDescriptorOrPath, Unused
 from abc import ABC, abstractmethod
 from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Generator, Iterator
 from types import TracebackType
-from typing import IO, Any, Generic, Protocol, TypeVar, overload, runtime_checkable
+from typing import IO, Any, Generic, Protocol, TypeVar, Never, overload, runtime_checkable
 from typing_extensions import ParamSpec, Self, TypeAlias
 
 __all__ = [
@@ -32,9 +32,11 @@ _T = TypeVar("_T")
 _T_co = TypeVar("_T_co", covariant=True)
 _T_io = TypeVar("_T_io", bound=IO[str] | None)
 _ExitT_co = TypeVar("_ExitT_co", covariant=True, bound=bool | None, default=bool | None)
+_ThrottledExcReturnT = TypeVar("_ThrottledExcReturnT", Never, None, default=Never)
 _F = TypeVar("_F", bound=Callable[..., Any])
 _G = TypeVar("_G", bound=Generator[Any, Any, Any] | AsyncGenerator[Any, Any], covariant=True)
 _P = ParamSpec("_P")
+_R = TypeVar("_R")
 
 _SendT_contra = TypeVar("_SendT_contra", contravariant=True, default=None)
 _ReturnT_co = TypeVar("_ReturnT_co", covariant=True, default=None)
@@ -64,9 +66,13 @@ class AbstractAsyncContextManager(ABC, Protocol[_T_co, _ExitT_co]):  # type: ign
         self, exc_type: type[BaseException] | None, exc_value: BaseException | None, traceback: TracebackType | None, /
     ) -> _ExitT_co: ...
 
-class ContextDecorator:
+# If __exit__ can return a truthy value, _ThrottledExcReturnT should be None,
+# since the context manager can throttle exceptions and cause __call__ to return None.
+# If __exit__ returns falsey values only, _ThrottledExcReturnT should be Never
+# to not affect the return type of __call__.
+class ContextDecorator(Generic[_ThrottledExcReturnT]):
     def _recreate_cm(self) -> Self: ...
-    def __call__(self, func: _F) -> _F: ...
+    def __call__(self, func: Callable[_P, _R]) -> Callable[_P, _R | _ThrottledExcReturnT]: ...
 
 class _GeneratorContextManagerBase(Generic[_G]):
     # Ideally this would use ParamSpec, but that requires (*args, **kwargs), which this isn't. see #6676
@@ -79,7 +85,7 @@ class _GeneratorContextManagerBase(Generic[_G]):
 class _GeneratorContextManager(
     _GeneratorContextManagerBase[Generator[_T_co, _SendT_contra, _ReturnT_co]],
     AbstractContextManager[_T_co, bool | None],
-    ContextDecorator,
+    ContextDecorator[None],  # maybe the type checker should figure out whether it's really None or Never?
 ):
     if sys.version_info >= (3, 9):
         def __exit__(
@@ -95,14 +101,15 @@ def contextmanager(func: Callable[_P, Iterator[_T_co]]) -> Callable[_P, _Generat
 if sys.version_info >= (3, 10):
     _AF = TypeVar("_AF", bound=Callable[..., Awaitable[Any]])
 
-    class AsyncContextDecorator:
+    # _ThrottledExcReturnT: see ContextDecorator.
+    class AsyncContextDecorator(Generic[_ThrottledExcReturnT]):
         def _recreate_cm(self) -> Self: ...
-        def __call__(self, func: _AF) -> _AF: ...
+        def __call__(self, func: Callable[_P, _R]) -> Callable[_P, _R | _ThrottledExcReturnT]: ...
 
     class _AsyncGeneratorContextManager(
         _GeneratorContextManagerBase[AsyncGenerator[_T_co, _SendT_contra]],
         AbstractAsyncContextManager[_T_co, bool | None],
-        AsyncContextDecorator,
+        AsyncContextDecorator[None],
     ):
         async def __aexit__(
             self, typ: type[BaseException] | None, value: BaseException | None, traceback: TracebackType | None

(I haven't considered variance here much.)

In case of GeneratorContextManager classes, type checkers would ideally figure out whether the context manager handles an exception (and which?) or not. I'm not sure this is possible.

Anyway, this approach can't work here just as explained in #13436: Callable[P, R] loses the information on overloads.

Therefore, this idea is blocked for the same reason, and amplifies the necessity of denoting callable types with lower-level access to the paramspec and the return type.