Skip to content

Commit f501cd0

Browse files
authored
Handle overloaded method ambiguity caused by self-types (#22011)
Fixes #11347 Closes #17239 I think we should handle this consistently with overloaded functions, essentially `foo.meth(x)` should behave the same as `Foo.meth(foo, x)`. Although this is a relatively niche situation, it seems to be important for some numerical libraries. It looks like there are no simple way to implement this properly. The only way I see requires a new attribute on `Overloaded` and a lot of plumbing (form binging site to the call site). I tried to apply various optimizations to reduce possible performance impact.
1 parent e3003dd commit f501cd0

8 files changed

Lines changed: 260 additions & 75 deletions

File tree

‎mypy/cache.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@
7272
from mypy_extensions import u8
7373

7474
# High-level cache layout format
75-
CACHE_VERSION: Final = 11
75+
CACHE_VERSION: Final = 12
7676

7777
# Type used internally to represent errors:
7878
# (path, line, column, end_line, end_column, severity, message, code)

‎mypy/checkexpr.py‎

Lines changed: 83 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -2898,6 +2898,7 @@ def check_overload_call(
28982898
callable_name,
28992899
object_type,
29002900
none_type_var_overlap,
2901+
callee.bound_args,
29012902
context,
29022903
)
29032904
except TooManyUnions:
@@ -2926,6 +2927,7 @@ def check_overload_call(
29262927
arg_names,
29272928
callable_name,
29282929
object_type,
2930+
callee.bound_args,
29292931
context,
29302932
)
29312933
# If any of checks succeed, perform deprecation tests and stop early.
@@ -3004,7 +3006,7 @@ def plausible_overload_call_targets(
30043006
arg_kinds: list[ArgKind],
30053007
arg_names: Sequence[str | None] | None,
30063008
overload: Overloaded,
3007-
) -> list[CallableType]:
3009+
) -> list[tuple[int, CallableType]]:
30083010
"""Returns all overload call targets that having matching argument counts.
30093011
30103012
If the given args contains a star-arg (*arg or **kwarg argument, except for
@@ -3014,16 +3016,19 @@ def plausible_overload_call_targets(
30143016
The only exception is if the starred argument is something like a Tuple or a
30153017
NamedTuple, which has a definitive "shape". If so, we don't move the corresponding
30163018
alternative to the front since we can infer a more precise match using the original
3017-
order."""
3019+
order.
3020+
3021+
The return also contains the original overload index for each plausible target.
3022+
"""
30183023

30193024
def has_shape(typ: Type) -> bool:
30203025
typ = get_proper_type(typ)
30213026
return isinstance(typ, (TupleType, TypedDictType)) or (
30223027
isinstance(typ, Instance) and typ.type.is_named_tuple
30233028
)
30243029

3025-
matches: list[CallableType] = []
3026-
star_matches: list[CallableType] = []
3030+
matches: list[tuple[int, CallableType]] = []
3031+
star_matches: list[tuple[int, CallableType]] = []
30273032

30283033
args_have_var_arg = False
30293034
args_have_kw_arg = False
@@ -3033,7 +3038,7 @@ def has_shape(typ: Type) -> bool:
30333038
if kind == ARG_STAR2 and not has_shape(typ):
30343039
args_have_kw_arg = True
30353040

3036-
for typ in overload.items:
3041+
for idx, typ in enumerate(overload.items):
30373042
formal_to_actual = map_actuals_to_formals(
30383043
arg_kinds, arg_names, typ.arg_kinds, typ.arg_names, lambda i: arg_types[i]
30393044
)
@@ -3044,28 +3049,29 @@ def has_shape(typ: Type) -> bool:
30443049
# is safe: it will be filtered out later.
30453050
# Unlike other var-args signatures, ParamSpec produces essentially
30463051
# a fixed signature, so there's no need to push them to the top.
3047-
matches.append(typ)
3052+
matches.append((idx, typ))
30483053
elif self.check_argument_count(
30493054
typ, arg_types, arg_kinds, arg_names, formal_to_actual, None
30503055
):
30513056
if args_have_var_arg and typ.is_var_arg:
3052-
star_matches.append(typ)
3057+
star_matches.append((idx, typ))
30533058
elif args_have_kw_arg and typ.is_kw_arg:
3054-
star_matches.append(typ)
3059+
star_matches.append((idx, typ))
30553060
else:
3056-
matches.append(typ)
3061+
matches.append((idx, typ))
30573062

30583063
return star_matches + matches
30593064

30603065
def infer_overload_return_type(
30613066
self,
3062-
plausible_targets: list[CallableType],
3067+
plausible_targets: list[tuple[int, CallableType]],
30633068
args: list[Expression],
30643069
arg_types: list[Type],
30653070
arg_kinds: list[ArgKind],
30663071
arg_names: Sequence[str | None] | None,
30673072
callable_name: str | None,
30683073
object_type: Type | None,
3074+
bound_args: list[Type] | None,
30693075
context: Context,
30703076
) -> tuple[Type, Type] | None:
30713077
"""Attempts to find the first matching callable from the given list.
@@ -3075,16 +3081,17 @@ def infer_overload_return_type(
30753081
If multiple targets match due to ambiguous Any parameters, returns (AnyType, AnyType).
30763082
If no targets match, returns None.
30773083
3078-
Assumes all of the given targets have argument counts compatible with the caller.
3084+
Assumes all the given targets have argument counts compatible with the caller.
30793085
"""
30803086

3081-
matches: list[CallableType] = []
3087+
matches: list[tuple[int, CallableType]] = []
30823088
return_types: list[Type] = []
30833089
inferred_types: list[Type] = []
3084-
args_contain_any = any(map(has_any_type, arg_types))
3090+
# bound_arg are recorded only if the original object type contained any.
3091+
args_contain_any = any(map(has_any_type, arg_types)) or bound_args is not None
30853092
type_maps: list[dict[Expression, Type]] = []
30863093

3087-
for typ in plausible_targets:
3094+
for idx, typ in plausible_targets:
30883095
assert self.msg is self.chk.msg
30893096
with self.msg.filter_errors(filter_revealed_type=True) as w:
30903097
with self.chk.local_type_map as m:
@@ -3108,48 +3115,66 @@ def infer_overload_return_type(
31083115
if isinstance(p_infer_type, CallableType):
31093116
# Prefer inferred types if possible, this will avoid false triggers for
31103117
# Any-ambiguity caused by arguments with Any passed to generic overloads.
3111-
matches.append(p_infer_type)
3118+
matches.append((idx, p_infer_type))
31123119
else:
3113-
matches.append(typ)
3120+
matches.append((idx, typ))
31143121
return_types.append(ret_type)
31153122
inferred_types.append(infer_type)
31163123
type_maps.append(m)
31173124

31183125
if not matches:
31193126
return None
3120-
elif any_causes_overload_ambiguity(matches, return_types, arg_types, arg_kinds, arg_names):
3127+
elif any_causes_overload_ambiguity(
3128+
matches, return_types, arg_types, arg_kinds, arg_names, bound_args
3129+
):
31213130
return_types = remove_dups(return_types)
31223131
# An argument of type or containing the type 'Any' caused ambiguity.
31233132
# We try returning a precise type if we can. If not, we give up and just return 'Any'.
31243133
if all_same_types(return_types):
31253134
self.chk.store_types(type_maps[0])
31263135
return return_types[0], inferred_types[0]
3127-
elif len(return_types) < MAX_PRECISE_OVERLOAD_FALLBACK and (
3128-
common := common_type(return_types)
3129-
):
3130-
self.chk.store_types(type_maps[0])
3131-
return common, erase_type(inferred_types[0])
3132-
elif all_same_types([erase_type(typ) for typ in return_types]):
3136+
3137+
# This is quite ad-hoc, but this is needed to support some numerical libraries.
3138+
# They add "fake" overload to some __op__ methods, to prevent fallback to __rop__
3139+
# where it would fail at runtime. We can ignore such overloads when deciding
3140+
# ambiguous overload fallback to get a more precise type, see #22011.
3141+
return_types = [
3142+
rt
3143+
for rt in return_types
3144+
if not isinstance(p := get_proper_type(rt), UninhabitedType) or p.ambiguous
3145+
]
3146+
3147+
# Before using the erased types, try checking if there is "best" fallback,
3148+
# such that it is a unique return type that is both (non-proper) subtype and
3149+
# supertype of all other return types. We do this only if there are few matches,
3150+
# since this check is expensive.
3151+
if len(return_types) < MAX_PRECISE_OVERLOAD_FALLBACK:
3152+
common = common_type(return_types)
3153+
if common is not None:
3154+
self.chk.store_types(type_maps[0])
3155+
return common, erase_type(inferred_types[0])
3156+
3157+
if all_same_types([erase_type(typ) for typ in return_types]):
31333158
self.chk.store_types(type_maps[0])
31343159
return erase_type(return_types[0]), erase_type(inferred_types[0])
3135-
else:
3136-
return self.check_call(
3137-
callee=AnyType(TypeOfAny.special_form),
3138-
args=args,
3139-
arg_kinds=arg_kinds,
3140-
arg_names=arg_names,
3141-
context=context,
3142-
callable_name=callable_name,
3143-
object_type=object_type,
3144-
)
3160+
3161+
return self.check_call(
3162+
callee=AnyType(TypeOfAny.special_form),
3163+
args=args,
3164+
arg_kinds=arg_kinds,
3165+
arg_names=arg_names,
3166+
context=context,
3167+
callable_name=callable_name,
3168+
object_type=object_type,
3169+
)
31453170
else:
31463171
# Success! No ambiguity; return the first match.
31473172
self.chk.store_types(type_maps[0])
31483173
return return_types[0], inferred_types[0]
31493174

31503175
def overload_erased_call_targets(
31513176
self,
3152-
plausible_targets: list[CallableType],
3177+
plausible_targets: list[tuple[int, CallableType]],
31533178
arg_types: list[Type],
31543179
arg_kinds: list[ArgKind],
31553180
arg_names: Sequence[str | None] | None,
@@ -3161,15 +3186,15 @@ def overload_erased_call_targets(
31613186
Assumes all of the given targets have argument counts compatible with the caller.
31623187
"""
31633188
matches: list[CallableType] = []
3164-
for typ in plausible_targets:
3189+
for _, typ in plausible_targets:
31653190
if self.erased_signature_similarity(
31663191
arg_types, arg_kinds, arg_names, args, typ, context
31673192
):
31683193
matches.append(typ)
31693194
return matches
31703195

31713196
def possible_none_type_var_overlap(
3172-
self, arg_types: list[Type], plausible_targets: list[CallableType]
3197+
self, arg_types: list[Type], plausible_targets: list[tuple[int, CallableType]]
31733198
) -> bool:
31743199
"""Heuristic to determine whether we need to try forcing union math.
31753200
@@ -3196,26 +3221,28 @@ def foo(x: T) -> list[T]: ...
31963221
if not has_optional_arg:
31973222
return False
31983223

3199-
min_prefix = min(len(c.arg_types) for c in plausible_targets)
3224+
min_prefix = min(len(c.arg_types) for _, c in plausible_targets)
32003225
for i in range(min_prefix):
32013226
if any(
3202-
isinstance(get_proper_type(c.arg_types[i]), NoneType) for c in plausible_targets
3227+
isinstance(get_proper_type(c.arg_types[i]), NoneType) for _, c in plausible_targets
32033228
) and any(
3204-
isinstance(get_proper_type(c.arg_types[i]), TypeVarType) for c in plausible_targets
3229+
isinstance(get_proper_type(c.arg_types[i]), TypeVarType)
3230+
for _, c in plausible_targets
32053231
):
32063232
return True
32073233
return False
32083234

32093235
def union_overload_result(
32103236
self,
3211-
plausible_targets: list[CallableType],
3237+
plausible_targets: list[tuple[int, CallableType]],
32123238
args: list[Expression],
32133239
arg_types: list[Type],
32143240
arg_kinds: list[ArgKind],
32153241
arg_names: Sequence[str | None] | None,
32163242
callable_name: str | None,
32173243
object_type: Type | None,
32183244
none_type_var_overlap: bool,
3245+
bound_args: list[Type] | None,
32193246
context: Context,
32203247
level: int = 0,
32213248
) -> list[tuple[Type, Type]] | None:
@@ -3225,7 +3252,7 @@ def union_overload_result(
32253252
Return a list of (<return type>, <inferred variant type>) if call succeeds for every
32263253
item of the desctructured union. Returns None if there is no match.
32273254
"""
3228-
# Step 1: If we are already too deep, then stop immediately. Otherwise mypy might
3255+
# Step 1: If we are already too deep, then stop immediately. Otherwise, mypy might
32293256
# hang for long time because of a weird overload call. The caller will get
32303257
# the exception and generate an appropriate note message, if needed.
32313258
if level >= MAX_UNIONS:
@@ -3247,6 +3274,7 @@ def union_overload_result(
32473274
arg_names,
32483275
callable_name,
32493276
object_type,
3277+
bound_args,
32503278
context,
32513279
)
32523280
if res is not None:
@@ -3265,6 +3293,7 @@ def union_overload_result(
32653293
arg_names,
32663294
callable_name,
32673295
object_type,
3296+
bound_args,
32683297
context,
32693298
)
32703299
if direct is not None and not isinstance(
@@ -3290,6 +3319,7 @@ def union_overload_result(
32903319
callable_name,
32913320
object_type,
32923321
none_type_var_overlap,
3322+
bound_args,
32933323
context,
32943324
level + 1,
32953325
)
@@ -6874,11 +6904,12 @@ def is_typetype_like(typ: ProperType) -> bool:
68746904

68756905

68766906
def any_causes_overload_ambiguity(
6877-
items: list[CallableType],
6907+
items: list[tuple[int, CallableType]],
68786908
return_types: list[Type],
68796909
arg_types: list[Type],
68806910
arg_kinds: list[ArgKind],
68816911
arg_names: Sequence[str | None] | None,
6912+
bound_args: list[Type] | None,
68826913
) -> bool:
68836914
"""May an argument containing 'Any' cause ambiguous result type on call to overloaded function?
68846915
@@ -6888,9 +6919,11 @@ def any_causes_overload_ambiguity(
68886919
68896920
Args:
68906921
items: Overload items matching the actual arguments
6922+
return_types: Corresponding inferred return type for each item
68916923
arg_types: Actual argument types
68926924
arg_kinds: Actual argument kinds
68936925
arg_names: Actual argument names
6926+
bound_args: Full list of self-types bound (if overload is a method)
68946927
"""
68956928
if all_same_types(return_types):
68966929
return False
@@ -6899,7 +6932,7 @@ def any_causes_overload_ambiguity(
68996932
map_formals_to_actuals(
69006933
arg_kinds, arg_names, item.arg_kinds, item.arg_names, lambda i: arg_types[i]
69016934
)
6902-
for item in items
6935+
for _, item in items
69036936
]
69046937

69056938
for arg_idx, arg_type in enumerate(arg_types):
@@ -6916,7 +6949,7 @@ def any_causes_overload_ambiguity(
69166949
matching_returns = []
69176950
matching_formals = []
69186951
for item_idx, formals in matching_formals_unfiltered:
6919-
matched_callable = items[item_idx]
6952+
_, matched_callable = items[item_idx]
69206953
matching_returns.append(matched_callable.ret_type)
69216954

69226955
# Note: if an actual maps to multiple formals of differing types within
@@ -6928,6 +6961,12 @@ def any_causes_overload_ambiguity(
69286961
if not all_same_types(matching_formals) and not all_same_types(matching_returns):
69296962
# Any maps to multiple different types, and the return types of these items differ.
69306963
return True
6964+
6965+
# If the original object type for methods contained Any, check the self-types as well.
6966+
if bound_args is not None:
6967+
matching_bound = [bound_args[idx] for idx, _ in items]
6968+
if not all_same_types(matching_bound):
6969+
return True
69316970
return False
69326971

69336972

‎mypy/copytype.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,7 @@ def visit_union_type(self, t: UnionType) -> ProperType:
120120
return self.copy_common(t, UnionType(t.items))
121121

122122
def visit_overloaded(self, t: Overloaded) -> ProperType:
123-
return self.copy_common(t, Overloaded(items=t.items))
123+
return self.copy_common(t, Overloaded(items=t.items, bound_args=t.bound_args))
124124

125125
def visit_type_type(self, t: TypeType) -> ProperType:
126126
# Use cast since the type annotations in TypeType are imprecise.

‎mypy/expandtype.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -531,7 +531,7 @@ def visit_overloaded(self, t: Overloaded) -> Type:
531531
assert isinstance(new_item, ProperType)
532532
assert isinstance(new_item, CallableType)
533533
items.append(new_item)
534-
return Overloaded(items)
534+
return Overloaded(items, t.bound_args)
535535

536536
def expand_type_list_with_unpack(self, typs: list[Type]) -> list[Type]:
537537
"""Expands a list of types that has an unpack."""

‎mypy/type_visitor.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -332,7 +332,7 @@ def visit_overloaded(self, t: Overloaded, /) -> Type:
332332
new = item.accept(self)
333333
assert isinstance(new, CallableType) # type: ignore[misc]
334334
items.append(new)
335-
return Overloaded(items=items)
335+
return Overloaded(items=items, bound_args=t.bound_args)
336336

337337
def visit_type_type(self, t: TypeType, /) -> Type:
338338
return TypeType.make_normalized(

0 commit comments

Comments
 (0)