diff --git a/conformance/third_party/conformance.exp b/conformance/third_party/conformance.exp index 72f2b792bc..300ddff73c 100644 --- a/conformance/third_party/conformance.exp +++ b/conformance/third_party/conformance.exp @@ -3844,8 +3844,8 @@ { "code": -2, "column": 12, - "concise_description": "revealed type: (*args: Any, **kwargs: Any) -> NoReturn", - "description": "revealed type: (*args: Any, **kwargs: Any) -> NoReturn", + "concise_description": "revealed type: (...) -> NoReturn", + "description": "revealed type: (...) -> NoReturn", "line": 99, "name": "reveal-type", "severity": "info", diff --git a/pyrefly/lib/solver/subset.rs b/pyrefly/lib/solver/subset.rs index 4996264641..8bdf155340 100644 --- a/pyrefly/lib/solver/subset.rs +++ b/pyrefly/lib/solver/subset.rs @@ -160,19 +160,6 @@ fn params_have_any_args_and_kwargs(params: &Params) -> bool { } } -/// Whether a callable-typed value should be treated as having gradual (`...`) parameters -/// because its definition had both `*args` and `**kwargs` typed `Any`. For a `Function` we -/// trust the definition-time flag, so an `Any` introduced by type-parameter substitution (e.g. -/// `Proto[Any]` over `*args: T, **kwargs: T`) does not count. A bare `Callable` has no such -/// metadata, so we fall back to inspecting its params. -fn sig_is_gradual_variadic(ty: &Type) -> bool { - match ty { - Type::Function(f) => f.metadata.flags.has_gradual_variadic_params, - Type::Callable(c) => params_have_any_args_and_kwargs(&c.params), - _ => false, - } -} - fn all( it: impl Iterator, mut check: impl FnMut(T) -> Result<(), SubsetError>, @@ -205,6 +192,25 @@ struct FreshForall { } impl<'solver, 'subset, Ans: LookupAnswer> Subset<'solver, 'subset, Ans> { + /// Whether `*args: Any, **kwargs: Any` in `params` form a gradual tail, which is equivalent + /// to `...`. `strict_callable_subtyping` turns this off. + fn params_have_gradual_tail(&self, params: &[Param]) -> bool { + !self.solver.config.strict_callable_subtyping && params_are_gradual_variadic(params) + } + + /// Like `params_have_gradual_tail`, for the signature of a callable-typed value. For a + /// `Function` we trust the definition-time flag, so an `Any` introduced by type-parameter + /// substitution (e.g. `Proto[Any]` over `*args: T, **kwargs: T`) does not count. A bare + /// `Callable` has no such metadata, so we fall back to inspecting its params. + fn sig_has_gradual_tail(&self, ty: &Type) -> bool { + !self.solver.config.strict_callable_subtyping + && match ty { + Type::Function(f) => f.metadata.flags.has_gradual_variadic_params, + Type::Callable(c) => params_have_any_args_and_kwargs(&c.params), + _ => false, + } + } + fn is_subset_literal_int_size( &mut self, literal: i64, @@ -236,28 +242,17 @@ impl<'solver, 'subset, Ans: LookupAnswer> Subset<'solver, 'subset, Ans> { } /// Can a function with l_args be called as a function with u_args? + /// + /// If `u_gradual` is set, the `*args: Any` and `**kwargs: Any` parameters of `u_args` are a + /// gradual tail (like `...`) that is compatible with whatever parameters of `l_args` remain + /// after matching the other parameters of `u_args`. A gradual `l_args` needs no flag: its + /// `*args: Any` and `**kwargs: Any` already accept every parameter of `u_args` that the + /// other parameters of `l_args` do not match. fn is_subset_param_list( &mut self, l_args: &[Param], u_args: &[Param], - l_gradual: bool, u_gradual: bool, - ) -> Result<(), SubsetError> { - // Don't short-circuit because we may want to pin/solve variables - let result = self.is_subset_param_list_impl(l_args, u_args); - match result { - Err(_) if !self.solver.config.strict_callable_subtyping && (l_gradual || u_gradual) => { - Ok(()) - } - _ => result, - } - } - - /// Can a function with l_args be called as a function with u_args? - fn is_subset_param_list_impl( - &mut self, - l_args: &[Param], - u_args: &[Param], ) -> Result<(), SubsetError> { let mut l_args = l_args.iter(); let mut u_args = u_args.iter(); @@ -278,6 +273,15 @@ impl<'solver, 'subset, Ans: LookupAnswer> Subset<'solver, 'subset, Ans> { break; } } + // The gradual `*args` of `u` stands for any remaining positional parameters of `l`. + // A `*args` of `l` is still matched below, so that `u` gives it context. + ( + None + | Some( + Param::PosOnly(..) | Param::Pos(..) | Param::KwOnly(..) | Param::Kwargs(..), + ), + Some(Param::Varargs(..)), + ) if u_gradual => break, ( Some(Param::PosOnly(_, l, l_req) | Param::Pos(_, l, l_req)), Some(Param::PosOnly(_, u, u_req)), @@ -472,7 +476,7 @@ impl<'solver, 'subset, Ans: LookupAnswer> Subset<'solver, 'subset, Ans> { l_arg = l_args.next(); u_arg = u_args.next(); } - (Some(_), Some(Param::KwOnly(_, _, _) | Param::Kwargs(_, _))) => { + (_, Some(Param::KwOnly(_, _, _) | Param::Kwargs(_, _))) => { break; } (None, Some(Param::PosOnly(_, ty, _) | Param::Pos(_, ty, _))) => { @@ -588,7 +592,8 @@ impl<'solver, 'subset, Ans: LookupAnswer> Subset<'solver, 'subset, Ans> { self.is_subset_eq(&u, &l)?; Some(l) } - (None, Some(_)) => { + // The gradual `**kwargs` of `u` stands for any remaining keyword parameters of `l`. + (None, Some(_)) if !u_gradual => { return Err(SubsetError::Other); } (l_kwargs, _) => l_kwargs, @@ -622,14 +627,14 @@ impl<'solver, 'subset, Ans: LookupAnswer> Subset<'solver, 'subset, Ans> { return Err(SubsetError::Other); } } - for (_, l_req) in l_keywords.values() { - if *l_req { - return Err(SubsetError::Other); - } + if !u_gradual && l_keywords.values().any(|(_, l_req)| *l_req) { + return Err(SubsetError::Other); } Ok(()) } + /// `l_gradual` and `u_gradual` say whether `l_params` and `u_params` have a gradual tail; see + /// `params_have_gradual_tail`. fn is_subset_params( &mut self, l_params: &Params, @@ -637,7 +642,7 @@ impl<'solver, 'subset, Ans: LookupAnswer> Subset<'solver, 'subset, Ans> { l_gradual: bool, u_gradual: bool, ) -> Result<(), SubsetError> { - let result = match (l_params, u_params) { + match (l_params, u_params) { (Params::Ellipsis, Params::ParamSpec(_, pspec)) => { self.is_subset_eq(&Type::Ellipsis, pspec) } @@ -668,12 +673,12 @@ impl<'solver, 'subset, Ans: LookupAnswer> Subset<'solver, 'subset, Ans> { ( Params::List(l_args) | Params::Partial(l_args), Params::List(u_args) | Params::Partial(u_args), - ) => self.is_subset_param_list(l_args.items(), u_args.items(), l_gradual, u_gradual), + ) => self.is_subset_param_list(l_args.items(), u_args.items(), u_gradual), (Params::List(ls) | Params::Partial(ls), Params::ParamSpec(args, pspec)) => { - self.is_paramlist_subset_of_paramspec(ls, args, pspec) + self.is_paramlist_subset_of_paramspec(ls, l_gradual, args, pspec) } (Params::ParamSpec(args, pspec), Params::List(ls) | Params::Partial(ls)) => { - self.is_paramspec_subset_of_paramlist(args, pspec, ls) + self.is_paramspec_subset_of_paramlist(args, pspec, ls, u_gradual) } (Params::ParamSpec(ls, p1), Params::ParamSpec(us, p2)) => { self.is_paramspec_subset_of_paramspec(ls, p1, us, p2) @@ -685,15 +690,9 @@ impl<'solver, 'subset, Ans: LookupAnswer> Subset<'solver, 'subset, Ans> { l_params, &Params::List(ParamList::everything()), l_gradual, - true, + !self.solver.config.strict_callable_subtyping, ) } - }; - match result { - Err(_) if !self.solver.config.strict_callable_subtyping && (l_gradual || u_gradual) => { - Ok(()) - } - _ => result, } } @@ -1082,9 +1081,27 @@ impl<'solver, 'subset, Ans: LookupAnswer> Subset<'solver, 'subset, Ans> { } } + /// The ParamSpec value for the parameters that remain after matching a `Concatenate` prefix. + /// If they are only the gradual `*args: Any, **kwargs: Any` tail of the signature, they are + /// equivalent to `...`. + fn param_spec_remainder(&self, params: &[Param], gradual: bool) -> Type { + if gradual + && params + .iter() + .all(|p| matches!(p, Param::Varargs(..) | Param::Kwargs(..))) + { + Type::Ellipsis + } else { + self.solver + .heap + .mk_param_spec_value(ParamList::new(params.to_vec())) + } + } + fn is_paramlist_subset_of_paramspec( &mut self, got: &ParamList, + got_gradual: bool, want_ts: &[PrefixParam], want_pspec: &Type, ) -> Result<(), SubsetError> { @@ -1094,29 +1111,17 @@ impl<'solver, 'subset, Ans: LookupAnswer> Subset<'solver, 'subset, Ans> { if got.len() < args.len() { // Run the regular parameter matcher first so it can distinguish a type mismatch // from parameters that are genuinely absent. A variadic parameter may consume the - // whole prefix, but inferring the remaining ParamSpec from it is not supported. - self.is_subset_param_list( - got.items(), - &args, - params_are_gradual_variadic(got.items()), - params_are_gradual_variadic(&args), - )?; - return Err(SubsetError::Other); + // whole prefix, but inferring the remaining ParamSpec from it is only supported + // for a gradual tail, which leaves `...`. + self.is_subset_param_list(got.items(), &args, false)?; + if !got_gradual { + return Err(SubsetError::Other); + } + return self.is_subset_eq(&Type::Ellipsis, want_pspec); } let (pre, post) = got.items().split_at(args.len()); - self.is_subset_param_list( - pre, - &args, - params_are_gradual_variadic(pre), - params_are_gradual_variadic(&args), - )?; - self.is_subset_eq( - &self - .solver - .heap - .mk_param_spec_value(ParamList::new(post.to_vec())), - want_pspec, - ) + self.is_subset_param_list(pre, &args, false)?; + self.is_subset_eq(&self.param_spec_remainder(post, got_gradual), want_pspec) } fn is_paramspec_subset_of_paramlist( @@ -1124,25 +1129,19 @@ impl<'solver, 'subset, Ans: LookupAnswer> Subset<'solver, 'subset, Ans> { got_ts: &[PrefixParam], got_pspec: &Type, want: &ParamList, + want_gradual: bool, ) -> Result<(), SubsetError> { - if want.len() < got_ts.len() { - return Err(SubsetError::Other); - } let args: Vec = got_ts.iter().map(|p| p.to_param_preserve_name()).collect(); - let (pre, post) = want.items().split_at(args.len()); - self.is_subset_param_list( - &args, - pre, - params_are_gradual_variadic(&args), - params_are_gradual_variadic(pre), - )?; - self.is_subset_eq( - got_pspec, - &self - .solver - .heap - .mk_param_spec_value(ParamList::new(post.to_vec())), - ) + // A gradual tail in `want` may stand for the prefix parameters it does not have. + let (pre, post) = if want.len() >= args.len() { + want.items().split_at(args.len()) + } else if want_gradual { + (want.items(), &[][..]) + } else { + return Err(SubsetError::Other); + }; + self.is_subset_param_list(&args, pre, want_gradual)?; + self.is_subset_eq(got_pspec, &self.param_spec_remainder(post, want_gradual)) } fn is_paramspec_subset_of_paramspec( @@ -2256,16 +2255,9 @@ impl<'solver, 'subset, Ans: LookupAnswer> Subset<'solver, 'subset, Ans> { (l, Type::Overloaded(branches)) => { all(branches.iter(), |branch| self.is_subset_eq(l, branch)) } - (l, Type::Overload(overload)) => { - let l_gradual = sig_is_gradual_variadic(l); - let result = all(overload.signatures.iter(), |u| { - self.is_subset_eq(l, &u.as_type()) - }); - match result { - Err(_) if !self.solver.config.strict_callable_subtyping && l_gradual => Ok(()), - _ => result, - } - } + (l, Type::Overload(overload)) => all(overload.signatures.iter(), |u| { + self.is_subset_eq(l, &u.as_type()) + }), (Type::Quantified(q), u) if !q.restriction().is_restricted() => self.is_subset_eq( &self .solver @@ -2351,8 +2343,8 @@ impl<'solver, 'subset, Ans: LookupAnswer> Subset<'solver, 'subset, Ans> { Type::Function(f) => &f.signature, _ => unreachable!("guarded by pattern above"), }; - let l_gradual = sig_is_gradual_variadic(got); - let u_gradual = sig_is_gradual_variadic(want); + let l_gradual = self.sig_has_gradual_tail(got); + let u_gradual = self.sig_has_gradual_tail(want); let argument_side = self.active_call_context.argument_side(); self.with_active_call_context( Some( @@ -2989,15 +2981,22 @@ impl<'solver, 'subset, Ans: LookupAnswer> Subset<'solver, 'subset, Ans> { (Type::ParamSpecValue(ls), Type::ParamSpecValue(us)) => self.is_subset_param_list( ls.items(), us.items(), - params_are_gradual_variadic(ls.items()), - params_are_gradual_variadic(us.items()), + self.params_have_gradual_tail(us.items()), ), - (Type::ParamSpecValue(ls), Type::Concatenate(us, u_pspec)) => { - self.is_paramlist_subset_of_paramspec(ls, us, u_pspec) - } - (Type::Concatenate(ls, l_pspec), Type::ParamSpecValue(us)) => { - self.is_paramspec_subset_of_paramlist(ls, l_pspec, us) - } + (Type::ParamSpecValue(ls), Type::Concatenate(us, u_pspec)) => self + .is_paramlist_subset_of_paramspec( + ls, + self.params_have_gradual_tail(ls.items()), + us, + u_pspec, + ), + (Type::Concatenate(ls, l_pspec), Type::ParamSpecValue(us)) => self + .is_paramspec_subset_of_paramlist( + ls, + l_pspec, + us, + self.params_have_gradual_tail(us.items()), + ), (Type::Concatenate(ls, l_pspec), Type::Concatenate(us, u_pspec)) => { self.is_paramspec_subset_of_paramspec(ls, l_pspec, us, u_pspec) } diff --git a/pyrefly/lib/test/callable.rs b/pyrefly/lib/test/callable.rs index 03053a7b4d..cacffa5d91 100644 --- a/pyrefly/lib/test/callable.rs +++ b/pyrefly/lib/test/callable.rs @@ -2498,6 +2498,109 @@ def f(n: NoArgs) -> None: "#, ); +// A gradual `*args: Any, **kwargs: Any` tail does not make the rest of the signature gradual: +// the other parameters are still checked. See https://github.com/pyrefly-org/pyrefly/issues/5194. +testcase!( + test_gradual_variadic_params_check_other_params, + r#" +from typing import Any, Callable, Protocol, TypedDict + +class Unannotated(Protocol): + def __call__(self, a: str, /, *args, **kwargs) -> None: ... + +class Annotated(Protocol): + def __call__(self, a: str, /, *args: Any, **kwargs: Any) -> None: ... + +def f(p1: Unannotated, p2: Annotated) -> None: + x1: Callable[[int], None] = p1 # E: `Unannotated` is not assignable to `(int) -> None` + x2: Callable[[int], None] = p2 # E: `Annotated` is not assignable to `(int) -> None` + x3: Callable[[str, int], None] = p2 + +def g(a: str, /, *args, **kwargs) -> None: ... +x4: Callable[[int], None] = g # E: is not assignable to `(int) -> None` +x5: Callable[[], None] = g # E: is not assignable to `() -> None` + +def h(*args: Any, k: str, **kwargs: Any) -> None: ... +x6: Callable[[], None] = h # E: is not assignable to `() -> None` + +class A(TypedDict): + x: int +class B(TypedDict): + y: int + +class KwStr(Protocol): + def __call__(self, *, b: int) -> None: ... +def kw_str(*args: Any, b: str, **kwargs: Any) -> None: ... +x7: KwStr = kw_str # E: is not assignable to `KwStr` + +class KwTypedDict(Protocol): + def __call__(self, *, b: B) -> None: ... +def kw_typed_dict(*args: Any, b: A, **kwargs: Any) -> None: ... +x8: KwTypedDict = kw_typed_dict # E: is not assignable to `KwTypedDict` + "#, +); + +// A gradual tail in the target accepts whatever parameters remain after matching the target's +// other parameters. +testcase!( + test_gradual_variadic_params_target_tail, + r#" +from typing import Any, Protocol + +class Prefix(Protocol): + def __call__(self, a: int, /, *args: Any, **kwargs: Any) -> None: ... + +class Keyword(Protocol): + def __call__(self, *args: Any, k: str, **kwargs: Any) -> None: ... + +def no_params() -> None: ... +def extra_params(a: int, b: str, /, *, m: str) -> None: ... +def wrong_prefix(a: str, b: str, /) -> None: ... +def varargs(*args: int) -> None: ... +def keyword(x: int, *, k: str) -> None: ... +def wrong_keyword(*, k: int) -> None: ... + +ok1: Prefix = extra_params +ok2: Prefix = varargs +ok3: Keyword = keyword +err1: Prefix = no_params # E: is not assignable to `Prefix` +err2: Prefix = wrong_prefix # E: is not assignable to `Prefix` +err3: Keyword = varargs # E: is not assignable to `Keyword` +err4: Keyword = wrong_keyword # E: is not assignable to `Keyword` + "#, +); + +// A gradual tail is `...` when matched against a `ParamSpec`. +testcase!( + test_gradual_variadic_params_paramspec, + r#" +from typing import Any, Callable, Concatenate, Protocol, reveal_type + +def drop_two[**P](f: Callable[Concatenate[int, str, P], None]) -> Callable[P, None]: ... + +def gradual(*args: Any, **kwargs: Any) -> None: ... +def gradual_prefix(a: int, b: str, c: bytes, /, *args: Any, **kwargs: Any) -> None: ... +def gradual_wrong_prefix(a: str, /, *args: Any, **kwargs: Any) -> None: ... + +reveal_type(drop_two(gradual)) # E: revealed type: (...) -> None +reveal_type(drop_two(gradual_prefix)) # E: revealed type: (c: bytes, /, *args: Any, **kwargs: Any) -> None +drop_two(gradual_wrong_prefix) # E: Argument `(a: str, /, *args: Any, **kwargs: Any) -> None` is not assignable to parameter `f` + +class Prefix(Protocol): + def __call__(self, a: int, /, *args: Any, **kwargs: Any) -> None: ... + +class WithParamSpec[**P](Protocol): + def __call__(self, a: int, /, *args: P.args, **kwargs: P.kwargs) -> None: ... + +class WithParamSpecStr[**P](Protocol): + def __call__(self, a: str, /, *args: P.args, **kwargs: P.kwargs) -> None: ... + +def f[**P](x: WithParamSpec[P], y: WithParamSpecStr[P]) -> None: + ok: Prefix = x + err: Prefix = y # E: is not assignable to `Prefix` + "#, +); + testcase!( test_defaultdict_frozenset, r#" diff --git a/pyrefly/lib/test/class_overrides.rs b/pyrefly/lib/test/class_overrides.rs index 96d9d73ff0..a60e678b27 100644 --- a/pyrefly/lib/test/class_overrides.rs +++ b/pyrefly/lib/test/class_overrides.rs @@ -353,14 +353,50 @@ class GradualBase: class Child(GradualBase): @override - def method1(self) -> None: ... + def method1(self) -> None: ... # E: Class member `Child.method1` overrides parent class `GradualBase` in an inconsistent manner + +class LooseChild(GradualBase): + @override + def method1(self, a: int, b: str, /, *, k: int, m: int) -> None: ... class Base: def method1(self) -> None: ... class GradualChild(Base): @override - def method1(self, a: int, /, *args: Any, k: int, **kwargs: Any) -> None: ... + def method1(self, a: int, /, *args: Any, k: int, **kwargs: Any) -> None: ... # E: Class member `GradualChild.method1` overrides parent class `Base` in an inconsistent manner + +class LooseGradualChild(Base): + @override + def method1(self, a: int = 0, /, *args: Any, k: int = 0, **kwargs: Any) -> None: ... + "#, +); + +// The parameters that precede a gradual `*args: Any, **kwargs: Any` tail are still checked, +// including against each signature of an overloaded parent method. +testcase!( + test_override_parameter_gradual_prefix, + r#" +from typing import Any, overload + +class Base: + @overload + def m(self, x: int) -> None: ... + @overload + def m(self, x: bytes) -> None: ... + def m(self, x: int | bytes) -> None: ... + +class Child(Base): + def m(self, x: str, *args: Any, **kwargs: Any) -> None: ... # E: Class member `Child.m` overrides parent class `Base` in an inconsistent manner + +class Base2: + def m(self, x: int) -> None: ... + +class Child2(Base2): + def m(self, x: str, *args: Any, **kwargs: Any) -> None: ... # E: Class member `Child2.m` overrides parent class `Base2` in an inconsistent manner + +class Child3(Base2): + def m(self, x: int, *args: Any, **kwargs: Any) -> None: ... "#, ); diff --git a/website/docs/configuration.mdx b/website/docs/configuration.mdx index 62e55978a2..22dffe3700 100644 --- a/website/docs/configuration.mdx +++ b/website/docs/configuration.mdx @@ -1196,9 +1196,9 @@ in per-file `pyrefly check`; for per-file `check`, Controls whether Pyrefly enforces strict parameter compatibility when checking callable subtyping for signatures with `*args: Any, **kwargs: Any`. -When set to `false` (the default), callables with `*args: Any, **kwargs: Any` -are treated as a gradual type — assignability checks in both directions succeed -when they otherwise wouldn't, similar to how `...` (ellipsis) parameters behave. +When set to `false` (the default), `*args: Any, **kwargs: Any` in a signature +is treated as gradual, like `...` (ellipsis): it is compatible with any +parameters, while the other parameters in the signature are still checked. This is a pragmatic default because many real-world stubs and decorators use `*args: Any, **kwargs: Any` as a wildcard signature.