diff --git a/pyrefly/benches/micro.rs b/pyrefly/benches/micro.rs index b8e2a68dfc..ae0c99742b 100644 --- a/pyrefly/benches/micro.rs +++ b/pyrefly/benches/micro.rs @@ -367,18 +367,26 @@ result: Base[object] = {expression} ) } -/// Each layer rejects the first hint and checks the same child under the second hint. -fn nested_init_only_constructor_hints(depth: usize) -> String { - let mut expression = "Node(value)".to_owned(); +/// Repeated contextual attempts check each child against the same union hint. +fn nested_init_only_constructor_hints(depth: usize, value_type: &str, callbacks: bool) -> String { + let callback = if callbacks { "lambda arg: None, " } else { "" }; + let callback_parameter = if callbacks { + "cb: Callable[[T], None], " + } else { + "" + }; + let mut expression = format!("Node({callback}value)"); for _ in 1..depth { - expression = format!("Node(value, {expression})"); + expression = format!("Node({callback}value, {expression})"); } format!( r#" +from typing import Callable + class Node[T]: - def __init__(self, value: T, child: Node[int] | Node[str] | None = None) -> None: ... + def __init__(self, {callback_parameter}value: T, child: Node[int] | Node[str] | None = None) -> None: ... -def f(value: str) -> None: +def f(value: {value_type}) -> None: root: Node[int] | Node[str] = {expression} "# ) @@ -659,6 +667,39 @@ fn overloads(c: &mut Criterion) { } fn nested_generic_constructor(c: &mut Criterion) { + for count in [3200, 6400] { + let calls = "A(lambda x: None), ".repeat(count); + measure( + c, + &format!("sibling_constructor_lambdas_{count}"), + format!( + r#" +from typing import Callable +class A[T]: + def __init__(self, cb: Callable[[T], None]) -> None: ... +items: list[A[int]] = [{calls}] +"#, + ), + 0, + ); + } + for count in [800, 1600] { + let calls = "A(lambda x: None, value), ".repeat(count); + measure( + c, + &format!("sibling_distributed_constructor_lambdas_{count}"), + format!( + r#" +from typing import Callable +class A[T]: + def __init__(self, cb: Callable[[T], None], value: T) -> None: ... +def f(value: int | str) -> None: + items: list[A[int] | A[str]] = [{calls}] +"#, + ), + 0, + ); + } measure( c, "nested_generic_constructor_soft_error_12", @@ -669,10 +710,35 @@ fn nested_generic_constructor(c: &mut Criterion) { measure( c, &format!("nested_init_only_constructor_hints_{depth}"), - nested_init_only_constructor_hints(depth), + nested_init_only_constructor_hints(depth, "str", false), 0, ); } + for depth in [8, 12] { + measure( + c, + &format!("nested_init_only_constructor_union_distribution_{depth}"), + nested_init_only_constructor_hints(depth, "int | str", false), + depth, + ); + } + // Nested union hints are disabled inside a constructor probe. + for depth in [3, 4, 6, 8] { + measure( + c, + &format!("nested_contextual_constructor_union_{depth}"), + nested_init_only_constructor_hints(depth, "int | str", true), + depth, + ); + } + for depth in [4, 6, 8] { + measure( + c, + &format!("nested_rejected_constructor_union_{depth}"), + nested_init_only_constructor_hints(depth, "int | str | bytes", false), + depth, + ); + } } fn nested_nongeneric_constructor(c: &mut Criterion) { diff --git a/pyrefly/lib/alt/answers.rs b/pyrefly/lib/alt/answers.rs index 7b1c7710ba..bdca3c9d57 100644 --- a/pyrefly/lib/alt/answers.rs +++ b/pyrefly/lib/alt/answers.rs @@ -41,6 +41,7 @@ use ruff_text_size::Ranged; use ruff_text_size::TextRange; use starlark_map::Hashed; use starlark_map::small_map::SmallMap; +use vec1::Vec1; use crate::alt::answers_solver::AnswerScope; use crate::alt::answers_solver::AnswersSolver; @@ -166,31 +167,12 @@ impl VisitMut for Traces { } } for callee in self.overloaded_callees.values_mut() { - // Note: `tparams` does not need to be visited, as this comes from an answer - // which is already visited. `TArgs` skips tparams for the same reason. - match callee { - OverloadedCallee::Resolved { callable: trace } => { - let OverloadTrace { - callable, - tparams: _, - } = trace; - callable.visit_mut(f); - } - OverloadedCallee::Candidates { all, closest, .. } => { - for OverloadTrace { - callable, - tparams: _, - } in all.iter_mut().chain(std::iter::once(closest)) - { - callable.visit_mut(f); - } - } - } + callee.recurse_mut(f); } } } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct OverloadTrace { callable: Callable, tparams: Option>, @@ -212,7 +194,7 @@ impl OverloadTrace { } } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Eq)] pub enum OverloadedCallee { Resolved { callable: OverloadTrace, @@ -222,6 +204,87 @@ pub enum OverloadedCallee { closest: OverloadTrace, is_closest_chosen: bool, }, + /// Chosen callees from mutually exclusive, accepted contextual checks. + Alternatives(Vec1), +} + +impl OverloadedCallee { + fn chosen_type(&self) -> Option { + match self { + Self::Resolved { callable } => Some(callable.as_type()), + Self::Candidates { + closest, + is_closest_chosen: true, + .. + } => Some(closest.as_type()), + Self::Candidates { .. } => None, + Self::Alternatives(alternatives) => alternatives + .iter() + .map(Self::chosen_type) + .collect::>>() + .map(Type::union), + } + } + + fn signatures(&self) -> (Vec, Option) { + match self { + Self::Resolved { callable } => (vec![callable.callable.clone()], Some(0)), + Self::Candidates { all, closest, .. } => ( + all.iter().map(|trace| trace.callable.clone()).collect(), + all.iter() + .position(|trace| trace.callable == closest.callable), + ), + Self::Alternatives(alternatives) => ( + alternatives + .iter() + .flat_map(|callee| callee.signatures().0) + .collect(), + None, + ), + } + } + + fn merge_alternative(&mut self, other: Self) { + if *self == other { + return; + } + match self { + Self::Alternatives(alternatives) => match other { + Self::Alternatives(others) => { + for other in others { + if !alternatives.contains(&other) { + alternatives.push(other); + } + } + } + other if !alternatives.contains(&other) => alternatives.push(other), + _ => {} + }, + _ => { + let mut alternatives = Vec1::new(self.clone()); + alternatives.push(other); + *self = Self::Alternatives(alternatives); + } + } + } +} + +impl VisitMut for OverloadedCallee { + fn recurse_mut(&mut self, f: &mut dyn FnMut(&mut Type)) { + match self { + Self::Resolved { callable } => callable.callable.visit_mut(f), + Self::Candidates { all, closest, .. } => { + for trace in all.iter_mut().chain(std::iter::once(closest)) { + trace.callable.visit_mut(f); + } + } + Self::Alternatives(alternatives) => { + for alternative in alternatives { + alternative.recurse_mut(f); + } + } + } + } } #[derive(Debug, Default)] @@ -263,6 +326,53 @@ pub struct TraceSideEffects { pub expected_types: SmallMap>, } +impl VisitMut for TraceSideEffects { + fn recurse_mut(&mut self, f: &mut dyn FnMut(&mut Type)) { + for map in [ + &mut self.types, + &mut self.invoked_properties, + &mut self.expected_types, + ] { + for ty in map.values_mut() { + f(Arc::make_mut(ty)); + } + } + for callee in self.overloaded_callees.values_mut() { + callee.recurse_mut(f); + } + } +} + +impl TraceSideEffects { + /// Preserve every accepted context at each source range, rather than the last writer. + pub(crate) fn merge_alternatives(&mut self, other: Self, union: impl Fn(Vec) -> Type) { + for (destination, source) in [ + (&mut self.types, other.types), + (&mut self.invoked_properties, other.invoked_properties), + (&mut self.expected_types, other.expected_types), + ] { + for (range, ty) in source { + destination + .entry(range) + .and_modify(|current| { + if current != &ty { + *current = Arc::new(union(vec![(**current).clone(), (*ty).clone()])); + } + }) + .or_insert(ty); + } + } + for (range, callee) in other.overloaded_callees { + match self.overloaded_callees.get_mut(&range) { + Some(current) => current.merge_alternative(callee), + None => { + self.overloaded_callees.insert(range, callee); + } + } + } + } +} + /// Invariants: /// /// * Every module name referenced anywhere MUST be present @@ -1877,15 +1987,7 @@ impl Answers { pub fn get_chosen_overload_trace(&self, range: TextRange) -> Option { let lock = self.trace.as_ref()?.lock(); - match lock.overloaded_callees.get(&range)? { - OverloadedCallee::Resolved { callable } => Some(callable.as_type()), - OverloadedCallee::Candidates { - closest, - is_closest_chosen, - .. - } if *is_closest_chosen => Some(closest.as_type()), - _ => None, - } + lock.overloaded_callees.get(&range)?.chosen_type() } /// Returns all the overload, and the index of a chosen one @@ -1894,18 +1996,7 @@ impl Answers { range: TextRange, ) -> Option<(Vec, Option)> { let lock = self.trace.as_ref()?.lock(); - match lock.overloaded_callees.get(&range)? { - OverloadedCallee::Resolved { callable } => { - Some((vec![callable.callable.clone()], Some(0))) - } - OverloadedCallee::Candidates { all, closest, .. } => { - let chosen_index = all - .iter() - .position(|signature| signature.callable == closest.callable); - let signatures = all.iter().map(|trace| trace.callable.clone()).collect(); - Some((signatures, chosen_index)) - } - } + Some(lock.overloaded_callees.get(&range)?.signatures()) } pub fn add_parent_method_mapping( diff --git a/pyrefly/lib/alt/answers_solver.rs b/pyrefly/lib/alt/answers_solver.rs index 4e2017bc0a..3264bc05d9 100644 --- a/pyrefly/lib/alt/answers_solver.rs +++ b/pyrefly/lib/alt/answers_solver.rs @@ -2010,15 +2010,21 @@ impl ThreadState { /// Captures expression traces and also retains them in the current calculation. pub(crate) fn capture_traces(&self, f: impl FnOnce() -> T) -> (T, Option) { + let (result, traces) = self.defer_traces(f); + if let Some(traces) = &traces { + self.replay_traces(traces); + } + (result, traces) + } + + /// Capture a speculative outcome without publishing its expression traces. + pub(crate) fn defer_traces(&self, f: impl FnOnce() -> T) -> (T, Option) { if self.trace_sink.borrow().is_none() { return (f(), None); } self.install_trace_sink(); let result = f(); let traces = self.take_trace_sink(); - if let Some(traces) = &traces { - self.replay_traces(traces); - } (result, traces) } @@ -2113,6 +2119,8 @@ pub struct AnswersSolver<'ctx, 'answer, Ans: LookupAnswer> { thread_state: &'answer ThreadState, answer_scope: &'answer AnswerScope, comprehension_frame: Option<(usize, Option)>, + /// Constructor probes do not recursively search nested union hints. + constructor_probe: bool, // The base solver is only used to reset the error collector at binding // boundaries. Answers code should generally use the error collector passed // along the call stack instead. @@ -2246,6 +2254,7 @@ impl<'ctx, 'answer, Ans: LookupAnswer> AnswersSolver<'ctx, 'answer, Ans> { thread_state, answer_scope, comprehension_frame: None, + constructor_probe: false, heap, } } @@ -2276,6 +2285,7 @@ impl<'ctx, 'answer, Ans: LookupAnswer> AnswersSolver<'ctx, 'answer, Ans> { thread_state: self.thread_state, answer_scope, comprehension_frame: self.comprehension_frame.clone(), + constructor_probe: self.constructor_probe, base_errors: self.base_errors, exports: self.exports, uniques: self.uniques, @@ -2329,6 +2339,24 @@ impl<'ctx, 'answer, Ans: LookupAnswer> AnswersSolver<'ctx, 'answer, Ans> { self.answers.get_class_fields(cls) } + pub(crate) fn in_constructor_probe(&self) -> bool { + self.constructor_probe + } + + /// Keep nested union searches out of probes without affecting independent solves. + pub(crate) fn for_constructor_probe(&self) -> AnswersSolver<'ctx, '_, Ans> { + let mut solver = self.for_answer_scope(self.answer_scope); + solver.constructor_probe = true; + solver + } + + /// A cached binding must not inherit an enclosing call's speculative precision limit. + fn for_binding(&self) -> AnswersSolver<'ctx, '_, Ans> { + let mut solver = self.for_answer_scope(self.answer_scope); + solver.constructor_probe = false; + solver + } + /// Start an independent lambda-hint or speculative-call inference attempt. pub(crate) fn enter_lambda_binding_context( &self, @@ -2446,13 +2474,12 @@ impl<'ctx, 'answer, Ans: LookupAnswer> AnswersSolver<'ctx, 'answer, Ans> { .cloned() } - /// Capture contextual lambda types owned by the current binding for a speculative call. - pub(crate) fn snapshot_lambda_param_types(&self) -> Vec<(LambdaParamId, Option)> { - let Some(CalcId(_, AnyIdx::Key(owner))) = self.stack().peek() else { - return Vec::new(); - }; - self.bindings() - .owned_lambda_params(owner) + /// Capture only the lambda parameters in the speculative call's argument subtree. + pub(crate) fn snapshot_lambda_param_types( + &self, + parameters: &[LambdaParamId], + ) -> Vec<(LambdaParamId, Option)> { + parameters .iter() .map(|id| (*id, self.get_lambda_param_type(*id))) .collect() @@ -2823,7 +2850,7 @@ impl<'ctx, 'answer, Ans: LookupAnswer> AnswersSolver<'ctx, 'answer, Ans> { let range = K::range_with(idx, self.bindings()); let local_errors = self.error_collector(); - let solve_result = K::solve(self, binding, range, &local_errors); + let solve_result = K::solve(&self.for_binding(), binding, range, &local_errors); // Take accumulated traces. let trace_side_effects = if tracing_enabled { @@ -3049,7 +3076,7 @@ impl<'ctx, 'answer, Ans: LookupAnswer> AnswersSolver<'ctx, 'answer, Ans> { // each result is independently deep-forced before convergence comparison. // Preserving sharing would require identifying the fully forced target // from the final generation and keeping that allocation alive. - let raw_answer = K::solve(self, binding, range, &local_errors) + let raw_answer = K::solve(&self.for_binding(), binding, range, &local_errors) .into_answer(|target| self.get_idx(target).clone()); // Take accumulated traces. Discard during cold iteration (like errors). diff --git a/pyrefly/lib/alt/call.rs b/pyrefly/lib/alt/call.rs index 1fb47a3670..ccb5bddc59 100644 --- a/pyrefly/lib/alt/call.rs +++ b/pyrefly/lib/alt/call.rs @@ -22,9 +22,11 @@ use pyrefly_types::types::CalleeKind; use pyrefly_types::types::TArgs; use pyrefly_types::types::TParams; use pyrefly_util::display::DisplayWithCtx; +use pyrefly_util::owner::Owner; use pyrefly_util::prelude::SliceExt; use pyrefly_util::prelude::VecExt; use pyrefly_util::visit::Visit; +use pyrefly_util::visit::VisitMut; use ruff_python_ast::Arguments; use ruff_python_ast::Expr; use ruff_python_ast::ExprCall; @@ -39,6 +41,7 @@ use vec1::Vec1; use crate::alt::answers::AttributeReferenceKind; use crate::alt::answers::LookupAnswer; +use crate::alt::answers::TraceSideEffects; use crate::alt::answers_solver::AnswersSolver; use crate::alt::attr::NoAccessReason; use crate::alt::callable::ArgMap; @@ -54,18 +57,21 @@ use crate::alt::class::dataclass::ReplaceKind; use crate::alt::expr::ExprCallee; use crate::alt::expr::TypeOrExpr; use crate::alt::nn_module_specials::is_nn_sequential; +use crate::alt::overload::ArgsExpander; use crate::alt::unwrap::HintRef; use crate::alt::unwrap::MAX_HINT_WIDTH; use crate::binding::binding::Key; use crate::binding::binding::LambdaParamId; use crate::config::error_kind::ErrorKind; use crate::error::collector::ErrorCollector; +use crate::error::collector::ModuleDiagnostics; use crate::error::context::ErrorContext; use crate::error::context::TypeCheckContext; use crate::error::context::TypeCheckKind; use crate::solver::solver::OverloadTable; use crate::solver::solver::QuantifiedHandle; use crate::solver::solver::TypeVarSpecializationError; +use crate::solver::solver::VarSnapshot; use crate::types::callable::Callable; use crate::types::callable::ParamList; use crate::types::callable::Params; @@ -87,6 +93,7 @@ use crate::types::types::Forallable; use crate::types::types::Overload; use crate::types::types::OverloadType; use crate::types::types::Type; +use crate::types::types::Var; pub enum CallStyle<'a> { Method(&'a Name), @@ -188,7 +195,7 @@ struct AcceptedIntersectionCall { arg_errors: ErrorCollector, call_errors: ErrorCollector, return_errors: ErrorCollector, - lambda_types: Vec<(LambdaParamId, Option)>, + ctor_targs: Option, } #[derive(Debug, Clone)] @@ -286,6 +293,19 @@ struct ConstructedInstance { } impl ConstructedInstance { + fn diagnostics(&self) -> [&ErrorCollector; 3] { + [&self.errors, &self.arg_errors, &self.return_errors] + } + + /// A hint match requires successful specialization and argument checks. + fn is_match(&self) -> bool { + self.matched_hint + && self.specialization_errors.is_none() + && !self.errors.has_hard() + && !self.arg_errors.has_hard() + && !self.return_errors.has_hard() + } + fn take( self, arguments_range: TextRange, @@ -310,7 +330,206 @@ impl ConstructedInstance { } } +/// The call-local roots and contextual expressions shared by speculative attempts. +#[derive(Clone)] +struct CallSpeculation { + roots: Vec, + lambda_params: Vec, + contextual_ranges: Vec, +} + +impl CallSpeculation { + fn new( + answers: &AnswersSolver, + args: &[CallArg], + keywords: &[CallKeyword], + roots: impl IntoIterator, + ) -> Self { + let mut result = Self { + roots: roots.into_iter().collect(), + lambda_params: Vec::new(), + contextual_ranges: Vec::new(), + }; + result.roots.extend(answers.active_first_use_types()); + for value in argument_values(args, keywords) { + match value.as_type_or_expr() { + TypeOrExpr::Type(ty, _) => result.roots.push(ty.clone()), + TypeOrExpr::Expr(expr) => result.add_expression(answers, expr), + } + } + result + } + + fn add_expression(&mut self, answers: &AnswersSolver, expr: &Expr) { + match expr { + Expr::Lambda(lambda) => { + self.contextual_ranges.push(expr.range()); + if let Some(parameters) = &lambda.parameters { + self.lambda_params.extend( + parameters + .iter_non_variadic_params() + .map(|param| answers.bindings().get_lambda_param_id(param.name())), + ); + for param in parameters.vararg.iter().chain(parameters.kwarg.iter()) { + self.lambda_params + .push(answers.bindings().get_lambda_param_id(¶m.name)); + } + } + } + Expr::ListComp(_) | Expr::SetComp(_) | Expr::DictComp(_) | Expr::Generator(_) => { + self.contextual_ranges.push(expr.range()); + } + _ => {} + } + expr.recurse(&mut |child: &Expr| self.add_expression(answers, child)); + } +} + +fn argument_values<'a, 'b>( + args: &'a [CallArg<'b>], + keywords: &'a [CallKeyword<'b>], +) -> impl Iterator> + 'a { + args.iter() + .map(|arg| match arg { + CallArg::Arg(value) | CallArg::Star(value, _) => *value, + }) + .chain(keywords.iter().map(|keyword| keyword.value)) +} + +/// Caller-owned state is restored after every probe and published once after selection. +struct SpeculationState { + vars: VarSnapshot, + lambda_types: Vec<(LambdaParamId, Option)>, +} + +impl SpeculationState { + fn capture(answers: &AnswersSolver, call: &CallSpeculation) -> Self { + let lambda_types = answers.snapshot_lambda_param_types(&call.lambda_params); + let mut roots = call.roots.iter().collect::>(); + roots.extend(lambda_types.iter().filter_map(|(_, ty)| ty.as_ref())); + Self { + vars: answers.solver().snapshot_reachable_vars(&roots), + lambda_types, + } + } + + fn restore(self, answers: &AnswersSolver) { + answers.solver().restore_vars(self.vars); + answers.restore_lambda_param_types(&self.lambda_types); + } +} + +/// A closed result and its effects remain private until the accepted group is committed. +struct Speculation { + value: T, + state: SpeculationState, + inferred: Vec<(Var, Type)>, + traces: Option, +} + +/// Select a complete match, or preserve the original contextual fallback preference. +fn select_constructor_hint( + args: &[CallArg], + keywords: &[CallKeyword], + hints: &[Type], + attempt: &impl Fn(&[CallArg], &[CallKeyword], Option<&Type>) -> Speculation, +) -> Speculation { + let mut fallback = None; + for hint in hints { + let result = attempt(args, keywords, Some(hint)); + if result.value.is_match() { + return result; + } + if !result.value.errors.has_hard() && fallback.is_none() { + fallback = Some(result); + } + } + fallback.unwrap_or_else(|| attempt(args, keywords, None)) +} + +impl Speculation { + fn commit(self, answers: &AnswersSolver) -> T { + self.state.restore(answers); + if let Some(traces) = self.traces { + answers.trace_state().replay_traces(&traces); + } + self.value + } +} + impl<'ctx, 'answer, Ans: LookupAnswer> AnswersSolver<'ctx, 'answer, Ans> { + /// Materialize traces before restoring caller-owned inference state. + /// `f` must materialize result types that depend on the attempt's variable assignments. + fn speculate(&self, call: &CallSpeculation, f: impl FnOnce() -> T) -> Speculation { + let before = SpeculationState::capture(self, call); + let _lambda_context = self.enter_lambda_binding_context([]); + let (value, mut traces) = self.trace_state().defer_traces(f); + let inferred = self.solver().inferred_bounds(&before.vars, self); + let mut state = SpeculationState::capture(self, call); + for (_, ty) in &mut state.lambda_types { + if let Some(ty) = ty { + *ty = self.solver().expand(ty.clone(), self); + } + } + if let Some(traces) = &mut traces { + traces.recurse_mut(&mut |ty| self.solver().expand_mut(ty, self)); + } + before.restore(self); + Speculation { + value, + state, + inferred, + traces, + } + } + + /// Publish one validated caller state and merge the accepted contextual effects once. + fn merge_accepted( + &self, + call: &CallSpeculation, + attempts: Vec>, + diagnostics: impl Fn(&T) -> [&ErrorCollector; 3], + ) -> Vec { + let mut attempts = attempts.into_iter(); + let mut first = attempts.next().expect("an accepted group is nonempty"); + let mut lambda_types = SmallMap::new(); + let mut snapshots = Vec::new(); + let mut values = Vec::new(); + let mut traces = TraceSideEffects::default(); + let mut add_effects = |attempt: &mut Speculation| { + for (id, ty) in &attempt.state.lambda_types { + let types = lambda_types.entry(*id).or_insert_with(Vec::new); + types.extend(ty.iter().cloned()); + } + snapshots.push(diagnostics(&attempt.value).map(ErrorCollector::take)); + if let Some(effects) = attempt.traces.take() { + traces.merge_alternatives(effects, |types| self.unions(types)); + } + }; + add_effects(&mut first); + for mut attempt in attempts { + add_effects(&mut attempt); + values.push(attempt.value); + } + first.state.lambda_types = lambda_types + .into_iter() + .map(|(id, types)| (id, (!types.is_empty()).then(|| self.unions(types)))) + .collect(); + first.traces = Some(traces); + for (role, destination) in diagnostics(&first.value).into_iter().enumerate() { + ModuleDiagnostics::replay_contexts( + &snapshots + .iter() + .map(|snapshot| snapshot[role].clone()) + .collect::>(), + destination, + &call.contextual_ranges, + ); + } + values.insert(0, first.commit(self)); + values + } + fn error_call_target( &self, errors: &ErrorCollector, @@ -1214,43 +1433,170 @@ impl<'ctx, 'answer, Ans: LookupAnswer> AnswersSolver<'ctx, 'answer, Ans> { } } - /// Selects a constructor outcome without publishing its diagnostics. - /// Prefers the first hint match without hard call errors or specialization failures. - /// Otherwise keeps the first outcome without hard call errors, or constructs without a hint. + /// Selects a complete match without publishing rejected inference effects or diagnostics. + /// Each argument expansion must match some hint member; otherwise retain the original fallback. /// Hints wider than `MAX_HINT_WIDTH` are ignored. fn construct_with_hint( &self, + targs: &TArgs, + args: &[CallArg], + keywords: &[CallKeyword], hint: Option, - construct: impl Fn(Option<&Type>) -> ConstructedInstance, + construct: impl Fn( + &AnswersSolver, + &[CallArg], + &[CallKeyword], + Option<&Type>, + ) -> ConstructedInstance, ) -> ConstructedInstance { - if let Some(hint) = hint - && let hints = hint.types() - && hints.len() <= MAX_HINT_WIDTH - { - let mut ret_no_match_hint = None; - for member_hint in hints.iter() { - let ret = construct(Some(member_hint)); - if ret.errors.has_hard() { - continue; - } - if ret.matched_hint && ret.specialization_errors.is_none() { - // Take the first successful match. We require the result to be assignable to the - // hint so that, in a case like `x: list[X] | None = [XChild()]`, we choose the - // `list[X]` branch with contextually typed results. - return ret; - } - if ret_no_match_hint.is_none() { - ret_no_match_hint = Some(ret); - } + let Some(hint) = hint.filter(|hint| hint.types().len() <= MAX_HINT_WIDTH) else { + return construct(self, args, keywords, None); + }; + if let [hint] = hint.types() { + let result = construct(self, args, keywords, Some(hint)); + return if !result.errors.has_hard() { + result + } else { + construct(self, args, keywords, None) + }; + } + // A probe checks nested expressions normally, but does not start another search. + // This bounds recursive speculation without sharing fuel between sibling calls. + if self.in_constructor_probe() { + return construct(self, args, keywords, None); + } + let call = CallSpeculation::new( + self, + args, + keywords, + targs.as_slice().iter().chain(hint.types()).cloned(), + ); + let solver = self.for_constructor_probe(); + let attempt = |args: &[CallArg], keywords: &[CallKeyword], hint: Option<&Type>| { + solver.speculate(&call, || { + let mut result = construct(&solver, args, keywords, hint); + result.ty = self.solver().expand(result.ty, self); + result + }) + }; + let unexpanded = select_constructor_hint(args, keywords, hint.types(), &attempt); + if unexpanded.value.is_match() { + return unexpanded.commit(self); + } + solver + .distribute_constructor(&call, args, keywords, hint.types(), &construct) + .unwrap_or_else(|| unexpanded.commit(self)) + } + + /// Each expanded input must have a complete hint match before any effect is published. + fn distribute_constructor( + &self, + call: &CallSpeculation, + args: &[CallArg], + keywords: &[CallKeyword], + hints: &[Type], + construct: &impl Fn( + &AnswersSolver, + &[CallArg], + &[CallKeyword], + Option<&Type>, + ) -> ConstructedInstance, + ) -> Option { + let mut expander = ArgsExpander::new(args.to_vec(), keywords.to_vec(), self); + let owner = Owner::new(); + let expansion_errors = self.error_collector(); + let mut expansion_traces = TraceSideEffects::default(); + loop { + let expansion = self.speculate(call, || expander.expand(&expansion_errors, &owner)); + if let Some(traces) = expansion.traces { + expansion_traces.merge_alternatives(traces, |types| self.unions(types)); } - if let Some(ret) = ret_no_match_hint { - // Even if none of the results were assignable to their hints, we still keep the - // first contextually typed result if it only produced specialization errors. - return ret; + let arg_lists = expansion.value?; + if expansion_errors.has_hard() { + return None; + } + let mut batch_call = call.clone(); + for (args, keywords) in &arg_lists { + batch_call + .roots + .extend(argument_values(args, keywords).filter_map(|value| { + match value.as_type_or_expr() { + TypeOrExpr::Type(ty, _) => Some(ty.clone()), + TypeOrExpr::Expr(_) => None, + } + })); + } + let batch = self.speculate(&batch_call, || { + let mut probes = + self.match_constructor_batch(&batch_call, &arg_lists, hints, construct)?; + let first = probes.first_mut().expect("argument expansion is nonempty"); + expansion_errors.take().replay(&first.value.arg_errors); + first + .traces + .get_or_insert_default() + .merge_alternatives(expansion_traces.clone(), |types| self.unions(types)); + let mut outcomes = self + .merge_accepted(&batch_call, probes, ConstructedInstance::diagnostics) + .into_iter(); + let mut result = outcomes.next().expect("argument expansion is nonempty"); + result.ty = self.unions( + iter::once(result.ty) + .chain(outcomes.map(|outcome| outcome.ty)) + .collect(), + ); + Some(result) + }); + if batch.value.is_some() { + return batch.commit(self); } } - // If the hint is too wide or always produces non-specialization errors, don't use it. - construct(None) + } + + /// Join caller evidence, then validate all alternatives against that common caller state. + fn match_constructor_batch( + &self, + call: &CallSpeculation, + arg_lists: &[(Vec, Vec)], + hints: &[Type], + construct: &impl Fn( + &AnswersSolver, + &[CallArg], + &[CallKeyword], + Option<&Type>, + ) -> ConstructedInstance, + ) -> Option>> { + let attempt = |args: &[CallArg], keywords: &[CallKeyword], hint: Option<&Type>| { + self.speculate(call, || { + let mut result = construct(self, args, keywords, hint); + result.ty = self.solver().expand(result.ty, self); + result + }) + }; + let probe_all = || { + arg_lists + .iter() + .map(|(args, keywords)| { + let probe = select_constructor_hint(args, keywords, hints, &attempt); + probe.value.is_match().then_some(probe) + }) + .collect::>>() + }; + let probes = probe_all()?; + let mut bounds = SmallMap::new(); + for probe in &probes { + for (var, ty) in &probe.inferred { + bounds.entry(*var).or_insert_with(Vec::new).push(ty.clone()); + } + } + if bounds.is_empty() { + return Some(probes); + } + for (var, types) in bounds { + if !self.is_subset_eq(&self.unions(types), &Type::Var(var)) { + return None; + } + } + probe_all() } fn construct_class( @@ -1347,18 +1693,24 @@ impl<'ctx, 'answer, Ans: LookupAnswer> AnswersSolver<'ctx, 'answer, Ans> { let prepared = ConstructorArguments::new(hint); let (args, keywords) = prepared.prepare(args, keywords); - self.construct_with_hint(hint, |hint| { - self.construct_class_inner( - cls.clone(), - constructor_kind.clone(), - &args, - &keywords, - arguments_range, - callee_range, - context, - hint, - ) - }) + self.construct_with_hint( + cls.targs(), + &args, + &keywords, + hint, + |solver, args, keywords, hint| { + solver.construct_class_inner( + cls.clone(), + constructor_kind.clone(), + args, + keywords, + arguments_range, + callee_range, + context, + hint, + ) + }, + ) .take( arguments_range, errors, @@ -1712,16 +2064,22 @@ impl<'ctx, 'answer, Ans: LookupAnswer> AnswersSolver<'ctx, 'answer, Ans> { let hint = HintRef::filter_for_constructor(hint, typed_dict.targs()); let prepared = ConstructorArguments::new(hint); let (args, keywords) = prepared.prepare(args, keywords); - self.construct_with_hint(hint, |hint| { - self.construct_typed_dict_inner( - typed_dict.clone(), - &args, - &keywords, - arguments_range, - context, - hint, - ) - }) + self.construct_with_hint( + typed_dict.targs(), + &args, + &keywords, + hint, + |solver, args, keywords, hint| { + solver.construct_typed_dict_inner( + typed_dict.clone(), + args, + keywords, + arguments_range, + context, + hint, + ) + }, + ) .take( arguments_range, errors, @@ -2370,81 +2728,73 @@ impl<'ctx, 'answer, Ans: LookupAnswer> AnswersSolver<'ctx, 'answer, Ans> { self.intersect_guarantees(&guarantees) }; let hint_ty = hint.map(|hint| self.unions(hint.types().to_vec())); - let mut roots = targets.iter().map(|member| &member.ty).collect::>(); - roots.extend( + let call_site = CallSpeculation::new( + self, + &args, + &keywords, targets .iter() - .flat_map(|member| &member.constructor_witnesses), + .map(|member| &member.ty) + .chain( + targets + .iter() + .flat_map(|member| &member.constructor_witnesses), + ) + .chain(hint_ty.as_ref()) + .chain( + ctor_targs + .as_ref() + .into_iter() + .flat_map(|targs| targs.as_slice()), + ) + .cloned(), ); - for arg in &args { - let (CallArg::Arg(value) | CallArg::Star(value, _)) = arg; - if let TypeOrExpr::Type(ty, _) = value.as_type_or_expr() { - roots.push(ty); - } - } - for keyword in &keywords { - if let TypeOrExpr::Type(ty, _) = keyword.value.as_type_or_expr() { - roots.push(ty); - } - } - roots.extend(hint_ty.as_ref()); - // Deferred containers can capture the unpinned answer whose first use is this call. - let first_use_types = self.active_first_use_types(); - roots.extend(first_use_types.iter()); - let lambda_snapshot = self.snapshot_lambda_param_types(); - roots.extend(lambda_snapshot.iter().filter_map(|(_, ty)| ty.as_ref())); let mut accepted = Vec::new(); let mut failures = Vec::new(); for (index, member) in targets.iter().enumerate() { - let _lambda_context = self.enter_lambda_binding_context([]); - let snapshot = self.solver().snapshot_reachable_vars(&roots); - let arg_errors = self.error_collector(); - let attempt_errors = self.error_collector(); - let attempt_return_errors = self.error_collector(); - let mut attempt_targs = ctor_targs.as_ref().map(|targs| (**targs).clone()); - let outcome = solver.call_infer_with_callee_range( - member.target.clone(), - &args, - &keywords, - arguments_range, - callee_range, - &arg_errors, - &attempt_errors, - &attempt_return_errors, - context, - hint_ty.as_ref().map(HintRef::soft), - attempt_targs.as_mut(), - ); - if attempt_errors.has_hard() { - failures.push((index, arg_errors, attempt_errors, attempt_return_errors)); - } else { + let attempt = solver.speculate(&call_site, || { + let arg_errors = self.error_collector(); + let attempt_errors = self.error_collector(); + let attempt_return_errors = self.error_collector(); + let mut attempt_targs = ctor_targs.as_ref().map(|targs| (**targs).clone()); + let outcome = solver.call_infer_with_callee_range( + member.target.clone(), + &args, + &keywords, + arguments_range, + callee_range, + &arg_errors, + &attempt_errors, + &attempt_return_errors, + context, + hint_ty.as_ref().map(HintRef::soft), + attempt_targs.as_mut(), + ); // Materialize each correlated world before its variable state is rolled back. let results = self.solver().per_row(&outcome.overload_table, || { add_constructor_witnesses(self.solver().expand(outcome.ty.clone(), self), index) }); let ty = Type::combine_overload_results(results.into_vec(), self.heap, self) .expect("an applicable call has at least one result"); - let lambda_types = self - .snapshot_lambda_param_types() - .into_iter() - .map(|(id, ty)| (id, ty.map(|ty| self.solver().expand(ty, self)))) - .collect::>(); - accepted.push(AcceptedIntersectionCall { + AcceptedIntersectionCall { index, ty, arg_errors, call_errors: attempt_errors, return_errors: attempt_return_errors, - lambda_types, - }); + ctor_targs: attempt_targs, + } + }); + if attempt.value.call_errors.has_hard() { + failures.push(attempt.value); + } else { + accepted.push(attempt); } - self.solver().restore_vars(snapshot); - self.restore_lambda_param_types(&lambda_snapshot); } let Some((representative, first)) = accepted .iter() .enumerate() - .min_by_key(|(_, attempt)| attempt.arg_errors.len_hard()) + .min_by_key(|(_, attempt)| attempt.value.arg_errors.len_hard()) else { let mut builder = call_errors .error_builder( @@ -2453,7 +2803,14 @@ impl<'ctx, 'answer, Ans: LookupAnswer> AnswersSolver<'ctx, 'answer, Ans> { "No intersection member accepts this call".to_owned(), ) .with_context(self, context); - for (index, arg_errors, attempt_errors, attempt_return_errors) in failures { + for AcceptedIntersectionCall { + index, + arg_errors, + call_errors: attempt_errors, + return_errors: attempt_return_errors, + .. + } in failures + { builder = builder .with_detail(format!(" {}", self.display(targets[index].ty.clone()))) .with_errors_as_details(attempt_errors); @@ -2463,8 +2820,8 @@ impl<'ctx, 'answer, Ans: LookupAnswer> AnswersSolver<'ctx, 'answer, Ans> { builder.emit(); return CallOutcome::of_ty(self.heap.mk_any_error()); }; - let first_index = first.index; - let fewest_arg_errors = first.arg_errors.len_hard(); + let first_index = first.value.index; + let fewest_arg_errors = first.value.arg_errors.len_hard(); if accepted.len() == 1 { self.report_call_deprecation( &targets[first_index].target, @@ -2497,62 +2854,45 @@ impl<'ctx, 'answer, Ans: LookupAnswer> AnswersSolver<'ctx, 'answer, Ans> { } return outcome; } - // Commit one applicable signature's ordinary inference, without letting it reject siblings. - let ignored = self.error_swallower(); - let first = solver.call_infer_with_callee_range( - targets[first_index].target.clone(), - &args, - &keywords, - arguments_range, - callee_range, - &ignored, - &ignored, - &ignored, - context, - hint_ty.as_ref().map(HintRef::soft), - ctor_targs, - ); - let first_results = self.solver().per_row(&first.overload_table, || { - add_constructor_witnesses(self.solver().expand(first.ty.clone(), self), first_index) - }); - accepted[representative].ty = - Type::combine_overload_results(first_results.into_vec(), self.heap, self) - .expect("an applicable call has at least one result"); let mut results = Vec::new(); - let mut lambda_types = SmallMap::new(); - for AcceptedIntersectionCall { - index, - ty, - arg_errors, - call_errors: attempt_errors, - return_errors: attempt_return_errors, - lambda_types: params, - } in accepted - { + for attempt in &accepted { self.report_call_deprecation( - &targets[index].target, + &targets[attempt.value.index].target, callee_range.unwrap_or(arguments_range), errors, context, ); - results.push(ty); - let publish_context = arg_errors.len_hard() == fewest_arg_errors; - if index == first_index { - errors.extend(arg_errors); - } - call_errors.extend(attempt_errors); - return_errors.extend(attempt_return_errors); - if publish_context { - for (id, ty) in params { - if let Some(ty) = ty { - lambda_types.entry(id).or_insert_with(Vec::new).push(ty); - } + results.push(attempt.value.ty.clone()); + } + // Return guarantees include every applicable signature. Contextual effects come from + // the best argument checks, with the representative's caller inference state. + accepted.swap(0, representative); + accepted.retain(|attempt| attempt.value.arg_errors.len_hard() == fewest_arg_errors); + if fewest_arg_errors != 0 { + // No contextual check succeeded. Report one complete failure instead of + // intersecting different hard errors into an empty diagnostic set. + accepted.truncate(1); + } + let outcomes = self.merge_accepted(&call_site, accepted, |outcome| { + [ + &outcome.call_errors, + &outcome.arg_errors, + &outcome.return_errors, + ] + }); + for outcome in outcomes { + if outcome.index == first_index { + errors.extend(outcome.arg_errors); + call_errors.extend(outcome.call_errors); + return_errors.extend(outcome.return_errors); + if let Some(ctor_targs) = ctor_targs { + *ctor_targs = outcome + .ctor_targs + .expect("constructor arguments are retained"); } + break; } } - for (id, types) in lambda_types { - self.set_lambda_param_type(id, self.unions(types)); - } CallOutcome::of_ty(self.intersect_guarantees(&results)) } diff --git a/pyrefly/lib/alt/overload.rs b/pyrefly/lib/alt/overload.rs index f827a0a00b..b249d8441a 100644 --- a/pyrefly/lib/alt/overload.rs +++ b/pyrefly/lib/alt/overload.rs @@ -42,6 +42,7 @@ use crate::alt::callable::CallKeyword; use crate::alt::callable::CallValue; use crate::alt::callable::CallWithTypes; use crate::alt::callable::ReturnTypeResolutionError; +use crate::alt::callable::contains_argument_bindings; use crate::alt::expr::TypeOrExpr; use crate::alt::unwrap::HintRef; use crate::config::error_kind::ErrorKind; @@ -152,24 +153,21 @@ impl<'a, Ans: LookupAnswer> ArgsExpander<'a, Ans> { return None; } }; - // These expressions need parameter context and cannot produce an expandable type. + // Expressions with bindings retain their syntax for contextual checks, including + // lambdas nested in conditional expressions or calls. + let inference_errors = self.solver.error_collector(); let expanded_types = match value.as_type_or_expr() { - TypeOrExpr::Expr( - Expr::Lambda(_) - | Expr::List(_) - | Expr::Dict(_) - | Expr::Set(_) - | Expr::ListComp(_) - | Expr::DictComp(_) - | Expr::SetComp(_) - | Expr::Generator(_), - ) => Vec::new(), - _ => self.expand_type(value.infer(self.solver, errors)), + TypeOrExpr::Expr(expr) if contains_argument_bindings(expr) => Vec::new(), + TypeOrExpr::Expr(Expr::List(_) | Expr::Dict(_) | Expr::Set(_)) => Vec::new(), + _ => self.expand_type(value.infer(self.solver, &inference_errors)), }; if expanded_types.is_empty() { - // Nothing to expand here, try the next argument. + // Unchanged expressions are checked with each candidate's context. Their probe + // diagnostics are not authoritative, particularly for nested lambdas. self.expand(errors, owner) } else { + // Expanded expressions become types and will not be checked again. + errors.extend(inference_errors); let expanded_types = expanded_types.into_map(|t| owner.push(t)); let mut new_arg_lists = Vec::new(); for (posargs, keywords) in self.arg_lists.iter() { diff --git a/pyrefly/lib/error/collector.rs b/pyrefly/lib/error/collector.rs index 9ef812223c..b58f055040 100644 --- a/pyrefly/lib/error/collector.rs +++ b/pyrefly/lib/error/collector.rs @@ -19,6 +19,7 @@ use pyrefly_util::lock::Mutex; use ruff_text_size::Ranged; use ruff_text_size::TextRange; use ruff_text_size::TextSize; +use starlark_map::small_set::SmallSet; use crate::config::error::ErrorConfig; use crate::config::error_kind::Severity; @@ -54,14 +55,50 @@ impl ModuleDiagnostics { /// Replay finalized diagnostics into an inference attempt while retaining the snapshot. pub(crate) fn replay(&self, errors: &ErrorCollector) { + self.replay_filtered(errors, |_| true); + } + + /// Replay diagnostics that remain valid in the receiving context. + pub(crate) fn replay_filtered( + &self, + errors: &ErrorCollector, + mut keep: impl FnMut(&Error) -> bool, + ) { if errors.is_active() { let replayed = ModuleErrors { clean: true, - items: self.0.to_vec(), + items: self.0.iter().filter(|error| keep(error)).cloned().collect(), }; errors.errors.lock().extend(replayed); } } + + /// Contextual diagnostics must agree in range, kind, and message across accepted checks. + /// Deprecation and directives remain visible even when only one context emits them. + pub(crate) fn replay_contexts( + alternatives: &[Self], + destination: &ErrorCollector, + contextual_ranges: &[TextRange], + ) { + let Some((first, rest)) = alternatives.split_first() else { + return; + }; + let mut common = first.iter().collect::>(); + for alternative in rest { + let current = alternative.iter().collect::>(); + common.retain(|error| current.contains(error)); + } + for alternative in alternatives { + alternative.replay_filtered(destination, |error| { + error.error_kind() == ErrorKind::Deprecated + || error.error_kind().is_directive() + || !contextual_ranges + .iter() + .any(|range| range.contains_range(error.range())) + || common.contains(error) + }); + } + } } impl ModuleErrors { @@ -875,6 +912,67 @@ mod tests { errors.error_builder(range, kind, msg).emit(); } + #[test] + fn test_contextual_diagnostic_agreement() { + let module = ModuleInfo::new( + ModuleName::from_str("main"), + ModulePath::filesystem(PathBuf::from("main.py")), + Arc::new("0123456789".to_owned()), + ); + let range = TextRange::new(TextSize::new(1), TextSize::new(2)); + let outside = TextRange::new(TextSize::new(8), TextSize::new(9)); + let first = ErrorCollector::new(module.dupe(), ErrorStyle::Delayed); + let second = ErrorCollector::new(module.dupe(), ErrorStyle::Delayed); + for collector in [&first, &second] { + add( + collector, + range, + ErrorKind::RedundantCast, + "common".to_owned(), + ); + } + add( + &first, + range, + ErrorKind::IncompatibleComparison, + "int comparison".to_owned(), + ); + add( + &second, + range, + ErrorKind::IncompatibleComparison, + "str comparison".to_owned(), + ); + add( + &first, + range, + ErrorKind::UnnecessaryTypeConversion, + "branch only".to_owned(), + ); + add( + &first, + range, + ErrorKind::Deprecated, + "deprecated".to_owned(), + ); + add( + &first, + outside, + ErrorKind::RedundantCast, + "outside".to_owned(), + ); + let result = ErrorCollector::new(module, ErrorStyle::Delayed); + ModuleDiagnostics::replay_contexts(&[first.take(), second.take()], &result, &[range]); + assert_eq!( + result + .finish() + .iter() + .map(Error::msg_header) + .collect::>(), + vec!["common", "deprecated", "outside"], + ); + } + #[test] fn test_error_builder_merges_details_and_metadata() { let mi = ModuleInfo::new( diff --git a/pyrefly/lib/solver/solver.rs b/pyrefly/lib/solver/solver.rs index 3416c5718d..5db914ad46 100644 --- a/pyrefly/lib/solver/solver.rs +++ b/pyrefly/lib/solver/solver.rs @@ -41,6 +41,7 @@ use pyrefly_util::prelude::SliceExt; use pyrefly_util::recurser::Guard; use pyrefly_util::recurser::Recurser; use pyrefly_util::uniques::UniqueFactory; +use pyrefly_util::visit::Visit; use pyrefly_util::visit::VisitMut; use ruff_python_ast::name::Name; use ruff_text_size::TextRange; @@ -919,6 +920,32 @@ impl Solver { ) } + /// Materialize evidence inferred for the caller's unfinished variables by a probe. + /// Replaying their combined lower bounds rechecks restrictions in the accepted context. + pub(crate) fn inferred_bounds( + &self, + before: &VarSnapshot, + env: &dyn TypeEnv, + ) -> Vec<(Var, Type)> { + before + .0 + .iter() + .filter(|(_, state)| { + matches!( + state.variable, + Variable::Quantified { .. } + | Variable::Unwrap(_) + | Variable::PartialQuantified(_) + | Variable::PartialContained(_) + ) + }) + .filter_map(|(var, _)| { + let ty = self.expand(Type::Var(*var), env); + (ty != Type::Var(*var) && !ty.may_contain_placeholder_var()).then_some((*var, ty)) + }) + .collect() + } + /// Returns the vars in `pending` together with every var reachable from them through /// union-find parents and the vars referenced by current bounds or answers. fn reachable_vars(variables: &Variables, mut pending: Vec) -> SmallSet { @@ -934,7 +961,9 @@ impl Solver { } VariableNode::Root(variable, _) => match variable.as_ref() { Variable::Quantified { - quantified, bounds, .. + quantified, + bounds, + bound_override, } => { pending.extend( Type::Quantified(Box::new(quantified.clone())).collect_all_vars(), @@ -946,6 +975,10 @@ impl Solver { .chain(&bounds.upper) .flat_map(Type::collect_all_vars), ); + if let Some(restriction) = bound_override { + restriction + .recurse(&mut |ty: &Type| pending.extend(ty.collect_all_vars())); + } } Variable::Unwrap(bounds) => pending.extend( bounds @@ -954,7 +987,15 @@ impl Solver { .chain(&bounds.upper) .flat_map(Type::collect_all_vars), ), - Variable::Answer { ty, .. } => pending.extend(ty.collect_all_vars()), + Variable::Answer { ty, restricted, .. } => { + pending.extend(ty.collect_all_vars()); + if let Some(restricted) = restricted { + pending.extend( + Type::Quantified(Box::new(restricted.param.clone())) + .collect_all_vars(), + ); + } + } Variable::PartialQuantified(quantified) => pending .extend(Type::Quantified(Box::new(quantified.clone())).collect_all_vars()), Variable::PartialContained(_) | Variable::Recursive => {} @@ -4851,6 +4892,51 @@ mod tests { quantified_with_restriction(kind, index, Restriction::Unrestricted) } + #[test] + fn speculative_inference_snapshot_restores_restricted_answers() { + let solver = Solver::new(SolverConfig::default()); + let uniques = UniqueFactory::new(); + let inner = Var::new(&uniques); + let outer = Var::new(&uniques); + let param = quantified_with_restriction( + QuantifiedKind::TypeVar, + 0, + Restriction::Bound(Type::Var(inner)), + ); + { + let mut variables = solver.variables.lock(); + variables.insert_fresh(inner, Variable::answer(Type::None)); + variables.insert_fresh( + outer, + Variable::Answer { + ty: Type::None, + frozen: false, + restricted: Some(Box::new(RestrictedAnswer { + error: Some(TypeVarSpecializationError::BadBoundSpecialization { + name: param.name().clone(), + got: Type::None, + want: Type::Var(inner), + }), + param, + })), + }, + ); + } + let snapshot = solver.snapshot_reachable_vars(&[&Type::Var(outer)]); + { + let variables = solver.variables.lock(); + variables.update(inner, Variable::answer(Type::any_implicit())); + variables.update(outer, Variable::answer(Type::None)); + } + solver.restore_vars(snapshot); + let variables = solver.variables.lock(); + assert!(matches!(&*variables.get(inner), Variable::Answer { ty, .. } if ty.is_none())); + assert!(matches!( + &*variables.get(outer), + Variable::Answer { restricted: Some(restricted), .. } if restricted.error.is_some() + )); + } + fn quantified_with_restriction( kind: QuantifiedKind, index: u32, diff --git a/pyrefly/lib/test/constructors.rs b/pyrefly/lib/test/constructors.rs index 34a457c68b..b60ba5e6be 100644 --- a/pyrefly/lib/test/constructors.rs +++ b/pyrefly/lib/test/constructors.rs @@ -9,6 +9,7 @@ use pyrefly_python::dunder; use pyrefly_types::callable::Callable; use pyrefly_types::callable::Params; use pyrefly_types::function::FuncMetadata; +use pyrefly_util::thread_pool::ThreadCount; use ruff_text_size::TextRange; use crate::alt::callable::CallArg; @@ -16,6 +17,7 @@ use crate::solver::solver::ArgumentKey; use crate::test::util::TestEnv; use crate::test::util::get_class; use crate::test::util::mk_state; +use crate::test::util::testcase_for_macro; use crate::testcase; #[test] @@ -78,6 +80,444 @@ fn test_explicit_new_receiver_with_ellipsis_inputs() { }) .expect("the test module has been solved"); } +// Regression tests for https://github.com/facebook/pyrefly/issues/1134. + +testcase!( + test_constructor_union_distribution, + r#" +from typing import assert_type + +class A[T]: + def __init__(self, value: T) -> None: ... + +class Pair[T, U]: + def __init__(self, x: T, y: U) -> None: ... + +class Same[T]: + def __init__(self, x: T, y: T) -> None: ... + +def f(value: int | str, other: int | str) -> None: + positional: A[int] | A[str] = A(value) + keyword: A[int] | A[str] = A(value=value) + independent: Pair[int, int] | Pair[int, str] | Pair[str, int] | Pair[str, str] = Pair(value, other) + mixed_keywords: Pair[int, int] | Pair[int, str] | Pair[str, int] | Pair[str, str] = Pair(value, y=other) + # Independent arguments include mixed pairs, not just (int, int) and (str, str). + invalid: Same[int] | Same[str] = Same(value, other) # E: `Same[int | str]` is not assignable to `Same[int] | Same[str]` + assert_type(positional, A[int] | A[str]) + assert_type(keyword, A[int] | A[str]) + +def unpacked(args: tuple[int | str]) -> None: + result: A[int] | A[str] = A(*args) +"#, +); + +testcase!( + test_constructor_union_distribution_bool_and_enum, + r#" +from enum import Enum +from typing import Literal + +class A[T]: + def __init__(self, value: T) -> None: ... + +class Choice(Enum): + INT = 1 + STR = "s" + +def f(flag: bool, choice: Choice) -> None: + boolean: A[Literal[True]] | A[Literal[False]] = A(flag) + enumeration: A[Literal[Choice.INT]] | A[Literal[Choice.STR]] = A(choice) +"#, +); + +testcase!( + test_constructor_union_distribution_limit, + r#" +from typing import Literal + +class Pair[T, U]: + def __init__(self, x: T, y: U) -> None: ... + +type Low = Literal[0, 1, 2, 3, 4] +type High = Literal[5, 6, 7, 8, 9] +type Hints = Pair[Low, Low] | Pair[Low, High] | Pair[High, Low] | Pair[High, High] +type Eight = Literal[0, 1, 2, 3, 4, 5, 6, 7] + +def within_limit(x: Eight, y: Eight) -> None: + result: Hints = Pair(x, y) + +def exceeds_limit(x: Low | High, y: Low | High) -> None: + # The expansion budget counts partial batches as well as complete combinations. + result: Hints = Pair(x, y) # E: is not assignable +"#, +); + +testcase!( + test_constructor_union_distribution_diagnostics, + TestEnv::new().enable_deprecated_error(), + r#" +from typing import cast +from typing_extensions import deprecated + +class A[T]: + @deprecated("old constructor") + def __init__(self, value: T) -> None: ... + +class B[T]: + def __init__(self, value: T, other: object = None) -> None: ... + +@deprecated("old factory") +def old_factory() -> int | str: ... + +def requires_int(value: int) -> object: ... + +def f(value: int | str) -> None: + constructor: A[int] | A[str] = A(value) # E: `A.__init__` is deprecated + factory: B[int] | B[str] = B(old_factory()) # E: `old_factory` is deprecated + redundant_cast: B[int] | B[str] = B(cast(int | str, value)) # E: Redundant cast + keyword_factory: B[int] | B[str] = B(value=old_factory()) # E: `old_factory` is deprecated + keyword_cast: B[int] | B[str] = B(value=cast(int | str, value)) # E: Redundant cast + untouched: B[int] | B[str] = B(value, old_factory()) # E: `old_factory` is deprecated + invalid: B[int] | B[str] = B(value, requires_int("bad")) # E: `B[int | str]` is not assignable # E: is not assignable to parameter `value` +"#, +); + +testcase!( + test_constructor_union_distribution_rejected_batches, + TestEnv::new().enable_deprecated_error(), + r#" +from typing_extensions import deprecated + +class Pair[T, U]: + @deprecated("old constructor") + def __init__(self, x: T, y: U) -> None: ... + +def f(x: int | str, y: int | str) -> None: + # Expanding x alone fails. Expanding y as well accepts all four combinations. + pair: Pair[int, int | str] | Pair[str, int] | Pair[str, str] = Pair(x, y) # E: `Pair.__init__` is deprecated + +class A[T]: + def __init__(self, value: T) -> None: ... + +def g(value: int | str | bytes) -> None: + # The bytes expansion must fail without retaining errors from rejected hints. + result: A[int] | A[str] = A(value) # E: `A[bytes | int | str]` is not assignable to `A[int] | A[str]` +"#, +); + +testcase!( + test_constructor_union_distribution_lambda_context, + TestEnv::new().enable_implicit_any_lambda_error(), + r#" +from typing import Callable + +class A[T]: + def __init__(self, callback: Callable[[T], None], value: T) -> None: ... + +class Box[T]: + def __init__(self, value: T) -> None: ... + +class Wrapped[T]: + def __init__(self, callback: Box[Callable[[T], None]], value: T) -> None: ... + +def f(value: int | str) -> None: + positional: A[int] | A[str] = A(lambda arg: None, value) + keyword: A[int] | A[str] = A(value=value, callback=lambda arg: None) + nested: Wrapped[int] | Wrapped[str] = Wrapped(Box(lambda arg: None), value) +"#, +); + +testcase!( + test_constructor_union_distribution_conditional_lambda, + r#" +from typing import Callable + +class A[T]: + def __init__(self, cb: Callable[[T], None] | None, value: T) -> None: ... + +def f(value: int | str, condition: bool) -> None: + valid: A[int] | A[str] = A((lambda arg: None) if condition else None, value) + invalid: A[int] | A[str] = A((lambda arg: arg.upper()) if condition else None, value) # E: is not assignable +"#, +); + +testcase!( + test_constructor_union_distribution_contextual_cast, + TestEnv::new().enable_deprecated_error(), + r#" +from typing import Callable, cast +from typing_extensions import deprecated + +class A[T]: + def __init__(self, callback: Callable[[T], int], value: T) -> None: ... + +@deprecated("old callback") +def old() -> int: ... + +def f(value: int | str, fixed: int) -> None: + necessary: A[int] | A[str] = A(lambda arg: cast(int, arg), value) + keyword: A[int] | A[str] = A(value=value, callback=lambda arg: cast(int, arg)) + redundant: A[int] | A[str] = A(lambda arg: cast(int, fixed), value) # E: Redundant cast + deprecated: A[int] | A[str] = A(lambda arg: cast(int, arg) + old(), value) # E: `old` is deprecated +"#, +); + +// Expansion probes retain ordinary contextual checks but do not distribute nested calls. +testcase!( + test_constructor_union_distribution_contextual_nested_limit, + TestEnv::new().enable_implicit_any_lambda_error(), + r#" +from typing import Callable + +class Node[T]: + def __init__(self, cb: Callable[[T], None], value: T, + child: Node[int] | Node[str] | None = None) -> None: ... + +def f(value: int | str) -> None: + result: Node[int] | Node[str] = Node(lambda arg: None, value, Node(lambda arg: None, value, Node(lambda arg: None, value))) # E: is not assignable # E: is not assignable # E: is not assignable +"#, +); + +testcase!( + test_constructor_union_distribution_rejected_nested, + r#" +class Node[T]: + def __init__(self, value: T, child: Node[int] | Node[str] | None = None) -> None: ... + +def f(value: int | str | bytes) -> None: + result: Node[int] | Node[str] = Node(value, Node(value, Node(value))) # E: is not assignable # E: is not assignable # E: is not assignable +"#, +); + +testcase!( + test_constructor_union_distribution_caller_bounds, + r#" +class A[T]: + def __init__(self, x: T) -> None: + self.x: T = x + +def bounded[T: int](item: A[T] | A[str]) -> T: ... +def constrained[T: (int, str)](item: A[T] | A[float]) -> None: ... + +def f(x: int | str | bytes) -> None: + bounded(A(x)) # E: upper bound + +def g(x: int | str | float | bytes) -> None: + constrained(A(x)) # E: constraints + +def valid(x: int | str, y: int | float) -> None: + bounded(A(x)) + constrained(A(y)) +"#, +); + +testcase!( + test_constructor_union_distribution_later_hint, + TestEnv::new().enable_implicit_any_lambda_error(), + r#" +from typing import Callable + +class A[T, U]: + def __init__(self, value: T, callback: Callable[[U], str]) -> None: ... + +def f(value: int | str) -> None: + original: A[int, str] | A[str, str] = A(value, lambda x: x.upper()) + extended: A[int, int] | A[int, str] | A[str, str] = A(value, lambda x: x.upper()) + keyword: A[int, int] | A[int, str] | A[str, str] = A(callback=lambda x: x.upper(), value=value) +"#, +); + +testcase!( + test_constructor_union_distribution_caller_mixed_arguments, + r#" +from typing import assert_type + +class Pair[T, U]: + def __init__(self, x: T, y: U) -> None: ... + +def same[T: int](item: Pair[T, T] | Pair[str, str]) -> None: ... +def mixed[T: int, U: str](item: Pair[T, U] | Pair[str, int]) -> None: ... +def infer[T](item: Pair[T, int] | Pair[T, str]) -> T: ... + +def invalid(x: int | str, y: int | str) -> None: + same(Pair(x, y)) # E: upper bound + mixed(Pair(x, y)) # E: upper bound # E: upper bound + +def valid(x: int | bytes, y: int | str) -> None: + assert_type(infer(Pair(x, y)), int | bytes) +"#, +); + +testcase!( + test_constructor_union_distribution_contextual_conversion, + TestEnv::new().enable_deprecated_error(), + r#" +from typing import Callable +from typing_extensions import deprecated + +class A[T]: + def __init__(self, cb: Callable[[T], object], value: T) -> None: ... + +@deprecated("old callback") +def old(value: object) -> object: ... + +def f(value: int | str, fixed: str) -> None: + necessary: A[int] | A[str] = A(lambda arg: str(arg), value) + redundant: A[int] | A[str] = A(lambda arg: str(fixed), value) # E: Unnecessary `str()` call + deprecated: A[int] | A[str] = A(lambda arg: old(str(arg)), value) # E: `old` is deprecated +"#, +); + +testcase!( + test_constructor_union_distribution_contextual_comparison, + TestEnv::new().enable_incompatible_comparison_error(), + r#" +from typing import Callable + +class A[T]: + def __init__(self, cb: Callable[[T], object], value: T) -> None: ... + +def f(value: int | str, fixed: int) -> None: + valid: A[int] | A[str] = A(lambda arg: arg == "a", value) + invalid: A[int] | A[str] = A(lambda arg: fixed == "a", value) # E: Comparison `==` between incompatible types +"#, +); + +testcase!( + test_constructor_union_distribution_binding_budget, + TestEnv::new().with_thread_count(ThreadCount::Inline), + r#" +class A[T]: + def __init__(self, value: T) -> None: ... +class Node[T]: + def __init__(self, value: T, + child: Node[int] | Node[str] | None = None) -> None: ... +class Wrap: + def __init__(self, a: Node[int] | Node[str], b: object) -> None: ... +def need(x: A[int] | A[str]) -> int: ... + +def f(w: int | str | bytes, v: int | str) -> None: + Wrap(Node(w, Node(w, Node(w, Node(w, Node(w))))), helper(v)) # type: ignore + +def helper(v: int | str): + return need(A(v)) +"#, +); + +testcase!( + test_constructor_union_distribution_sibling_budget, + TestEnv::new().with_thread_count(ThreadCount::Inline), + r#" +class A[T]: + def __init__(self, value: T) -> None: ... +class Node[T]: + def __init__(self, value: T, + child: Node[int] | Node[str] | None = None) -> None: ... +class Wrap: + def __init__(self, a: object, b: object) -> None: ... +def need(x: A[int] | A[str]) -> int: ... + +def f(w: int | str | bytes, v: int | str) -> None: + Wrap( + Node(w, Node(w, Node(w, Node(w, Node(w))))), # type: ignore + need(A(v)), + ) + Wrap( + need(A(v)), + Node(w, Node(w, Node(w, Node(w, Node(w))))), # type: ignore + ) +"#, +); + +#[test] +fn test_constructor_union_distribution_plain_calls_budget() -> anyhow::Result<()> { + let calls = "P(), ".repeat(1001); + let source = format!( + r#" +class P: ... +class Wrap: + def __init__(self, value: object) -> None: ... +class A[T]: + def __init__(self, value: T, *other: object) -> None: ... +def need(value: A[int] | A[str]) -> int: ... +def f(value: int | str) -> None: + Wrap(({calls}need(A(value)))) + expanded: A[int] | A[str] = A(value, ({calls})) +"# + ); + testcase_for_macro( + TestEnv::new().with_thread_count(ThreadCount::Inline), + &source, + file!(), + line!(), + ) +} + +testcase!( + test_constructor_union_distribution_new_and_init, + r#" +from typing import Self + +class New[T]: + def __new__(cls, value: T) -> Self: ... + +class Both[T]: + def __new__(cls, value: T) -> Self: ... + def __init__(self, value: T) -> None: ... + +def f(value: int | str) -> None: + new: New[int] | New[str] = New(value) + both: Both[int] | Both[str] = Both(value) +"#, +); + +testcase!( + test_constructor_union_distribution_typed_dict, + r#" +from typing import TypedDict + +class Fields[T](TypedDict): + value: T + +def f(value: int | str) -> None: + fields: Fields[int] | Fields[str] = Fields(value=value) +"#, +); + +testcase!( + test_constructor_union_distribution_nested, + r#" +class Node[T]: + def __init__(self, value: T, child: Node[int] | Node[str] | None = None) -> None: ... + +def f(value: int | str) -> None: + child: Node[int] | Node[str] = Node(value) + separate: Node[int] | Node[str] = Node(value, child) + # Nested constructors retain ordinary inference inside a distribution probe. + root: Node[int] | Node[str] = Node( # E: is not assignable + value, + Node( # E: is not assignable + value, + Node( # E: is not assignable + value, + Node( # E: is not assignable + value, + Node( # E: is not assignable + value, + Node( # E: is not assignable + value, + Node( # E: is not assignable + value, + Node(value), # E: is not assignable + ), + ), + ), + ), + ), + ), + ) +"#, +); testcase!( test_constructor_prepared_argument_contexts, diff --git a/pyrefly/lib/test/intersection_calls.rs b/pyrefly/lib/test/intersection_calls.rs index d9c4342417..3ce410b9b4 100644 --- a/pyrefly/lib/test/intersection_calls.rs +++ b/pyrefly/lib/test/intersection_calls.rs @@ -5,6 +5,7 @@ * LICENSE file in the root directory of this source tree. */ +use crate::test::util::TestEnv; use crate::testcase; testcase!( @@ -634,6 +635,38 @@ def intrinsic_errors(a: A): "#, ); +testcase!( + test_intersection_call_contextual_diagnostics, + TestEnv::new() + .enable_deprecated_error() + .enable_incompatible_comparison_error(), + r#" +from typing import Callable, cast +from typing_extensions import deprecated + +class A: + def method(self, cb: Callable[[int], object]) -> object: ... +class B: + def method(self, cb: Callable[[str], object]) -> object: ... + +@deprecated("old callback") +def old(value: object) -> object: ... + +def f(a: A, fixed: str) -> None: + if isinstance(a, B): + a.method(lambda arg: arg == "a") + a.method(lambda arg: str(arg)) + a.method(lambda arg: cast(int, arg)) + a.method(lambda arg: str(fixed)) # E: Unnecessary `str()` call + a.method(lambda arg: old(arg)) # E: `old` is deprecated + a.method(lambda arg: arg.missing) # E: has no attribute `missing` + +def reversed_order(b: B) -> None: + if isinstance(b, A): + b.method(lambda arg: arg.missing) # E: has no attribute `missing` +"#, +); + testcase!( test_intersection_constructor_generic_self_required_argument, r#" diff --git a/pyrefly/lib/test/lsp/definition.rs b/pyrefly/lib/test/lsp/definition.rs index 5430cbf194..dd25e06427 100644 --- a/pyrefly/lib/test/lsp/definition.rs +++ b/pyrefly/lib/test/lsp/definition.rs @@ -64,6 +64,25 @@ fn get_test_report(state: &State, handle: &Handle, position: TextSize) -> String } } +#[test] +fn goto_def_constructor_union_contexts() { + let code = r#" +from typing import Callable +class A[T]: + def __init__(self, cb: Callable[[T], object], value: T) -> None: ... +class X: + def meth(self) -> int: ... +class Y: + def meth(self) -> str: ... +def f(value: X | Y) -> None: + result: A[X] | A[Y] = A(lambda arg: arg.meth(), value) +# ^ +"#; + let report = get_batched_lsp_operations_report(&[("main", code)], get_test_report); + assert!(report.contains("def meth(self) -> int"), "{report}"); + assert!(report.contains("def meth(self) -> str"), "{report}"); +} + fn get_test_report_do_not_jump_through_renamed_import( state: &State, handle: &Handle, diff --git a/pyrefly/lib/test/lsp/hover.rs b/pyrefly/lib/test/lsp/hover.rs index 093e28b8e9..ef7b2f7421 100644 --- a/pyrefly/lib/test/lsp/hover.rs +++ b/pyrefly/lib/test/lsp/hover.rs @@ -43,6 +43,33 @@ fn get_test_report(state: &State, handle: &Handle, position: TextSize) -> String } } +#[test] +fn hover_constructor_union_contexts() { + let code = r#" +from typing import Callable +class A[T]: + def __init__(self, cb: Callable[[T], object], value: T) -> None: ... +class X: + def meth(self) -> int: ... +class Y: + def meth(self) -> str: ... +def f(value: X | Y) -> None: + result: A[X] | A[Y] = A(lambda arg: arg.meth(), value) +# ^ + result: A[X] | A[Y] = A(lambda arg: arg.meth(), value) +# ^ +"#; + let report = get_batched_lsp_operations_report(&[("main", code)], get_test_report); + assert!( + report.contains("((self: X) -> int) | ((self: Y) -> str)"), + "{report}" + ); + assert!( + report.contains("((arg: X) -> int) | ((arg: Y) -> str)"), + "{report}" + ); +} + #[test] fn hover_symbol_literals() { let code = r#" diff --git a/pyrefly/lib/test/util.rs b/pyrefly/lib/test/util.rs index 3cb58db8ee..846abfaa6d 100644 --- a/pyrefly/lib/test/util.rs +++ b/pyrefly/lib/test/util.rs @@ -304,6 +304,7 @@ pub struct TestEnv { no_any_return_explicit_error: bool, no_any_return_implicit_error: bool, implicit_any_lambda_error: bool, + deprecated_error: bool, invalid_abstract_method_error: bool, abstract_method_with_implementation_error: bool, property_type_mismatch_error: bool, @@ -389,6 +390,7 @@ impl TestEnv { no_any_return_explicit_error: false, no_any_return_implicit_error: false, implicit_any_lambda_error: false, + deprecated_error: false, invalid_abstract_method_error: false, abstract_method_with_implementation_error: false, property_type_mismatch_error: false, @@ -749,6 +751,11 @@ impl TestEnv { self } + pub fn enable_deprecated_error(mut self) -> Self { + self.deprecated_error = true; + self + } + pub fn enable_invalid_abstract_method_error(mut self) -> Self { self.invalid_abstract_method_error = true; self @@ -1078,6 +1085,9 @@ impl TestEnv { if self.implicit_any_lambda_error { errors.set_error_severity(ErrorKind::ImplicitAnyLambda, Severity::Error); } + if self.deprecated_error { + errors.set_error_severity(ErrorKind::Deprecated, Severity::Error); + } if self.invalid_abstract_method_error { errors.set_error_severity(ErrorKind::InvalidAbstractMethod, Severity::Error); }