From 313a85100a1a48fcee2bb3642254848c4f68b4cb Mon Sep 17 00:00:00 2001 From: Rich Chiodo Date: Fri, 18 Sep 2026 15:22:04 -0700 Subject: [PATCH 1/3] Retain bounded alternatives for ambiguous overload calls Preserve complete overload return alternatives and whole-operation witnesses without erasing container precision or losing ordinary diagnostics. Enable bounded lazy activation by default while retaining the internal baseline control and leaving checker live-cache handoff disabled. Carry reviewed regression coverage and documentation onto pinned main, preserving upstream complexity, TypeForm cache, and TypedDict behavior. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- docs/overloadResultExperiment.md | 118 + docs/type-concepts-advanced.md | 3 +- .../pyright-internal/src/analyzer/checker.ts | 15 +- .../src/analyzer/overloadResult.ts | 290 ++ .../src/analyzer/overloadResultAdmission.ts | 493 ++++ .../src/analyzer/overloadResultController.ts | 1242 +++++++++ .../src/analyzer/overloadResultSelector.ts | 160 ++ .../src/analyzer/parseTreeUtils.ts | 4 + .../pyright-internal/src/analyzer/program.ts | 44 +- .../src/analyzer/sourceFile.ts | 25 +- .../src/analyzer/tracePrinter.ts | 5 + .../src/analyzer/typeCacheUtils.ts | 45 + .../src/analyzer/typeComplexity.ts | 5 + .../src/analyzer/typeEvaluator.ts | 416 ++- .../src/analyzer/typePrinter.ts | 25 + .../src/analyzer/typeUtils.ts | 34 +- .../src/analyzer/typeWalker.ts | 15 + .../pyright-internal/src/analyzer/types.ts | 145 +- .../src/common/configOptions.ts | 3 + .../src/languageService/completionProvider.ts | 6 +- .../pyright-internal/src/tests/config.test.ts | 51 + .../src/tests/languageServer.test.ts | 77 + .../src/tests/overloadResult.test.ts | 247 ++ .../tests/overloadResultController.test.ts | 2380 +++++++++++++++++ .../src/tests/samples/overloadCall11.py | 10 +- .../samples/overloadResultActivationOrder1.py | 84 + .../tests/samples/overloadResultAutomatic1.py | 171 ++ .../samples/overloadResultController1.py | 112 + .../samples/overloadResultDynamicReturns1.py | 27 + .../tests/samples/overloadResultReader1.py | 14 + .../tests/testOnlyCacheTransaction.test.ts | 70 + .../pyright-internal/src/tests/testUtils.ts | 34 +- .../src/tests/typeCacheUtils.test.ts | 68 + .../src/tests/typeEvaluator6.test.ts | 29 +- .../src/tests/typeUtils.test.ts | 46 +- .../typeServer/protocol/typeServerProtocol.ts | 8 + .../typeServer/typeServerConversionUtils.ts | 10 + 37 files changed, 6459 insertions(+), 72 deletions(-) create mode 100644 docs/overloadResultExperiment.md create mode 100644 packages/pyright-internal/src/analyzer/overloadResult.ts create mode 100644 packages/pyright-internal/src/analyzer/overloadResultAdmission.ts create mode 100644 packages/pyright-internal/src/analyzer/overloadResultController.ts create mode 100644 packages/pyright-internal/src/analyzer/overloadResultSelector.ts create mode 100644 packages/pyright-internal/src/tests/overloadResult.test.ts create mode 100644 packages/pyright-internal/src/tests/overloadResultController.test.ts create mode 100644 packages/pyright-internal/src/tests/samples/overloadResultActivationOrder1.py create mode 100644 packages/pyright-internal/src/tests/samples/overloadResultAutomatic1.py create mode 100644 packages/pyright-internal/src/tests/samples/overloadResultController1.py create mode 100644 packages/pyright-internal/src/tests/samples/overloadResultDynamicReturns1.py create mode 100644 packages/pyright-internal/src/tests/samples/overloadResultReader1.py create mode 100644 packages/pyright-internal/src/tests/testOnlyCacheTransaction.test.ts diff --git a/docs/overloadResultExperiment.md b/docs/overloadResultExperiment.md new file mode 100644 index 000000000000..5aa1ed30b23f --- /dev/null +++ b/docs/overloadResultExperiment.md @@ -0,0 +1,118 @@ +# Bounded local overload results + +Supported automatic overload results are enabled by default in `ConfigOptions` +and `Program`, including CLI, language-server and background-worker analysis. +There is no new public project, command-line or editor setting. The internal +`ConfigOptions.experimentalOverloadResults = false` escape hatch retains the +ordinary evaluator for baseline comparisons. Omitted options use the default, +including after configuration serialization, service cloning and reload. + +Explicit internal `Program` controller injection takes precedence over the +configuration default, including fixed-seed and test-only checking controls. +Injecting both controller kinds remains an error. Automatic activation does +**not** imply checker live-cache handoff: production configuration supplies +`checkerHandoff: false`, and even an injected automatic controller requires +explicit `checkerHandoff: true` to exercise that separate, unadopted experiment. + +This is not full general overload conformance. Within the existing bounded +proof and local-use admission domain, an ambiguous `list[Any]` call can retain +`list[int]` and `list[str]` alternatives. Supported assignments and operations +require one complete successful alternative for the entire operation, not a +combination of partial successes. Public queries display these alternatives as +`OverloadResult[list[int], list[str]]`, not `Any`, `Unknown`, or an ordinary union +that would reject invariant assignments. `OverloadResult` is an internal type +representation, not new Python annotation syntax. + +The producer restriction remains direct positional Name calls using supported +function parameters. Nested invariant and nominal generic shapes are supported +only within the existing selector proof domain. Constructors, bound-method +producers, unsupported or escaping use graphs, and unsupported gradual argument +shapes remain ordinary. Concrete calls preserve ordinary precision; admission +or resource cutoffs do not grant more permissive fallback types. + +## Automatic activation + +Ordinary overload matching first evaluates a real argument/parameter match. +For a complete, successful, unexpanded match at a supported direct assignment +call, the controller checks whether its arguments could contain invariant +`Any`. Concrete calls continue through the ordinary evaluator without module +scanning, candidate operations, reservations or private cache restoration. +Evaluator routing and projection hooks stay disconnected until an admitted +activation installs operation roots. Controller initialization and disposal +still occur, and ordinary checking retains the statement restart boundary. + +A potentially ambiguous call runs the existing bounded selector. This probe +is additional real matching work, not a claim that all alternatives have +already been solved. A successful probe queues activation; incomplete or +rejected probes do not install a carrier or an ordinary-result owner. + +When a request started outside speculation and return inference, a selected +probe immediately unwinds to that request boundary before activation is +flushed. Continuing the provisional ordinary evaluation could otherwise cache +dependent values or publish false member errors. A top-level baseline operation can +discover an independent producer outside its own node set through native code +flow; it likewise unwinds before retaining the interrupted outcome. Candidate +trials do not discover new producers. A request entered from an already-active +external speculative context still leaves activation pending until a safe +ordinary request. + +At a non-speculative, non-inference evaluator checkpoint, the controller runs +the existing bounded module admission graph. It considers all syntactic seeds +when establishing the graph, preserving the existing dependency/escape +decisions and whole-module resource limits. Only records depending on activated, +admitted producers receive operation ownership and reservations. + +Activation restarts the enclosing evaluator request, or the enclosing checker +statement when checking is in progress. The checker statement boundary is +necessary to publish assignment and checker diagnostics from the newly owned +operation. No partially evaluated `TypeResult` is promoted to a carrier. +The existing private cache transactions, baseline operations, real alternative +validations and diagnostic publication then perform the work. + +The candidate/proof/graph/node limits and actual candidate-call accounting are +unchanged. Active producers still perform the existing charged candidate +discovery operation; the earlier native probe is separately reported through +`activationArgumentChecks`, `activationProbes` and `activationProofUnits`. +Rejected concrete calls no longer perform the old charged discovery operation. + +## Ownership and invalidation + +Activation state is local to an evaluator/controller and keyed by parse-node +identity. Replacement parse generations do not inherit activations or pending +probes. Disposal clears pending work and retires owned outcomes. + +Ordinary public queries do not become experimental operations merely because +another function in the module activated a producer. Records that already own +ordinary or failed outcomes still require their private restoration and +cache-validity guards; automatic activation does not bypass those guards. + +Native overload matching may speculatively query an earlier operation to +determine whether a call returns. Exact-node cache isolation permits this only +when the operation root is disjoint from every active speculative root. +Overlapping roots in either direction remain forbidden, as does isolation in +return inference. The native speculative stack and its undo entries are not +disabled or replaced. + +An earlier, unowned statement in the same function can also be evaluated as +a native flow prerequisite of a private operation. Its cache entries survive +exact-node isolation, so its node-associated diagnostics are sent to the +surrounding native sink rather than being discarded with the private trial. +Existing native suppression, reachability and diagnostic deduplication still +apply. Diagnostics within an owned operation remain private until its complete +witness or ordinary fallback is checked. + +The regression tests cover analysis-first and cold queries, same-file ordinary +functions, annotated producer diagnostics, cancellation/reentry and resource +cutoffs. The activation-order matrix distinguishes independent `int`/`str` and +`bytes`/`float` producers, their aliases and generic projections, repeated public +queries, and original/edited/restored parse generations. It compares complete +native diagnostics and TypeResult metadata, proves actual carrier selection, +and requires a single complete witness for mixed-element `extend` operations. +Separate tests preserve pending activation across external call speculation and +exercise disjoint/overlapping cache roots under completion, error and cancellation. + +Timing is measured separately from observer instrumentation; no timing threshold +is asserted in CI. Earlier V3 residual measurements are historical evidence, +not measurements of subsequent correctness or default-wiring changes. Enabling +the default is not zero-overhead certification; current comparisons must keep +analysis, public-query work and observation counters separate. diff --git a/docs/type-concepts-advanced.md b/docs/type-concepts-advanced.md index 448d02266caa..950ddc0cebf3 100644 --- a/docs/type-concepts-advanced.md +++ b/docs/type-concepts-advanced.md @@ -326,9 +326,10 @@ Some functions or methods can return one of several different types. In cases wh 3. If only one overload remains, it is the “winner”. -4. If more than one overload remains, the “winner” is chosen based on the order in which the overloads are declared. In general, the first remaining overload is the “winner”. There are two exceptions to this rule. +4. If more than one overload remains, the “winner” is chosen based on the order in which the overloads are declared. In general, the first remaining overload is the “winner”. The following exceptions apply. Exception 1: When an `*args` (unpacked) argument matches a `*args` parameter in one of the overload signatures, this overrides the normal order-based rule. Exception 2: When two or more overloads match because an argument evaluates to `Any` or `Unknown`, the matching overload is ambiguous. In this case, pyright examines the return types of the remaining overloads and eliminates types that are duplicates or are subsumed by (i.e. proper subtypes of) other types in the list. If only one type remains after this coalescing step, that type is used. If more than one type remains after this coalescing step, the type of the call expression evaluates to `Unknown`. For example, if two overloads are matched due to an argument that evaluates to `Any`, and those two overloads have return types of `str` and `LiteralString`, pyright will coalesce this to just `str` because `LiteralString` is a proper subtype of `str`. If the two overloads have return types of `str` and `bytes`, the call expression will evaluate to `Unknown` because `str` and `bytes` have no overlap. + Exception 3: For supported local assignments with nested invariant `Any` arguments, such as `list[Any]`, Pyright can retain multiple fully known return alternatives. For example, `list[int]` and `list[str]` are displayed as `OverloadResult[list[int], list[str]]`. Each supported operation must succeed for one complete alternative; this is not an ordinary union or `Any`. This behavior is enabled by default within a [bounded local-use and proof domain](overloadResultExperiment.md); unsupported cases retain ordinary behavior. 5. If no overloads remain, Pyright considers whether any of the arguments are union types. If so, these union types are expanded into their constituent subtypes, and the entire process of overload matching is repeated with the expanded argument types. If two or more overloads match, the union of their respective return types form the final return type for the call expression. This "union expansion" can result in a combinatoric explosion if many arguments evaluate to union types. For example, if four arguments are present, and they all evaluate to unions that expand to ten subtypes, this could result in 10^4 combinations. Pyright expands unions for arguments left to right and halts expansion when the number of signatures exceeds 64. diff --git a/packages/pyright-internal/src/analyzer/checker.ts b/packages/pyright-internal/src/analyzer/checker.ts index 506fefd0895e..d5936a4665fe 100644 --- a/packages/pyright-internal/src/analyzer/checker.ts +++ b/packages/pyright-internal/src/analyzer/checker.ts @@ -237,7 +237,8 @@ export class Checker extends ParseTreeWalker { private _evaluator: TypeEvaluator, parseResults: ParserOutput, private _dependentFiles: ParserOutput[] | undefined, - nodeInfoReader: AnalyzerNodeInfo.AnalyzerNodeInfoReader + nodeInfoReader: AnalyzerNodeInfo.AnalyzerNodeInfoReader, + private _walkOperation?: (node: ParseNode, callback: () => void) => void ) { // Forward the reader to the base walker so the structural walk expands both tier-1 // (parser-derived) and tier-2 (evaluator-discovered, e.g. `cast("Foo", v)`) string @@ -293,6 +294,18 @@ export class Checker extends ParseTreeWalker { } override walk(node: ParseNode) { + if (this._walkOperation) { + this._walkOperation(node, () => { + if (!this._nodeInfo.isCodeUnreachable(node)) { + super.walk(node); + } else { + this._evaluator.suppressDiagnostics(node, () => { + super.walk(node); + }); + } + }); + return; + } if (!this._nodeInfo.isCodeUnreachable(node)) { super.walk(node); } else { diff --git a/packages/pyright-internal/src/analyzer/overloadResult.ts b/packages/pyright-internal/src/analyzer/overloadResult.ts new file mode 100644 index 000000000000..15aac4c34046 --- /dev/null +++ b/packages/pyright-internal/src/analyzer/overloadResult.ts @@ -0,0 +1,290 @@ +/* + * overloadResult.ts + * Copyright (c) Microsoft Corporation. + * Licensed under the MIT license. + * + * Passive baseline projection for overload-result witnesses. + */ + +import { + combineTypes, + FunctionType, + isClass, + isFunction, + isFunctionOrOverloaded, + isTypeVar, + maxTypeRecursionCount, + OverloadedType, + PropertyMethodInfo, + Type, + TypeBase, + TypeCategory, + UnionType, +} from './types'; + +export interface OverloadResultProjectionLimits { + maxDepth?: number; + maxNodes?: number; +} + +export type OverloadResultProjection = + | { kind: 'projected'; type: Type; visitedNodes: number } + | { kind: 'blocked'; reason: 'depth' | 'nodes' | 'cycle' | 'category'; visitedNodes: number }; + +class ProjectionBlocked extends Error { + constructor(readonly reason: 'depth' | 'nodes' | 'cycle' | 'category') { + super(`Overload-result baseline projection blocked: ${reason}`); + } +} + +// A blocked result is not a usable fallback type. Callers must not replace it +// with Any, an unprojected input, or an arbitrary candidate. +export function projectOverloadResultBaseline( + type: Type, + limits: OverloadResultProjectionLimits = {} +): OverloadResultProjection { + const maxDepth = limits.maxDepth ?? maxTypeRecursionCount; + const maxNodes = limits.maxNodes ?? 4096; + const active = new Set(); + const memo = new Map(); + let visitedNodes = 0; + + const optional = (value: Type | undefined, depth: number) => + value === undefined ? undefined : project(value, depth); + + function project(value: Type, depth: number): Type { + const cached = memo.get(value); + if (cached) { + return cached; + } + if (depth > maxDepth) { + throw new ProjectionBlocked('depth'); + } + if (visitedNodes >= maxNodes) { + throw new ProjectionBlocked('nodes'); + } + visitedNodes++; + if (active.has(value)) { + throw new ProjectionBlocked('cycle'); + } + active.add(value); + const result = projectValue(value, depth + 1); + active.delete(value); + memo.set(value, result); + return result; + } + + function projectValue(value: Type, depth: number): Type { + let result: Type = value; + switch (value.category) { + case TypeCategory.OverloadResult: + return project(value.priv.baselineType, depth); + + case TypeCategory.Class: { + const typeArgs = value.priv.typeArgs?.map((arg) => project(arg, depth)); + const tupleTypeArgs = value.priv.tupleTypeArgs?.map((arg) => ({ + ...arg, + type: project(arg.type, depth), + })); + const partialCallType = optional(value.priv.partialCallType, depth); + const accessor = (info: PropertyMethodInfo | undefined) => { + if (!info) { + return undefined; + } + const methodType = project(info.methodType, depth); + const classType = optional(info.classType, depth); + if (!isFunctionOrOverloaded(methodType) || (classType && !isClass(classType))) { + throw new ProjectionBlocked('category'); + } + return methodType === info.methodType && classType === info.classType + ? info + : { ...info, methodType, classType }; + }; + const fgetInfo = accessor(value.priv.fgetInfo); + const fsetInfo = accessor(value.priv.fsetInfo); + const fdelInfo = accessor(value.priv.fdelInfo); + const narrowedEntries = value.priv.typedDictNarrowedEntries; + const entries = narrowedEntries + ? new Map( + [...narrowedEntries].map(([name, entry]) => [ + name, + { ...entry, valueType: project(entry.valueType, depth) }, + ]) + ) + : undefined; + if ( + typeArgs?.some((arg, index) => arg !== value.priv.typeArgs![index]) || + tupleTypeArgs?.some((arg, index) => arg.type !== value.priv.tupleTypeArgs![index].type) || + partialCallType !== value.priv.partialCallType || + fgetInfo !== value.priv.fgetInfo || + fsetInfo !== value.priv.fsetInfo || + fdelInfo !== value.priv.fdelInfo || + (entries && + [...entries].some(([name, entry]) => entry.valueType !== narrowedEntries!.get(name)!.valueType)) + ) { + const clone = TypeBase.cloneType(value); + Object.assign(clone.priv, { + typeArgs, + tupleTypeArgs, + partialCallType, + fgetInfo, + fsetInfo, + fdelInfo, + typedDictNarrowedEntries: entries, + }); + result = clone; + } + break; + } + + case TypeCategory.Function: { + const parameterTypes = value.shared.parameters.map((_, index) => + project(FunctionType.getParamType(value, index), depth) + ); + const parameterDefaultTypes = value.shared.parameters.map((_, index) => + optional(FunctionType.getParamDefaultType(value, index), depth) + ); + const returnType = optional(FunctionType.getEffectiveReturnType(value), depth); + const inferredReturnType = optional(value.shared.inferredReturnType?.type, depth); + const strippedFirstParamType = optional(value.priv.strippedFirstParamType, depth); + const boundToType = optional(value.priv.boundToType, depth); + if (boundToType && !isClass(boundToType)) { + throw new ProjectionBlocked('category'); + } + if ( + parameterTypes.some((arg, index) => arg !== FunctionType.getParamType(value, index)) || + parameterDefaultTypes.some( + (arg, index) => arg !== FunctionType.getParamDefaultType(value, index) + ) || + returnType !== FunctionType.getEffectiveReturnType(value) || + inferredReturnType !== value.shared.inferredReturnType?.type || + strippedFirstParamType !== value.priv.strippedFirstParamType || + boundToType !== value.priv.boundToType + ) { + const clone = FunctionType.specialize(value, { parameterTypes, parameterDefaultTypes, returnType }); + clone.priv.strippedFirstParamType = strippedFirstParamType; + clone.priv.boundToType = boundToType; + clone.priv.callSiteReturnTypeCache = undefined; + if (value.shared.inferredReturnType && inferredReturnType) { + clone.shared = { + ...clone.shared, + inferredReturnType: { ...value.shared.inferredReturnType, type: inferredReturnType }, + }; + } + result = clone; + } + break; + } + + case TypeCategory.Overloaded: { + const signatures = OverloadedType.getOverloads(value); + const projected = signatures.map((signature) => { + const candidate = project(signature, depth); + if (!isFunction(candidate)) { + throw new ProjectionBlocked('category'); + } + return candidate; + }); + const implementation = OverloadedType.getImplementation(value); + const projectedImplementation = optional(implementation, depth); + if ( + projected.some((signature, index) => signature !== signatures[index]) || + implementation !== projectedImplementation + ) { + const clone = OverloadedType.create( + projected.map((signature) => TypeBase.cloneType(signature)), + projectedImplementation && isFunction(projectedImplementation) + ? TypeBase.cloneType(projectedImplementation) + : projectedImplementation + ); + clone.flags = value.flags; + clone.props = value.props; + result = clone; + } + break; + } + + case TypeCategory.Union: { + const subtypes = value.priv.subtypes.map((subtype) => project(subtype, depth)); + if (subtypes.some((subtype, index) => subtype !== value.priv.subtypes[index])) { + result = combineTypes(subtypes); + if (result.category === TypeCategory.Union) { + UnionType.addTypeAliasSource(result, value); + } + } + break; + } + + case TypeCategory.TypeVar: { + const boundType = optional(value.shared.boundType, depth); + const defaultType = project(value.shared.defaultType, depth); + const constraints = value.shared.constraints.map((constraint) => project(constraint, depth)); + const freeTypeVar = optional(value.priv.freeTypeVar, depth); + if (freeTypeVar && !isTypeVar(freeTypeVar)) { + throw new ProjectionBlocked('category'); + } + if ( + boundType !== value.shared.boundType || + defaultType !== value.shared.defaultType || + constraints.some((constraint, index) => constraint !== value.shared.constraints[index]) || + freeTypeVar !== value.priv.freeTypeVar + ) { + const clone = TypeBase.cloneType(value); + clone.shared = { ...value.shared, boundType, defaultType, constraints }; + clone.priv.freeTypeVar = freeTypeVar; + result = clone; + } + break; + } + + case TypeCategory.Unknown: { + const possibleType = optional(value.priv.possibleType, depth); + if (possibleType !== value.priv.possibleType) { + const clone = TypeBase.cloneType(value); + clone.priv.possibleType = possibleType; + result = clone; + } + break; + } + + case TypeCategory.Any: + case TypeCategory.Unbound: + case TypeCategory.Never: + case TypeCategory.Module: + break; + } + + const props = result.props; + if (props) { + const typeForm = optional(props.typeForm, depth); + const specialForm = optional(props.specialForm, depth); + if (specialForm && !isClass(specialForm)) { + throw new ProjectionBlocked('category'); + } + const typeArgs = props.typeAliasInfo?.typeArgs?.map((arg) => project(arg, depth)); + if ( + typeForm !== props.typeForm || + specialForm !== props.specialForm || + typeArgs?.some((arg, index) => arg !== props.typeAliasInfo!.typeArgs![index]) + ) { + result = TypeBase.cloneType(result); + result.props = { + ...props, + typeForm, + specialForm, + typeAliasInfo: props.typeAliasInfo ? { ...props.typeAliasInfo, typeArgs } : undefined, + }; + } + } + return result; + } + + try { + return { kind: 'projected', type: project(type, 0), visitedNodes }; + } catch (error) { + if (error instanceof ProjectionBlocked) { + return { kind: 'blocked', reason: error.reason, visitedNodes }; + } + throw error; + } +} diff --git a/packages/pyright-internal/src/analyzer/overloadResultAdmission.ts b/packages/pyright-internal/src/analyzer/overloadResultAdmission.ts new file mode 100644 index 000000000000..54e1df4cac45 --- /dev/null +++ b/packages/pyright-internal/src/analyzer/overloadResultAdmission.ts @@ -0,0 +1,493 @@ +/* + * overloadResultAdmission.ts + * Copyright (c) Microsoft Corporation. + * Licensed under the MIT license. + * + * Conservative local-use admission for overload alternatives. + */ + +import { CallNode, isExpressionNode, ModuleNode, NameNode, ParseNode, ParseNodeType } from '../parser/parseNodes'; +import { AnalyzerNodeInfoAccessor } from './analyzerNodeInfo'; +import { FlowAssignment, FlowFlags, FlowLabel, FlowNode } from './codeFlowTypes'; +import { DeclarationType } from './declaration'; +import type { FixedOverloadResultSeed, LocalOverloadOperation, OverloadResultLimits } from './overloadResultController'; +import { getEnclosingFunction, isImplicitRevealTypeName } from './parseTreeUtils'; +import { ParseTreeWalker } from './parseTreeWalker'; +import { Symbol } from './symbol'; +import { TypeEvaluator } from './typeEvaluatorTypes'; + +export class OverloadResultAdmissionLimit extends Error {} + +const maxProjectionDeclarations = 32; +const maxReachingVisits = 256; + +class BoundedNodes extends ParseTreeWalker { + readonly nodes: ParseNode[] = []; + + constructor(private _limit: number) { + super(); + } + + override visitNode(node: ParseNode) { + if (this.nodes.length >= this._limit) { + throw new OverloadResultAdmissionLimit('node-limit'); + } + this.nodes.push(node); + return super.visitNode(node); + } +} + +export function collectOverloadResultNodes(node: ParseNode, limit: number): ParseNode[] { + const walker = new BoundedNodes(limit); + walker.walk(node); + return walker.nodes; +} + +export interface LocalOverloadAdmission { + nodes: readonly ParseNode[]; + records: Map; + byNode: Map; + definitions: Map; + reaching: Map; + decisions: Map>; + dependencies: Map>; + graphUnits: number; + complete: boolean; + reason?: string; +} + +export function admitLocalOverloadResults( + module: ModuleNode, + seeds: readonly FixedOverloadResultSeed[], + evaluator: TypeEvaluator, + info: AnalyzerNodeInfoAccessor, + limits: OverloadResultLimits +): LocalOverloadAdmission { + const result: LocalOverloadAdmission = { + nodes: [], + records: new Map(), + byNode: new Map(), + definitions: new Map(), + reaching: new Map(), + decisions: new Map(), + dependencies: new Map(), + graphUnits: 0, + complete: false, + }; + const charge = () => { + if (result.graphUnits >= limits.graphUnits) { + throw new OverloadResultAdmissionLimit('graph-limit'); + } + result.graphUnits++; + }; + const symbolCache = new Map(); + const symbol = (node: NameNode) => { + if (!symbolCache.has(node)) { + charge(); + symbolCache.set(node, evaluator.lookUpSymbolRecursive(node, node.d.value, true)?.symbol); + } + return symbolCache.get(node); + }; + const reject = (origins: ReadonlySet, reason: string) => { + origins.forEach((seed) => result.decisions.get(seed)?.add(reason)); + }; + try { + result.nodes = collectOverloadResultNodes(module, limits.nodes); + if (seeds.length > limits.records) { + throw new OverloadResultAdmissionLimit('seed-limit'); + } + const nodeSet = new Set(result.nodes); + const seedMap = new Map(seeds.map((seed) => [seed.node, seed])); + for (const node of result.nodes) { + if (node.nodeType !== ParseNodeType.StatementList || node.d.statements.length !== 1) { + continue; + } + const statement = node.d.statements[0]; + const fn = getEnclosingFunction(statement); + if (!fn || (statement.nodeType !== ParseNodeType.Assignment && !isExpressionNode(statement))) { + continue; + } + let nodes: ParseNode[]; + try { + nodes = collectOverloadResultNodes(node, limits.rootNodes); + } catch (error) { + if (!(error instanceof OverloadResultAdmissionLimit)) { + throw error; + } + continue; + } + const assignment = statement.nodeType === ParseNodeType.Assignment ? statement : undefined; + const left = assignment?.d.leftExpr; + const target = left?.nodeType === ParseNodeType.TypeAnnotation ? left.d.valueExpr : left; + if (target && target.nodeType !== ParseNodeType.Name) { + continue; + } + if (result.records.size >= limits.records) { + throw new OverloadResultAdmissionLimit('record-limit'); + } + const expression = statement.nodeType === ParseNodeType.Assignment ? statement.d.rightExpr : statement; + const containedSeeds = nodes.filter( + (n): n is CallNode => n.nodeType === ParseNodeType.Call && seedMap.has(n) + ); + const seed = containedSeeds.length === 1 ? seedMap.get(containedSeeds[0]) : undefined; + const valueNodes = new Set(collectOverloadResultNodes(expression, limits.rootNodes)); + if (target) { + valueNodes.add(target); + } + const record: LocalOverloadOperation = { + root: node, + statement, + expression, + assignment, + target, + annotated: left?.nodeType === ParseNodeType.TypeAnnotation || !!assignment?.d.annotationComment, + fn, + nodes, + ids: new Set(nodes.map((n) => n.id)), + valueNodes, + seed, + admitted: false, + reader: false, + }; + result.records.set(node, record); + nodes.forEach((n) => result.byNode.set(n, record)); + if (target) { + result.definitions.set(target, record); + } + if (seed) { + result.decisions.set(record, new Set()); + } else if (containedSeeds.length > 1) { + // No partial injection of a statement containing several seeds. + record.admitted = false; + } + } + + const origins = new Map>(); + const symbolDefs = new Map(); + for (const record of result.records.values()) { + charge(); + origins.set(record, new Set(record.seed ? [record] : [])); + const sym = record.target && symbol(record.target); + if (sym) { + const defs = symbolDefs.get(sym.id) ?? []; + defs.push(record); + symbolDefs.set(sym.id, defs); + } + } + const reads = new Map(); + for (const node of result.nodes) { + if ( + node.nodeType !== ParseNodeType.Name || + result.definitions.has(node) || + (node.parent?.nodeType === ParseNodeType.MemberAccess && node.parent.d.member === node) + ) { + continue; + } + charge(); + const sym = symbol(node); + const declarations = sym && symbolDefs.get(sym.id); + if (!sym || !declarations) { + continue; + } + const targets = new Set(); + const visited = new Set(); + let complex = false; + function walk(flow: FlowNode | undefined) { + if (!flow || visited.has(flow.id)) { + return; + } + charge(); + visited.add(flow.id); + if (visited.size > maxReachingVisits) { + throw new OverloadResultAdmissionLimit('flow-limit'); + } + if (flow.flags & FlowFlags.Assignment) { + // FlowNode is flag-discriminated throughout the analyzer. + const assignment = flow as FlowAssignment; + if (assignment.targetSymbolId === sym!.id) { + targets.add(assignment.node); + return; + } + } + if (flow.flags & FlowFlags.LoopLabel) { + complex = true; + return; + } + if (flow.flags & FlowFlags.Start) { + return; + } + if ('antecedents' in flow && Array.isArray(flow.antecedents)) { + // Use the existing FlowLabel shape rather than interpreting + // arbitrary fields on a ParseNode. + (flow as FlowLabel).antecedents.forEach(walk); + } else if ('antecedent' in flow) { + walk((flow as FlowAssignment).antecedent); + } + } + walk(info.getFlowNode(node)); + let defs = [...targets] + .filter((n): n is NameNode => n.nodeType === ParseNodeType.Name) + .map((n) => result.definitions.get(n)) + .filter((d): d is LocalOverloadOperation => !!d); + const untracked = defs.length !== targets.size; + const captured = declarations.filter((d) => d.fn !== getEnclosingFunction(node)); + defs = [...new Set([...defs, ...captured])]; + if (complex || untracked || (!targets.size && !captured.length)) { + defs = declarations; + } + const ambiguous = complex || untracked || targets.size !== 1 || defs.length !== 1; + reads.set(node, { defs, ambiguous, captured: captured.length > 0 }); + if (!ambiguous && !captured.length) { + result.reaching.set(node, defs[0]); + } + } + const valueOrigins = (record: LocalOverloadOperation) => + record.annotated ? new Set() : origins.get(record)!; + let changed = true; + while (changed) { + changed = false; + for (const record of result.records.values()) { + if (record.annotated) { + continue; + } + for (const node of record.nodes) { + if (node.nodeType !== ParseNodeType.Name) { + continue; + } + for (const definition of reads.get(node)?.defs ?? []) { + charge(); + for (const origin of valueOrigins(definition)) { + charge(); + if (!origins.get(record)!.has(origin)) { + origins.get(record)!.add(origin); + changed = true; + } + } + } + } + } + } + const perStatement = new Map[]>(); + const nestedReaderInputs = new Map>(); + const callableValues = new Map(); + for (const [node, read] of reads) { + const affected = new Set(read.defs.flatMap((d) => [...valueOrigins(d)])); + if (!affected.size) { + continue; + } + charge(); + if (read.captured) { + reject(affected, 'closure-capture'); + } + if (read.ambiguous) { + reject(affected, 'ambiguous-reaching-definition'); + } + const record = result.byNode.get(node); + if (!record) { + reject(affected, 'unsupported-statement-or-escape'); + continue; + } + const uses = perStatement.get(record) ?? []; + uses.push(affected); + perStatement.set(record, uses); + if ( + record.statement.nodeType === ParseNodeType.Call && + (node.parent?.nodeType !== ParseNodeType.Argument || node.parent.parent !== record.statement) + ) { + const nested = nestedReaderInputs.get(record) ?? new Set(); + affected.forEach((origin) => nested.add(origin)); + nestedReaderInputs.set(record, nested); + } + if (record.root.parent !== record.fn.d.suite) { + reject(affected, 'non-straight-line-statement'); + } + let current: ParseNode | undefined = node; + while (current && current !== record.statement) { + charge(); + const child = current; + current = current.parent; + if (!current) { + reject(affected, 'unsupported-expression-path'); + break; + } + if ( + current.nodeType === ParseNodeType.Call && + current.d.leftExpr.nodeType === ParseNodeType.MemberAccess && + child.nodeType === ParseNodeType.Argument + ) { + reject(affected, 'alternative-argument-to-member-call'); + } + if (current.nodeType === ParseNodeType.Call && current.d.leftExpr.nodeType === ParseNodeType.Name) { + const callee = current.d.leftExpr; + if (!callableValues.has(callee)) { + const declarations = symbol(callee)?.getDeclarations() ?? []; + const opaque = + declarations.length > maxProjectionDeclarations || + declarations.some((d) => { + const resolved = evaluator.resolveAliasDeclaration(d, true); + return ( + resolved?.type === DeclarationType.Variable || + resolved?.type === DeclarationType.Param + ); + }); + callableValues.set(callee, { + opaque, + nestedReader: !opaque && current !== record.statement && diagnosticReader(callee), + }); + } + // Callable values can alias a diagnostic reader without a + // declaration exposing its argument roles. + if (callableValues.get(callee)!.opaque) { + reject(affected, 'unsupported-callable-value'); + } + if (callableValues.get(callee)!.nestedReader) { + reject(affected, 'unsupported-nested-reader'); + } + } + if ( + ![ + ParseNodeType.Argument, + ParseNodeType.Assignment, + ParseNodeType.Call, + ParseNodeType.MemberAccess, + ParseNodeType.StatementList, + ParseNodeType.Ternary, + ].includes(current.nodeType) + ) { + reject(affected, 'container-or-unsupported-expression'); + } + } + if ( + record.target && + !record.annotated && + record.expression.nodeType !== ParseNodeType.Name && + record.expression.nodeType !== ParseNodeType.Call + ) { + reject(affected, 'unsupported-stored-value'); + } + } + + function typingFunction(name: NameNode, expectedNames: readonly string[]) { + const declarations = symbol(name)?.getDeclarations() ?? []; + if (declarations.length !== 1) { + return false; + } + charge(); + const resolved = evaluator.resolveAliasDeclaration(declarations[0], true); + if (resolved?.type !== DeclarationType.Function || !expectedNames.includes(resolved.node.d.name.d.value)) { + return false; + } + const file = info.getFileInfo(resolved.node); + return file.isTypingStubFile || file.isTypingExtensionsStubFile; + } + function diagnosticReader(name: NameNode) { + return ( + (isImplicitRevealTypeName(name) && !symbol(name)) || + typingFunction(name, ['assert_type', 'reveal_type']) + ); + } + function projectionShape(call: CallNode) { + if (call.d.leftExpr.nodeType !== ParseNodeType.Name) { + return false; + } + const declarations = symbol(call.d.leftExpr)?.getDeclarations() ?? []; + if (!declarations.length || declarations.length > maxProjectionDeclarations) { + return false; + } + let signatures = 0; + for (const [index, declaration] of declarations.entries()) { + charge(); + if (declaration.type !== DeclarationType.Function || !nodeSet.has(declaration.node)) { + return false; + } + for (const decorator of declaration.node.d.decorators) { + if ( + decorator.d.expr.nodeType !== ParseNodeType.Name || + !typingFunction(decorator.d.expr, ['overload']) + ) { + return false; + } + } + const annotation = declaration.node.d.returnAnnotation; + const base = annotation?.nodeType === ParseNodeType.Index ? annotation.d.leftExpr : annotation; + if (base?.nodeType !== ParseNodeType.Name) { + return false; + } + const returns = symbol(base)?.getDeclarations() ?? []; + if (!returns.length || returns.length > maxProjectionDeclarations) { + return false; + } + let classReturn = false; + for (const returned of returns) { + charge(); + if (evaluator.resolveAliasDeclaration(returned, true)?.type === DeclarationType.Class) { + classReturn = true; + } + } + if (!classReturn && !(index === declarations.length - 1 && signatures > 1)) { + return false; + } + if (classReturn) { + signatures++; + } + } + return signatures > 0; + } + const inputs = new Map>(); + for (const [record, uses] of perStatement) { + const affected = new Set(uses.flatMap((use) => [...use])); + if (uses.length > 1) { + reject(affected, 'multiple-alternative-occurrences'); + } + if (record.seed) { + reject(new Set([...affected, record]), 'dependent-fixed-seeds'); + } + if ( + record.target && + !record.annotated && + record.expression.nodeType === ParseNodeType.Call && + !projectionShape(record.expression) + ) { + reject(affected, 'unsupported-stored-call-declaration'); + } + inputs.set(record, affected); + } + for (const record of result.decisions.keys()) { + if (!record.target || record.expression !== record.seed?.node || record.root.parent !== record.fn.d.suite) { + reject(new Set([record]), 'unsupported-seed-storage'); + } + } + for (const record of result.records.values()) { + if ( + record.statement.nodeType === ParseNodeType.Call && + record.statement.d.leftExpr.nodeType === ParseNodeType.Name + ) { + record.reader = diagnosticReader(record.statement.d.leftExpr); + const nested = nestedReaderInputs.get(record); + // A reader can consume a direct local value, but must not inject + // it into an unadapted nested member/call or type expression. + if (record.reader && nested) { + reject(nested, 'unsupported-reader-expression'); + } + } + } + for (const record of result.records.values()) { + const dependencies = new Set([...(origins.get(record) ?? []), ...(inputs.get(record) ?? [])]); + result.dependencies.set(record, dependencies); + record.admitted = + dependencies.size > 0 && [...dependencies].every((d) => result.decisions.get(d)?.size === 0); + } + result.complete = true; + } catch (error) { + if (!(error instanceof OverloadResultAdmissionLimit)) { + throw error; + } + result.reason = error.message; + result.records.forEach((r) => { + r.admitted = false; + }); + result.decisions.forEach((reasons) => reasons.add(error.message)); + result.reaching.clear(); + } + return result; +} diff --git a/packages/pyright-internal/src/analyzer/overloadResultController.ts b/packages/pyright-internal/src/analyzer/overloadResultController.ts new file mode 100644 index 000000000000..473a3b753b9f --- /dev/null +++ b/packages/pyright-internal/src/analyzer/overloadResultController.ts @@ -0,0 +1,1242 @@ +/* + * overloadResultController.ts + * Copyright (c) Microsoft Corporation. + * Licensed under the MIT license. + * + * Bounded local overload alternatives and operation control. + */ + +import { assert } from '../common/debug'; +import { Diagnostic, DiagnosticCategory } from '../common/diagnostic'; +import { TextRangeDiagnosticSink } from '../common/diagnosticSink'; +import { + AssignmentNode, + CallNode, + ExpressionNode, + FunctionNode, + isExpressionNode, + ModuleNode, + NameNode, + ParseNode, + ParseNodeType, +} from '../parser/parseNodes'; +import { AnalyzerFileInfo } from './analyzerFileInfo'; +import { AnalyzerNodeInfoAccessor, AnalyzerNodeInfoReader, createAnalyzerNodeInfoAccessor } from './analyzerNodeInfo'; +import { + admitLocalOverloadResults, + collectOverloadResultNodes, + LocalOverloadAdmission, + OverloadResultAdmissionLimit, +} from './overloadResultAdmission'; +import { + AutomaticOverloadSelection, + findAutomaticOverloadSeeds, + hasNestedOverloadAny, + isAutomaticOverloadSeedSyntax, + isFixedOverloadArgumentShape, + OverloadSelectionBudget, +} from './overloadResultSelector'; +import { EvalFlags, TypeEvaluator, TypeResult } from './typeEvaluatorTypes'; +import { isClassInstance, isOverloadResult, OverloadResultType, Type, TypeCategory } from './types'; +import { InferenceContext } from './typeUtils'; + +export interface OverloadResultLimits { + nodes: number; + records: number; + rootNodes: number; + graphUnits: number; + candidates: number; + calls: number; + proofUnits: number; +} + +export interface FixedOverloadResultSeed { + node: CallNode; + // The injection contract supplies complete ordinary return alternatives, + // not Types containing a buried carrier. + candidates: (evaluator: TypeEvaluator) => readonly Type[]; + uncertainty: TypeCategory.Any | TypeCategory.Unknown; +} + +export interface ExperimentalOverloadResultOptions { + seeds?: (module: ModuleNode) => readonly FixedOverloadResultSeed[]; + automatic?: boolean; + limits?: Partial; + producerCalls?: number; + consumerCalls?: number; + // Separate test-only experiment; never implied by automatic activation. + checkerHandoff?: boolean; + observe?: (event: OverloadResultEvent) => void; +} + +export type OverloadResultEvent = + | { kind: 'activationProbe'; node: CallNode; selection: AutomaticOverloadSelection } + | { kind: 'selection'; node: CallNode; selection: AutomaticOverloadSelection } + | { kind: 'scan'; module: ModuleNode; nodes: number; seeds: number; cutoff: boolean } + | { kind: 'admission'; module: ModuleNode; admission: LocalOverloadAdmission } + | { kind: 'enter'; module: ModuleNode; root: ParseNode; mode: OperationMode } + | { + kind: 'trial'; + root: ParseNode; + mode: OperationMode; + result?: TypeResult; + diagnostics: readonly Diagnostic[]; + failed: boolean; + retained: boolean; + nestedSpeculative: boolean; + exception: boolean; + } + | { kind: 'value'; root: ParseNode; canonical: TypeResult; baseline: TypeResult; cutoff: boolean } + | { kind: 'project'; node: ExpressionNode; source: Type; result: TypeResult; mode: OperationMode } + | { kind: 'publish'; root: ParseNode; diagnostics: readonly Diagnostic[] } + | { kind: 'retire'; module: ModuleNode; outcomes: number } + | { kind: 'context'; node: ExpressionNode; flags: EvalFlags; context?: InferenceContext; result: TypeResult } + | { kind: 'beforeCall'; node: CallNode; mode: OperationMode; seen: ReadonlyMap }; + +export interface LocalOverloadOperation { + root: ParseNode; + statement: AssignmentNode | ExpressionNode; + expression: ExpressionNode; + assignment?: AssignmentNode; + target?: NameNode; + annotated: boolean; + fn: FunctionNode; + nodes: readonly ParseNode[]; + ids: ReadonlySet; + valueNodes: ReadonlySet; + seed?: FixedOverloadResultSeed & { automatic?: boolean }; + admitted: boolean; + reader: boolean; + flags?: EvalFlags; + context?: InferenceContext; +} + +export type OverloadOperationRequest = + | { kind: 'query'; node: ExpressionNode } + | { kind: 'expression'; node: ExpressionNode; flags: EvalFlags; context?: InferenceContext } + | { kind: 'statement'; node: AssignmentNode; evaluate: () => void }; + +type OperationMode = + | 'baseline' + | 'candidate' + | 'reader' + | 'check-baseline' + | 'check-candidate' + | 'materialize-baseline' + | 'materialize-witness' + | 'context'; +interface Use { + node: NameNode; + value: TypeResult; +} +interface Inputs { + uses: readonly Use[]; + alternative?: Use; +} +interface OrdinaryOutcome { + result: TypeResult; + diagnostics: Diagnostic[]; + failed: boolean; +} +interface Outcome { + baseline: OrdinaryOutcome; + canonical: TypeResult; + value?: TypeResult; + inputs: Inputs; + successes: { candidate: Type; outcome: OrdinaryOutcome }[]; +} +interface Reservation { + used: number; + attempts: number; + limit: number; +} +export interface OverloadResultWork { + candidateCalls: number; + candidateTrials: number; + ordinaryCalls: number; + ordinaryRequests: number; + projectionSetup: number; + contextualRequests: number; + retryReservations: number; + checkerHandoffs: number; +} +interface Generation { + module: ModuleNode; + file: AnalyzerFileInfo; + admission: LocalOverloadAdmission; + outcomes: Map; + variants: Map>; + slots: Map; + interrupted: Set; + building: Set; + preparing: boolean; + published: Set; + ordinaryQueryCaches: Map boolean>; + work: OverloadResultWork; +} +interface ActiveOperation { + generation: Generation; + record: LocalOverloadOperation; + parent?: ActiveOperation; + previousSink: TextRangeDiagnosticSink; + mode: OperationMode; + inputs: ReadonlyMap; + candidate?: Type; + reservation?: Reservation; + seen: Map; +} +const defaultLimits: Readonly = { + nodes: 4096, + records: 256, + rootNodes: 256, + graphUnits: 32768, + candidates: 32, + calls: 256, + proofUnits: 512, +}; +class CandidateCutoff extends Error {} +class ActivationRestart extends Error {} + +export class ExperimentalOverloadResultController { + readonly roots = new Map>(); + private readonly _limits: OverloadResultLimits; + private readonly _info: AnalyzerNodeInfoAccessor; + private readonly _generations = new Map(); + private readonly _emptyModules = new WeakSet(); + private readonly _expiredModules = new WeakSet(); + private _active?: ActiveOperation; + private _disposed = false; + private _activationBoundary = false; + private _activationCanRestart = false; + private _canActivate?: () => boolean; + private _flushActivation?: () => void; + private _discovering = false; + private readonly _activationChecked = new WeakSet(); + private readonly _activated = new WeakSet(); + private readonly _pendingActivation = new Set(); + private _isolate?: (root: ParseNode, callback: () => TypeResult, retain?: boolean) => TypeResult; + private _evict?: (root: ParseNode) => void; + private _select?: (node: CallNode, limit: number, proofUnits: number) => AutomaticOverloadSelection; + private _captureQueryCache?: (root: ParseNode, node: NameNode) => (() => boolean) | undefined; + private _checkerHandoff?: ( + root: ParseNode, + expression: ExpressionNode, + result: TypeResult, + callback: () => void + ) => boolean; + private _automaticWork = { + modules: 0, + nodes: 0, + scanCutoffs: 0, + seeds: 0, + graphAdmitted: 0, + selections: 0, + selected: 0, + proofUnits: 0, + activationArgumentChecks: 0, + activationProbes: 0, + activationProofUnits: 0, + activations: 0, + reasons: new Map(), + }; + + constructor( + private _options: ExperimentalOverloadResultOptions, + private _evaluator: () => TypeEvaluator, + reader: AnalyzerNodeInfoReader + ) { + this._limits = { ...defaultLimits, ..._options.limits }; + assert(!_options.automatic || !_options.seeds, 'Automatic discovery cannot use injected seeds'); + for (const key of Object.keys(defaultLimits) as (keyof OverloadResultLimits)[]) { + const value = this._limits[key]; + assert(Number.isInteger(value) && value >= 0 && value <= defaultLimits[key], `Invalid ${key} limit`); + } + for (const value of [_options.producerCalls, _options.consumerCalls]) { + assert(value === undefined || (Number.isInteger(value) && value >= 0 && value <= defaultLimits.calls)); + } + this._info = createAnalyzerNodeInfoAccessor(reader); + } + + get isAutomatic() { + return !!this._options.automatic; + } + + get hasPendingActivation() { + return this._pendingActivation.size > 0; + } + + install = (isolate: (root: ParseNode, callback: () => TypeResult, retain?: boolean) => TypeResult) => { + this._isolate = isolate; + }; + + installEviction = (evict: (root: ParseNode) => void) => { + this._evict = evict; + }; + + installSelector = (select: (node: CallNode, limit: number, proofUnits: number) => AutomaticOverloadSelection) => { + this._select = select; + }; + + installActivationBoundary = (canActivate: () => boolean, flush: () => void) => { + this._canActivate = canActivate; + this._flushActivation = flush; + }; + + installQueryCache = (capture: (root: ParseNode, node: NameNode) => (() => boolean) | undefined) => { + this._captureQueryCache = capture; + }; + + installCheckerHandoff = ( + handoff: (root: ParseNode, expression: ExpressionNode, result: TypeResult, callback: () => void) => boolean + ) => { + this._checkerHandoff = handoff; + }; + + getAutomaticStats() { + return { ...this._automaticWork, reasons: Object.fromEntries(this._automaticWork.reasons) }; + } + + getDiagnosticSink(node: ParseNode) { + const active = this._active; + const record = active?.generation.admission.byNode.get(node); + if ( + !active || + !record || + this._isEnabled(active.generation, record) || + record.fn !== active.record.fn || + record.root.start >= active.record.root.start + ) { + return undefined; + } + // Earlier ordinary flow prerequisites keep their native caches. Their + // diagnostics must likewise outlive this operation's private trial. + let outer = active; + while (outer.parent?.generation === active.generation) { + outer = outer.parent; + } + return outer.previousSink; + } + + needsActivationBoundary() { + return this._options.automatic && !this._activationBoundary && !this._active; + } + + runWithActivation(callback: () => T): T { + assert(!this._disposed, 'Retired overload-result controller'); + assert(!this._activationBoundary); + this._activationBoundary = true; + try { + this._activationCanRestart = this._canActivate!(); + while (true) { + try { + if (this._activationCanRestart && this.hasPendingActivation) { + this._flushActivation!(); + } + return callback(); + } catch (error) { + if (!(error instanceof ActivationRestart)) { + throw error; + } + } + } + } finally { + this._activationBoundary = false; + this._activationCanRestart = false; + } + } + + considerActivation(node: CallNode, args: readonly { argType: Type }[]) { + if ( + !this._options.automatic || + (this._active && + (!this._activationCanRestart || + this._active.parent || + this._active.mode !== 'baseline' || + this._active.record.ids.has(node.id))) || + this._discovering || + !this._activationBoundary || + this._activationChecked.has(node) || + !isAutomaticOverloadSeedSyntax(node) + ) { + return; + } + this._automaticWork.activationArgumentChecks++; + if ( + !args.some( + (arg) => + isClassInstance(arg.argType) && + arg.argType.priv.typeArgs?.length && + isFixedOverloadArgumentShape(arg.argType, new OverloadSelectionBudget(this._limits.proofUnits)) && + hasNestedOverloadAny(arg.argType) + ) + ) { + return; + } + this._discovering = true; + try { + if (!findAutomaticOverloadSeeds([node], this._evaluator()).length) { + return; + } + const selection = this._select!(node, this._limits.candidates, this._limits.proofUnits); + this._automaticWork.activationProbes++; + this._automaticWork.activationProofUnits += selection.units; + this._emit({ kind: 'activationProbe', node, selection }); + if (selection.reason === 'selected') { + this._activationChecked.add(node); + this._pendingActivation.add(node); + // Unwind native speculation before it can publish or cache + // further uses of the provisional ordinary result. + if (this._activationCanRestart) { + throw new ActivationRestart(); + } + } + } finally { + this._discovering = false; + } + } + + finishActivation() { + if (this._active || this._discovering || !this._pendingActivation.size) { + return; + } + let activated = false; + this._discovering = true; + try { + for (const node of this._pendingActivation) { + this._activated.add(node); + const generation = this._ensure(node, true); + const record = generation?.admission.byNode.get(node); + if (generation && record?.admitted && record.seed?.node === node) { + this._reserveGeneration(generation); + this._automaticWork.activations++; + activated = true; + } else { + this._activated.delete(node); + } + } + this._pendingActivation.clear(); + } finally { + this._discovering = false; + } + if (activated) { + throw new ActivationRestart(); + } + } + + dispose = () => { + for (const generation of this._generations.values()) { + this._retire(generation); + } + this._generations.clear(); + this._pendingActivation.clear(); + this._disposed = true; + }; + + // Kept structurally compatible with the original default-off cache seam. + dispatch = (_node: ExpressionNode, _flags: EvalFlags, _context?: InferenceContext): TypeResult | undefined => + undefined; + + route = (request: OverloadOperationRequest): { result?: TypeResult } | undefined => { + assert(!this._disposed, 'Retired overload-result controller'); + if (request.kind !== 'statement') { + const result = this._query( + request.node, + request.kind === 'expression' ? request.flags : EvalFlags.None, + request.kind === 'expression' ? request.context : undefined, + request.kind === 'query' + ); + return result ? { result } : undefined; + } + const generation = this._ensure(request.node); + const record = generation?.admission.byNode.get(request.node); + if ( + !generation || + !record || + !this._isEnabled(generation, record) || + record.assignment !== request.node || + this._active?.record === record + ) { + return undefined; + } + assert(!generation.building.has(record)); + const outcome = this._build(generation, record); + const selected = record.annotated ? outcome.successes[0] : undefined; + this._trial( + generation, + record, + selected ? 'materialize-witness' : 'materialize-baseline', + outcome.inputs, + selected?.candidate, + request.evaluate, + true + ); + return {}; + }; + + project = (node: ExpressionNode, result: TypeResult): TypeResult => { + const active = this._active; + if (!active) { + return result; + } + if (active.record.seed?.node === node && active.candidate) { + result = { ...result, type: active.candidate }; + } + const use = active.inputs.get(node); + // Contextual requests use ordinary assignment restoration, not value + // substitution that could overwrite a flag-sensitive Name failure. + if (use && !active.record.reader && active.mode !== 'context') { + const value = use.value.type; + result = { + ...result, + type: isOverloadResult(value) ? active.candidate ?? value.priv.baselineType : value, + }; + this._emit({ kind: 'project', node, source: value, result, mode: active.mode }); + } + if (active.record.ids.has(node.id)) { + active.seen.set(node, result); + } + return result; + }; + + beforeCall = (node: CallNode) => { + const active = this._active; + if (!active) { + return; + } + this._emit({ kind: 'beforeCall', node, mode: active.mode, seen: active.seen }); + if (active.mode === 'candidate') { + const reservation = active.reservation!; + if (reservation.used >= reservation.limit) { + throw new CandidateCutoff(); + } + reservation.used++; + active.generation.work.candidateCalls++; + } else { + active.generation.work.ordinaryCalls++; + } + }; + + check = (module: ModuleNode) => { + const initialGeneration = this._ensure(module); + if (!initialGeneration && !this._options.automatic) { + return undefined; + } + initialGeneration?.published.clear(); + const walk = (node: ParseNode, callback: () => void): void => { + if (node.nodeType === ParseNodeType.StatementList && this.needsActivationBoundary()) { + return this.runWithActivation(() => walk(node, callback)); + } + const generation = this._ensure(module); + const record = generation?.admission.records.get(node); + if (this._active || !generation || !record || !this._isEnabled(generation, record)) { + callback(); + return; + } + let handedOff: OrdinaryOutcome | undefined; + const canHandoff = + this._options.automatic && + this._options.checkerHandoff === true && + !this._options.observe && + !record.assignment && + !record.seed && + !record.reader && + !record.context && + !record.flags && + record.root.nodeType === ParseNodeType.StatementList && + record.statement === record.expression && + record.expression.nodeType === ParseNodeType.Call; + const output = this._build( + generation, + record, + canHandoff + ? (inputs, candidate, result, diagnostics) => { + if (diagnostics.length || result.typeErrors || result.isIncomplete) { + return; + } + this._checkerHandoff?.(record.root, record.expression, result, () => { + handedOff = this._trial( + generation, + record, + 'check-candidate', + inputs, + candidate, + callback, + false, + undefined, + undefined, + true + ); + generation.work.checkerHandoffs++; + }); + } + : undefined + ); + const selected = output.successes[0]; + const mode = record.reader ? 'reader' : selected ? 'check-candidate' : 'check-baseline'; + const committed = + handedOff ?? this._trial(generation, record, mode, output.inputs, selected?.candidate, callback); + assert(!generation.published.has(record)); + generation.published.add(record); + committed.diagnostics.forEach((d) => generation.file.diagnosticSink.addDiagnostic(d)); + this._emit({ kind: 'publish', root: record.root, diagnostics: committed.diagnostics }); + }; + return { + walk, + dispose: () => {}, + }; + }; + + evict(module: ModuleNode) { + assert(!this._active); + const generation = this._ensure(module); + generation?.admission.records.forEach((record) => { + if (this._isEnabled(generation, record)) { + this._evict?.(record.root); + } + }); + } + + getStats(module: ModuleNode) { + if (this._disposed) { + return undefined; + } + const generation = this._generations.get(this._info.getFileInfo(module).fileUri.key); + if (!generation || generation.module !== module) { + return undefined; + } + return { + ...generation.work, + records: generation.admission.records.size, + outcomes: generation.outcomes.size, + variants: [...generation.variants.values()].reduce((n, v) => n + v.size, 0), + slots: generation.slots.size, + candidateCallCeiling: [...generation.slots.values()].reduce((n, s) => n + s.limit, 0), + interruptedSlots: generation.interrupted.size, + }; + } + + private _emit(event: OverloadResultEvent) { + this._options.observe?.(event); + } + + private _retire(generation: Generation) { + this._emit({ kind: 'retire', module: generation.module, outcomes: generation.outcomes.size }); + this._expiredModules.add(generation.module); + generation.admission.records.forEach((record) => this.roots.delete(record.root)); + generation.variants.forEach((_, node) => this.roots.delete(node)); + generation.outcomes.clear(); + generation.variants.clear(); + generation.slots.clear(); + generation.interrupted.clear(); + generation.ordinaryQueryCaches.clear(); + } + + private _ensure(node: ParseNode, activate = false): Generation | undefined { + if (this._options.automatic && !activate && !this._generations.size) { + return undefined; + } + let root = node; + while (root.parent) { + root = root.parent; + } + if (root.nodeType !== ParseNodeType.Module || this._emptyModules.has(root)) { + return undefined; + } + assert(!this._expiredModules.has(root), 'Expired overload-result parse generation'); + const file = this._info.getFileInfo(root); + const old = this._generations.get(file.fileUri.key); + if (old?.module === root) { + return old; + } + if (old) { + this._retire(old); + this._generations.delete(file.fileUri.key); + } + if (this._options.automatic && !activate) { + return undefined; + } + let seeds = this._options.seeds?.(root) ?? []; + if (this._options.automatic) { + let nodes: ParseNode[] = []; + let cutoff = false; + try { + nodes = collectOverloadResultNodes(root, this._limits.nodes); + seeds = findAutomaticOverloadSeeds(nodes, this._evaluator()).map((node) => ({ + node, + uncertainty: TypeCategory.Any, + automatic: true, + candidates: () => { + const selection = this._select!(node, this._limits.candidates, this._limits.proofUnits); + this._automaticWork.selections++; + this._automaticWork.selected += selection.reason === 'selected' ? 1 : 0; + this._automaticWork.proofUnits += selection.units; + const reasons = this._automaticWork.reasons; + reasons.set(selection.reason, (reasons.get(selection.reason) ?? 0) + 1); + this._emit({ kind: 'selection', node, selection }); + return selection.candidates; + }, + })); + } catch (error) { + if (!(error instanceof OverloadResultAdmissionLimit)) { + throw error; + } + cutoff = true; + } + this._automaticWork.modules++; + this._automaticWork.nodes += cutoff ? this._limits.nodes + 1 : nodes.length; + this._automaticWork.seeds += seeds.length; + this._automaticWork.scanCutoffs += cutoff ? 1 : 0; + this._emit({ kind: 'scan', module: root, nodes: nodes.length, seeds: seeds.length, cutoff }); + } + if (!seeds.length) { + this._emptyModules.add(root); + return undefined; + } + const admission = admitLocalOverloadResults(root, seeds, this._evaluator(), this._info, this._limits); + if (this._options.automatic) { + this._automaticWork.graphAdmitted += [...admission.records.values()].filter( + (r) => r.seed && r.admitted + ).length; + for (const decision of admission.decisions.values()) { + for (const reason of decision) { + const key = `admission:${reason}`; + this._automaticWork.reasons.set(key, (this._automaticWork.reasons.get(key) ?? 0) + 1); + } + } + if (!admission.complete) { + const key = `admission:${admission.reason}`; + this._automaticWork.reasons.set(key, (this._automaticWork.reasons.get(key) ?? 0) + 1); + } + } + const generation: Generation = { + module: root, + file, + admission, + outcomes: new Map(), + variants: new Map(), + slots: new Map(), + interrupted: new Set(), + building: new Set(), + preparing: false, + published: new Set(), + ordinaryQueryCaches: new Map(), + work: { + candidateCalls: 0, + candidateTrials: 0, + ordinaryCalls: 0, + ordinaryRequests: 0, + projectionSetup: 0, + contextualRequests: 0, + retryReservations: 0, + checkerHandoffs: 0, + }, + }; + this._emit({ kind: 'admission', module: root, admission }); + this._generations.set(file.fileUri.key, generation); + this._reserveGeneration(generation); + return generation; + } + + private _isEnabled(generation: Generation, record: LocalOverloadOperation) { + return ( + record.admitted && + (!this._options.automatic || + [...(generation.admission.dependencies.get(record) ?? [])].every( + (origin) => !!origin.seed && this._activated.has(origin.seed.node) + )) + ); + } + + private _reserveGeneration(generation: Generation) { + const reserve = (node: ParseNode, flags: EvalFlags, seed: boolean) => { + const key = this._key(node, flags); + if (!generation.slots.has(key)) { + generation.slots.set(key, { + used: 0, + attempts: 0, + limit: (seed ? this._options.producerCalls : this._options.consumerCalls) ?? this._limits.calls, + }); + } + }; + for (const record of generation.admission.records.values()) { + if (!this._isEnabled(generation, record)) { + continue; + } + this.roots.set(record.root, record.ids); + reserve(record.root, EvalFlags.None, !!record.seed); + for (const value of record.valueNodes) { + if (isExpressionNode(value)) { + const seed = record.seed?.node; + const containsSeed = + !!seed && seed.start >= value.start && seed.start + seed.length <= value.start + value.length; + reserve(value, EvalFlags.None, containsSeed); + reserve(value, EvalFlags.NoSpecialize, containsSeed); + } + } + } + } + + private _key(node: ParseNode, flags = EvalFlags.None) { + return `${node.id}:${flags}`; + } + + private _definition(generation: Generation, node: NameNode) { + return generation.admission.definitions.get(node) ?? generation.admission.reaching.get(node); + } + + private _inputs(generation: Generation, record: LocalOverloadOperation): Inputs { + const uses: Use[] = []; + for (const node of record.nodes) { + if (node.nodeType !== ParseNodeType.Name || node === record.target) { + continue; + } + const producer = this._definition(generation, node); + if (!producer || !this._isEnabled(generation, producer) || producer === record) { + continue; + } + const value = this._build(generation, producer).value; + if (value) { + uses.push({ node, value }); + } + } + const alternatives = uses.filter((use) => isOverloadResult(use.value.type)); + assert(alternatives.length <= 1, 'Prepublication admission invariant'); + return { uses, alternative: alternatives[0] }; + } + + private _trial( + generation: Generation, + record: LocalOverloadOperation, + mode: OperationMode, + inputs: Inputs, + candidate?: Type, + callback?: () => void, + retain = false, + reservation?: Reservation, + continuation?: (result: TypeResult, diagnostics: readonly Diagnostic[]) => void, + liveCheckerCache = false + ): OrdinaryOutcome { + const evaluator = this._evaluator(); + const parent = this._active; + const previousSink = generation.file.diagnosticSink; + const sink = new TextRangeDiagnosticSink(generation.file.lines); + const nestedSpeculative = !!parent && evaluator.isSpeculativeModeInUse(undefined); + assert(!nestedSpeculative || retain); + assert( + !liveCheckerCache || + (parent?.record === record && + parent.mode === 'candidate' && + mode === 'check-candidate' && + !nestedSpeculative && + !retain && + !reservation) + ); + const projections = new Map(inputs.uses.map((use) => [use.node, use])); + generation.work.projectionSetup += projections.size; + assert(projections.size <= this._limits.rootNodes); + const active: ActiveOperation = { + generation, + record, + parent, + previousSink, + mode, + inputs: projections, + candidate, + reservation, + seen: new Map(), + }; + if (mode === 'candidate') { + generation.work.candidateTrials++; + } else { + generation.work.ordinaryRequests++; + } + this._active = active; + generation.file.diagnosticSink = sink; + let result: TypeResult | undefined; + let diagnostics: Diagnostic[] = []; + let completed = false; + try { + this._emit({ kind: 'enter', module: generation.module, root: record.root, mode }); + const evaluate = () => { + if (callback) { + callback(); + } else if (record.assignment) { + evaluator.evaluateTypesForStatement(record.assignment); + } else { + return evaluator.getTypeOfExpression( + record.expression, + record.flags ?? EvalFlags.None, + record.context + ); + } + return evaluator.getTypeOfExpression(record.expression, record.flags ?? EvalFlags.None, record.context); + }; + const evaluateAndContinue = () => { + const value = evaluate(); + if (continuation) { + diagnostics = sink.fetchAndClear(); + continuation(value, diagnostics); + } + return value; + }; + result = + nestedSpeculative || liveCheckerCache + ? evaluateAndContinue() + : this._isolate!(record.root, evaluateAndContinue, retain); + completed = true; + } finally { + diagnostics.push(...sink.fetchAndClear()); + generation.file.diagnosticSink = previousSink; + this._active = parent; + this._emit({ + kind: 'trial', + root: record.root, + mode, + result, + diagnostics, + failed: !!result?.typeErrors || diagnostics.some((d) => d.category === DiagnosticCategory.Error), + retained: retain, + nestedSpeculative, + exception: !completed, + }); + } + return { + result, + diagnostics, + failed: !!result.typeErrors || diagnostics.some((d) => d.category === DiagnosticCategory.Error), + }; + } + + private _build( + generation: Generation, + record: LocalOverloadOperation, + checkerContinuation?: ( + inputs: Inputs, + candidate: Type, + result: TypeResult, + diagnostics: readonly Diagnostic[] + ) => void + ): Outcome { + const cached = generation.outcomes.get(record); + if (cached) { + return cached; + } + if (!this._active && !generation.building.size && !generation.preparing) { + // A cold read can ask code flow about earlier calls that do not + // return. Their speculative checks may restore earlier assignments. + // Complete those static local records before entering a trial, so + // reentry restores owned outcomes rather than discovering new ones + // inside somebody else's speculative context. + generation.preparing = true; + try { + for (const prior of generation.admission.records.values()) { + if ( + this._isEnabled(generation, prior) && + prior.fn === record.fn && + prior.root.start < record.root.start + ) { + this._build(generation, prior); + } + } + } finally { + generation.preparing = false; + } + } + if (!this._active && !generation.building.size) { + for (const slot of generation.interrupted) { + slot.used = 0; + slot.attempts = 0; + generation.work.retryReservations++; + } + generation.interrupted.clear(); + } + const reservation = generation.slots.get(this._key(record.root, record.flags)); + assert(reservation && !generation.interrupted.has(reservation)); + assert(!generation.building.has(record)); + generation.building.add(record); + try { + const inputs = this._inputs(generation, record); + const baseline = this._trial(generation, record, 'baseline', inputs); + const alias = + !!record.target && + !record.annotated && + record.expression === inputs.alternative?.node && + !baseline.failed; + const successes: Outcome['successes'] = []; + let cutoff = false; + if ( + !alias && + !record.reader && + (!record.target || !baseline.failed || record.annotated) && + !baseline.result.isIncomplete + ) { + const alternative = inputs.alternative?.value.type; + try { + let candidates: readonly Type[] = []; + if (record.seed && !baseline.failed) { + if (record.seed.automatic) { + // Discovery runs inside the same private ordinary cache + // and diagnostic transaction as the complete operation. + const discovery = this._trial( + generation, + record, + 'candidate', + inputs, + undefined, + () => { + candidates = record.seed!.candidates(this._evaluator()); + }, + false, + reservation + ); + if (discovery.failed || discovery.result.isIncomplete) { + candidates = []; + } + } else { + candidates = record.seed.candidates(this._evaluator()); + } + } else if (alternative && isOverloadResult(alternative)) { + candidates = alternative.priv.candidates; + } + if (candidates.length > this._limits.candidates) { + throw new CandidateCutoff(); + } + for (const [index, candidate] of candidates.entries()) { + assert(!isOverloadResult(candidate), 'Fixed seeds must be ordinary root alternatives'); + if (reservation.used >= reservation.limit) { + throw new CandidateCutoff(); + } + assert(reservation.attempts < this._limits.candidates); + reservation.attempts++; + const outcome = this._trial( + generation, + record, + 'candidate', + inputs, + candidate, + undefined, + false, + reservation, + checkerContinuation && !successes.length && index === candidates.length - 1 + ? (result, diagnostics) => checkerContinuation(inputs, candidate, result, diagnostics) + : undefined + ); + if (outcome.result.isIncomplete) { + throw new CandidateCutoff(); + } + if (!outcome.failed) { + successes.push({ candidate, outcome }); + } + } + } catch (error) { + if (!(error instanceof CandidateCutoff)) { + throw error; + } + successes.length = 0; + cutoff = true; + } + } + let canonical = baseline.result; + if (record.reader) { + canonical = this._trial(generation, record, 'reader', inputs).result; + } else if (alias) { + canonical = inputs.alternative!.value; + } else if (successes.length) { + const source = inputs.alternative?.value.type; + const uncertainty = + record.seed?.uncertainty ?? + (source && isOverloadResult(source) ? source.priv.uncertaintyKind : TypeCategory.Any); + canonical = { + ...baseline.result, + type: OverloadResultType.create( + successes.map((s) => s.outcome.result.type), + baseline.result.type, + uncertainty + ), + }; + } + let value: TypeResult | undefined; + if (record.target) { + value = + record.annotated && successes.length + ? successes[0].outcome.result + : !baseline.failed + ? canonical + : undefined; + } + const outcome = { baseline, canonical, value, inputs, successes }; + generation.outcomes.set(record, outcome); + this._emit({ kind: 'value', root: record.root, canonical, baseline: baseline.result, cutoff }); + return outcome; + } catch (error) { + generation.interrupted.add(reservation); + throw error; + } finally { + generation.building.delete(record); + } + } + + private _variant(generation: Generation, record: LocalOverloadOperation, node: ExpressionNode, flags: EvalFlags) { + let variants = generation.variants.get(node); + if (!variants) { + variants = new Map(); + generation.variants.set(node, variants); + } + let variant = variants.get(flags); + if (!variant) { + variant = this._expressionRecord(record, node, flags); + variants.set(flags, variant); + this.roots.set(node, variant.ids); + } + return variant; + } + + private _expressionRecord( + record: LocalOverloadOperation, + node: ExpressionNode, + flags: EvalFlags, + context?: InferenceContext + ) { + const nodes = collectOverloadResultNodes(node, this._limits.rootNodes); + const ordinaryContext = + context && + (isOverloadResult(context.expectedType) || + (context.returnTypeOverride && isOverloadResult(context.returnTypeOverride))) + ? { + ...context, + expectedType: isOverloadResult(context.expectedType) + ? context.expectedType.priv.baselineType + : context.expectedType, + returnTypeOverride: + context.returnTypeOverride && isOverloadResult(context.returnTypeOverride) + ? context.returnTypeOverride.priv.baselineType + : context.returnTypeOverride, + } + : context; + return { + ...record, + root: node, + statement: node, + expression: node, + nodes, + ids: new Set(nodes.map((n) => n.id)), + assignment: undefined, + target: undefined, + annotated: false, + reader: record.reader && node === record.expression, + seed: record.seed && nodes.includes(record.seed.node) ? record.seed : undefined, + flags, + context: ordinaryContext, + }; + } + + private _queryOutcome(generation: Generation, record: LocalOverloadOperation, queryNode?: NameNode): TypeResult { + const outcome = this._build(generation, record); + if (!isOverloadResult(outcome.canonical.type) && !outcome.inputs.alternative) { + if (queryNode && generation.ordinaryQueryCaches.get(queryNode)?.()) { + return outcome.canonical; + } + // A declined operation still owns its diagnostics. Restore ordinary + // subnode caches privately rather than bypassing evaluation or + // replaying its diagnostics into a public query's live sink. + const result = this._trial( + generation, + record, + 'materialize-baseline', + outcome.inputs, + undefined, + undefined, + true + ).result; + if (queryNode) { + generation.ordinaryQueryCaches.delete(queryNode); + const guard = !result.isIncomplete && this._captureQueryCache?.(record.root, queryNode); + if (guard) { + generation.ordinaryQueryCaches.set(queryNode, guard); + } + } + return result; + } + return outcome.canonical; + } + + private _query( + node: ExpressionNode, + flags: EvalFlags, + context?: InferenceContext, + publicQuery = false + ): TypeResult | undefined { + const isCanonicalContext = + (flags === EvalFlags.None || flags === EvalFlags.NoSpecialize) && + (!context || node.nodeType === ParseNodeType.Name); + if (this._active) { + // Diagnostic readers also evaluate type arguments. Leave their + // flag-sensitive validation to the ordinary evaluator in this owner. + return this._active.mode === 'reader' && isCanonicalContext + ? this._active.inputs.get(node)?.value + : undefined; + } + const generation = this._ensure(node); + const record = generation?.admission.byNode.get(node); + if (!generation || !record || !this._isEnabled(generation, record) || !record.valueNodes.has(node)) { + return undefined; + } + if (!isCanonicalContext) { + // Context-dependent requests are complete ordinary operations, not + // canonical memo entries. Their actual flags and context never key a + // persistent table or change the context-free published value. + const contextual = this._expressionRecord(record, node, flags, context); + const prior = this.roots.get(node); + this.roots.set(node, contextual.ids); + try { + generation.work.contextualRequests++; + const result = this._trial( + generation, + contextual, + 'context', + this._inputs(generation, contextual) + ).result; + this._emit({ kind: 'context', node, flags, context, result }); + return result; + } finally { + if (prior) { + this.roots.set(node, prior); + } else { + this.roots.delete(node); + } + } + } + // Name evaluation does not use an expected type. Its value is the + // canonical reaching definition under both supported value flags. + if (node.nodeType === ParseNodeType.Name) { + if (node.parent?.nodeType === ParseNodeType.MemberAccess && node.parent.d.member === node) { + // A member token represents its enclosing access, not a local. + return this._query(node.parent, flags, context); + } + const definition = this._definition(generation, node); + const value = + definition && this._isEnabled(generation, definition) + ? this._build(generation, definition).value + : undefined; + if (value && !isOverloadResult(value.type)) { + // Only public context-free Name readers can reuse a previously + // owned restoration. The ordinary reader still returns the + // actual cache entry; expression/flag requests always evaluate. + this._queryOutcome(generation, record, publicQuery ? node : undefined); + } + return value && isOverloadResult(value.type) ? value : undefined; + } + if (flags === EvalFlags.None && (node === record.expression || node === record.statement)) { + return this._queryOutcome(generation, record); + } + if ( + flags === EvalFlags.NoSpecialize || + node.nodeType === ParseNodeType.Call || + node.nodeType === ParseNodeType.MemberAccess || + node.nodeType === ParseNodeType.Ternary + ) { + return this._queryOutcome(generation, this._variant(generation, record, node, flags)); + } + return undefined; + } +} diff --git a/packages/pyright-internal/src/analyzer/overloadResultSelector.ts b/packages/pyright-internal/src/analyzer/overloadResultSelector.ts new file mode 100644 index 000000000000..7e98aab5297e --- /dev/null +++ b/packages/pyright-internal/src/analyzer/overloadResultSelector.ts @@ -0,0 +1,160 @@ +/* + * overloadResultSelector.ts + * Copyright (c) Microsoft Corporation. + * Licensed under the MIT license. + * + * Bounded proof domain for automatic local overload results. + */ + +import { ArgCategory, CallNode, NameNode, ParseNode, ParseNodeType } from '../parser/parseNodes'; +import { DeclarationType } from './declaration'; +import { TypeEvaluator } from './typeEvaluatorTypes'; +import { ClassType, isAny, isClassInstance, isTypeSame, Type, Variance } from './types'; + +export type OverloadCoverage = 'covered' | 'not-covered' | 'unsupported'; + +export interface AutomaticOverloadSelection { + candidates: readonly Type[]; + reason: string; + coverage: OverloadCoverage[]; + matches: number[]; + units: number; +} + +export class OverloadSelectionBudget { + used = 0; + exceeded = false; + + constructor(readonly limit: number) {} + + take() { + if (this.used >= this.limit) { + this.exceeded = true; + return false; + } + this.used++; + return true; + } +} + +// No MRO adaptation, protocols, variance inference, tuple/union expansion or +// TypeVar solving is part of this proof domain. Those remain ordinary calls. +export function isFixedOverloadArgumentShape(type: Type, budget: OverloadSelectionBudget): boolean { + return isFixedOverloadShape(type, budget, /* allowAny */ true); +} + +export function isFixedOverloadReturnShape(type: Type, budget: OverloadSelectionBudget): boolean { + // Any is a coverage hole for arguments, never a concrete return witness. + return isFixedOverloadShape(type, budget, /* allowAny */ false); +} + +function isFixedOverloadShape(type: Type, budget: OverloadSelectionBudget, allowAny: boolean): boolean { + if (!budget.take()) { + return false; + } + if (isAny(type)) { + return allowAny; + } + if ( + !isClassInstance(type) || + ClassType.isProtocolClass(type) || + ClassType.isTypedDictClass(type) || + ClassType.isPartiallyEvaluated(type) || + type.priv.tupleTypeArgs || + type.priv.literalValue !== undefined || + type.props?.condition || + type.shared.typeParams.some((p) => p.shared.declaredVariance !== Variance.Invariant) + ) { + return false; + } + const args = type.priv.typeArgs ?? []; + return ( + args.length === type.shared.typeParams.length && + args.every((arg) => isFixedOverloadShape(arg, budget, allowAny)) + ); +} + +export function getFixedOverloadCoverage( + source: Type, + destination: Type, + budget: OverloadSelectionBudget +): OverloadCoverage { + if (!budget.take()) { + return 'unsupported'; + } + if (isAny(destination)) { + return 'covered'; + } + if (isAny(source)) { + return 'not-covered'; + } + if ( + !isClassInstance(source) || + !isClassInstance(destination) || + !ClassType.isSameGenericClass(source, destination) + ) { + return 'unsupported'; + } + const sourceArgs = source.priv.typeArgs ?? []; + const destArgs = destination.priv.typeArgs ?? []; + if (sourceArgs.length !== destArgs.length) { + return 'unsupported'; + } + if (!sourceArgs.length) { + return isTypeSame(source, destination) ? 'covered' : 'unsupported'; + } + const coverage = sourceArgs.map((arg, index) => getFixedOverloadCoverage(arg, destArgs[index], budget)); + return coverage.includes('unsupported') + ? 'unsupported' + : coverage.includes('not-covered') + ? 'not-covered' + : 'covered'; +} + +export function hasNestedOverloadAny(type: Type): boolean { + return isClassInstance(type) && !!type.priv.typeArgs?.some((arg) => isAny(arg) || hasNestedOverloadAny(arg)); +} + +// This is an over-approximation for local-use preflight, not a type match. +// Parameter-only operands avoid contextual inference and dependent producers. +export function isAutomaticOverloadSeedSyntax(node: ParseNode): node is CallNode { + return ( + node.nodeType === ParseNodeType.Call && + node.d.leftExpr.nodeType === ParseNodeType.Name && + node.parent?.nodeType === ParseNodeType.Assignment && + node.parent.d.rightExpr === node && + node.d.args.length > 0 && + node.d.args.length <= 8 && + node.d.args.every( + (arg) => + arg.d.argCategory === ArgCategory.Simple && + !arg.d.name && + arg.d.valueExpr.nodeType === ParseNodeType.Name + ) + ); +} + +export function findAutomaticOverloadSeeds(nodes: readonly ParseNode[], evaluator: TypeEvaluator): CallNode[] { + const declarations = (name: NameNode) => + evaluator.lookUpSymbolRecursive(name, name.d.value, true)?.symbol.getDeclarations() ?? []; + return nodes.filter((node): node is CallNode => { + if (!isAutomaticOverloadSeedSyntax(node) || node.d.leftExpr.nodeType !== ParseNodeType.Name) { + return false; + } + const callee = declarations(node.d.leftExpr); + if ( + callee.length < 2 || + callee.length > 33 || + callee.some((d) => d.type !== DeclarationType.Function || d.isMethod || !d.node.d.returnAnnotation) + ) { + return false; + } + return node.d.args.every((arg) => { + if (arg.d.valueExpr.nodeType !== ParseNodeType.Name) { + return false; + } + const decls = declarations(arg.d.valueExpr); + return decls.length === 1 && decls[0].type === DeclarationType.Param; + }); + }); +} diff --git a/packages/pyright-internal/src/analyzer/parseTreeUtils.ts b/packages/pyright-internal/src/analyzer/parseTreeUtils.ts index 03351f757e3f..911e95fdc7d0 100644 --- a/packages/pyright-internal/src/analyzer/parseTreeUtils.ts +++ b/packages/pyright-internal/src/analyzer/parseTreeUtils.ts @@ -589,6 +589,10 @@ export function printOperator(operator: OperatorType): string { // If the name node is the LHS of a call expression or is a member // name in the LHS of a call expression, returns the call node. +export function isImplicitRevealTypeName(node: ExpressionNode): node is NameNode { + return node.nodeType === ParseNodeType.Name && node.d.value === 'reveal_type'; +} + export function getCallForName(node: NameNode): CallNode | undefined { if (node.parent?.nodeType === ParseNodeType.Call && node.parent.d.leftExpr === node) { return node.parent; diff --git a/packages/pyright-internal/src/analyzer/program.ts b/packages/pyright-internal/src/analyzer/program.ts index b0931902bfd6..b40447b8a748 100644 --- a/packages/pyright-internal/src/analyzer/program.ts +++ b/packages/pyright-internal/src/analyzer/program.ts @@ -39,15 +39,17 @@ import { CacheManager } from './cacheManager'; import { CircularDependency } from './circularDependency'; import { ImportResolver } from './importResolver'; import { ImportResult, ImportType } from './importResult'; +import { ExperimentalOverloadResultController, ExperimentalOverloadResultOptions } from './overloadResultController'; import { getDocString } from './parseTreeUtils'; import { ISourceFileFactory } from './programTypes'; import { Scope } from './scope'; -import { IPythonMode, SourceFile } from './sourceFile'; +import { CheckOperationFactory, IPythonMode, SourceFile } from './sourceFile'; import { SourceFileInfo } from './sourceFileInfo'; import { isUserCode, verifyNoCyclesInChainedFiles } from './sourceFileInfoUtils'; import { SourceMapper } from './sourceMapper'; import { Symbol, SymbolTable } from './symbol'; import { createTracePrinter } from './tracePrinter'; +import { EvaluatorOptions } from './typeEvaluator'; import { PrintTypeOptions, TypeEvaluator } from './typeEvaluatorTypes'; import { createTypeEvaluatorWithTracker } from './typeEvaluatorWithTracker'; import { getPrintTypeFlags } from './typePrinter'; @@ -173,6 +175,7 @@ export class Program { private _configOptions: ConfigOptions; private _importResolver: ImportResolver; private _evaluator: TypeEvaluator | undefined; + private _experimentalOverloadResult?: ExperimentalOverloadResultController; private _disposed = false; private _parsedFileCount = 0; private _preCheckCallback: PreCheckCallback | undefined; @@ -185,8 +188,17 @@ export class Program { readonly serviceProvider: ServiceProvider, logTracker?: LogTracker, private _disableChecker?: boolean, - id?: string + id?: string, + private _testOnlyChecking?: { + expression: () => EvaluatorOptions['testOnlyExpression']; + check: CheckOperationFactory; + }, + private _experimentalOverloadResultOptions?: ExperimentalOverloadResultOptions ) { + assert( + !_testOnlyChecking || !_experimentalOverloadResultOptions, + 'Only one operation controller may be installed' + ); this._console = serviceProvider.tryGet(ServiceKeys.console) || new StandardConsole(); this._logTracker = logTracker ?? new LogTracker(this._console, 'FG'); this._importResolver = initialImportResolver; @@ -218,6 +230,10 @@ export class Program { return this._evaluator; } + get experimentalOverloadResultController(): ExperimentalOverloadResultController | undefined { + return this._experimentalOverloadResult; + } + get configOptions(): ConfigOptions { return this._configOptions; } @@ -250,6 +266,9 @@ export class Program { } dispose() { + if ((this._testOnlyChecking || this._experimentalOverloadResult) && !this._disposed) { + this._evaluator?.disposeEvaluator(); + } this.disposeInternal(this._disposed); this._analyzerNodeInfoContext.dispose(); @@ -1981,6 +2000,19 @@ export class Program { this._evaluator.disposeEvaluator(); } + const overloadResultOptions = + this._experimentalOverloadResultOptions ?? + (!this._testOnlyChecking && this._configOptions.experimentalOverloadResults !== false + ? { automatic: true, checkerHandoff: false } + : undefined); + assert(!this._testOnlyChecking || !overloadResultOptions); + this._experimentalOverloadResult = overloadResultOptions + ? new ExperimentalOverloadResultController( + overloadResultOptions, + () => this._evaluator!, + this._analyzerNodeInfoContext + ) + : undefined; this._evaluator = createTypeEvaluatorWithTracker( this._lookUpImport, { @@ -1991,6 +2023,8 @@ export class Program { verifyTypeCacheEvaluatorFlags: !!this._configOptions.internalTestMode, nodeInfoReader: this._analyzerNodeInfoContext, maxCodeComplexity: this._configOptions.maxCodeComplexity, + testOnlyExpression: this._testOnlyChecking?.expression(), + experimentalOverloadResult: this._experimentalOverloadResult, }, this._logTracker, this._configOptions.logTypeEvaluationTime @@ -2328,7 +2362,11 @@ export class Program { this._importResolver, this._evaluator!, dependentFiles, - this._analyzerNodeInfoContext + this._analyzerNodeInfoContext, + this._testOnlyChecking?.check ?? + (this._experimentalOverloadResult + ? (_uri, parsed) => this._experimentalOverloadResult!.check(parsed.parseTree) + : undefined) ); } } diff --git a/packages/pyright-internal/src/analyzer/sourceFile.ts b/packages/pyright-internal/src/analyzer/sourceFile.ts index 0832d1a077b5..a92ea8efb611 100644 --- a/packages/pyright-internal/src/analyzer/sourceFile.ts +++ b/packages/pyright-internal/src/analyzer/sourceFile.ts @@ -60,6 +60,13 @@ const _maxImportCyclesPerFile = 4; // https://github.com/microsoft/vscode/blob/1e750a7514f365585d8dab1a7a82e0938481ea2f/src/vs/editor/common/model/textModel.ts#L194 export const maxSourceFileSize = 50 * 1024 * 1024; +export type CheckOperationFactory = ( + uri: Uri, + parserOutput: ParserOutput, + evaluator: TypeEvaluator, + nodeInfoReader: AnalyzerNodeInfo.AnalyzerNodeInfoReader +) => { walk: (node: ParseNode, callback: () => void) => void; dispose: () => void } | undefined; + interface ResolveImportResult { imports: ImportResult[]; builtinsImportResult?: ImportResult | undefined; @@ -993,7 +1000,8 @@ export class SourceFile { importResolver: ImportResolver, evaluator: TypeEvaluator, dependentFiles: ParserOutput[] | undefined, - nodeInfoReader: AnalyzerNodeInfo.AnalyzerNodeInfoReader + nodeInfoReader: AnalyzerNodeInfo.AnalyzerNodeInfoReader, + checkOperation?: CheckOperationFactory ) { assert(!this.isParseRequired(), `Check called before parsing: state=${this._writableData.debugPrint()}`); assert(!this.isBindingRequired(), `Check called before binding: state=${this._writableData.debugPrint()}`); @@ -1007,15 +1015,26 @@ export class SourceFile { timingStats.typeCheckerTime.timeOperation(() => { const checkDuration = new Duration(); const nodeInfo = AnalyzerNodeInfo.createAnalyzerNodeInfoAccessor(nodeInfoReader); + const operationOwner = checkOperation?.( + this._uri, + this._writableData.parserOutput!, + evaluator, + nodeInfoReader + ); const checker = new Checker( importResolver, evaluator, this._writableData.parserOutput!, dependentFiles, - nodeInfo + nodeInfo, + operationOwner?.walk ); this._writableData.isCheckingInProgress = true; - checker.check(); + try { + checker.check(); + } finally { + operationOwner?.dispose(); + } this._writableData.isCheckingNeeded = false; const fileInfo = nodeInfo.getFileInfo(this._writableData.parserOutput!.parseTree)!; diff --git a/packages/pyright-internal/src/analyzer/tracePrinter.ts b/packages/pyright-internal/src/analyzer/tracePrinter.ts index a0df80cfd0d9..76e934e10036 100644 --- a/packages/pyright-internal/src/analyzer/tracePrinter.ts +++ b/packages/pyright-internal/src/analyzer/tracePrinter.ts @@ -100,6 +100,11 @@ export function createTracePrinter( case TypeCategory.Unknown: return `Unknown ${wrap(type.props?.typeAliasInfo?.shared.fullName)}`; + case TypeCategory.OverloadResult: + return `OverloadResult [${type.priv.candidates + .map((candidate) => printType(candidate)) + .join(',')}]`; + default: assertNever(type); } diff --git a/packages/pyright-internal/src/analyzer/typeCacheUtils.ts b/packages/pyright-internal/src/analyzer/typeCacheUtils.ts index 44af60d54f2c..85ce1f006bc5 100644 --- a/packages/pyright-internal/src/analyzer/typeCacheUtils.ts +++ b/packages/pyright-internal/src/analyzer/typeCacheUtils.ts @@ -119,6 +119,33 @@ export function addContextualTypeCacheEntry( return newCacheEntries; } +// Exact-node experiment support. Unrelated cache entries are never cleared. +export function useTestOnlyCacheIsolation( + cache: Map, + nodeIds: ReadonlySet, + callback: () => T, + retain = false +): T { + const saved = new Map(); + nodeIds.forEach((id) => { + if (cache.has(id)) { + saved.set(id, cache.get(id)!); + } + cache.delete(id); + }); + let completed = false; + try { + const result = callback(); + completed = true; + return result; + } finally { + if (!retain || !completed) { + nodeIds.forEach((id) => cache.delete(id)); + saved.forEach((value, id) => cache.set(id, value)); + } + } +} + // This class maintains a stack of "speculative type contexts". When // a context is popped off the stack, all of the speculative type cache // entries that were created within that context are removed from the @@ -135,6 +162,24 @@ export class SpeculativeTypeTracker { private _speculativeTypeCache = new Map(); private _activeDependentTypes: DependentType[] = []; + testOnlyEvict(nodeIds: ReadonlySet) { + assert(this._speculativeContextStack.length === 0); + nodeIds.forEach((id) => this._speculativeTypeCache.delete(id)); + } + + canUseTestOnlyCacheIsolation(root: ParseNode) { + return this._speculativeContextStack.every( + (context) => + !ParseTreeUtils.isNodeContainedWithin(root, context.speculativeRootNode) && + !ParseTreeUtils.isNodeContainedWithin(context.speculativeRootNode, root) + ); + } + + useTestOnlyCacheIsolation(nodeIds: ReadonlySet, callback: () => T, root?: ParseNode): T { + assert(root ? this.canUseTestOnlyCacheIsolation(root) : this._speculativeContextStack.length === 0); + return useTestOnlyCacheIsolation(this._speculativeTypeCache, nodeIds, callback); + } + enterSpeculativeContext(speculativeRootNode: ParseNode, options?: SpeculativeModeOptions) { this._speculativeContextStack.push({ speculativeRootNode, diff --git a/packages/pyright-internal/src/analyzer/typeComplexity.ts b/packages/pyright-internal/src/analyzer/typeComplexity.ts index 30f66b9d3e89..6ce95c64df17 100644 --- a/packages/pyright-internal/src/analyzer/typeComplexity.ts +++ b/packages/pyright-internal/src/analyzer/typeComplexity.ts @@ -63,6 +63,11 @@ export function getComplexityScoreForType(type: Type, recursionCount = 0): numbe case TypeCategory.Class: { return getComplexityScoreForClass(type, recursionCount); } + + case TypeCategory.OverloadResult: + return Math.max( + ...type.priv.candidates.map((candidate) => getComplexityScoreForType(candidate, recursionCount)) + ); } // For all other types, return a score of 0. diff --git a/packages/pyright-internal/src/analyzer/typeEvaluator.ts b/packages/pyright-internal/src/analyzer/typeEvaluator.ts index de28507c54da..cf44979af4bb 100644 --- a/packages/pyright-internal/src/analyzer/typeEvaluator.ts +++ b/packages/pyright-internal/src/analyzer/typeEvaluator.ts @@ -186,6 +186,7 @@ import { findContextualTypeCacheEntry, SpeculativeModeOptions, SpeculativeTypeTracker, + useTestOnlyCacheIsolation, } from './typeCacheUtils'; import { assignToTypedDict, @@ -389,6 +390,16 @@ import { validateTypeVarDefault, } from './typeUtils'; +import { + AutomaticOverloadSelection, + getFixedOverloadCoverage, + hasNestedOverloadAny, + isFixedOverloadArgumentShape, + isFixedOverloadReturnShape, + OverloadCoverage, + OverloadSelectionBudget, +} from './overloadResultSelector'; + interface GetTypeArgsOptions { isAnnotatedClass?: boolean; hasCustomClassGetItem?: boolean; @@ -615,6 +626,28 @@ export interface EvaluatorOptions { verifyTypeCacheEvaluatorFlags: boolean; nodeInfoReader: AnalyzerNodeInfo.AnalyzerNodeInfoReader; maxCodeComplexity: number; + experimentalOverloadResult?: import('./overloadResultController').ExperimentalOverloadResultController; + // Artifact-only root-operation experiment. No normal evaluator installs this hook. + testOnlyMemberAccess?: { + nodes: ReadonlySet; + install: (getMember: (node: MemberAccessNode, baseTypeResult: TypeResult) => TypeResult) => void; + }; + testOnlyExpression?: { + roots: ReadonlyMap>; + install: (isolate: (root: ParseNode, callback: () => TypeResult, retain?: boolean) => TypeResult) => void; + route?: ( + request: + | { kind: 'query'; node: ExpressionNode } + | { kind: 'expression'; node: ExpressionNode; flags: EvalFlags; context?: InferenceContext } + | { kind: 'statement'; node: AssignmentNode; evaluate: () => void } + ) => { result?: TypeResult } | undefined; + query?: (node: ExpressionNode) => TypeResult | undefined; + dispose?: () => void; + installEviction?: (evict: (root: ParseNode) => void) => void; + dispatch: (node: ExpressionNode, flags: EvalFlags, context?: InferenceContext) => TypeResult | undefined; + project: (node: ExpressionNode, result: TypeResult) => TypeResult; + beforeCall: (node: CallNode) => void; + }; } // Describes a "deferred class completion" that is run when a class type is @@ -662,6 +695,8 @@ export function createTypeEvaluator( ): TypeEvaluator { const nodeInfo = AnalyzerNodeInfo.createAnalyzerNodeInfoAccessor(evaluatorOptions.nodeInfoReader); const maxCodeComplexity = evaluatorOptions.maxCodeComplexity; + const operationController = evaluatorOptions.experimentalOverloadResult ?? evaluatorOptions.testOnlyExpression; + let operationRouter = evaluatorOptions.experimentalOverloadResult?.isAutomatic ? undefined : operationController; const symbolResolutionStack: SymbolResolutionStackEntry[] = []; const speculativeTypeTracker = new SpeculativeTypeTracker(); const suppressedNodeStack: SuppressedNodeStackEntry[] = []; @@ -745,6 +780,7 @@ export function createTypeEvaluator( // circular references in complex data structures, so it fails // to clean up the objects if we don't help it out. function disposeEvaluator() { + operationController?.dispose?.(); functionRecursionMap = new Map(); codeFlowAnalyzerCache = new Map(); typeCache = new Map(); @@ -1016,10 +1052,42 @@ export function createTypeEvaluator( return undefined; } + function finishOverloadActivation() { + if (!returnTypeInferenceTypeCache && !isSpeculativeModeInUse(undefined)) { + try { + evaluatorOptions.experimentalOverloadResult!.finishActivation(); + } finally { + // Install routing before an activation restart re-enters evaluation. + if (evaluatorOptions.experimentalOverloadResult!.roots.size) { + operationRouter = operationController; + } + } + } + } + + function runWithOverloadActivation(callback: () => T): T { + return evaluatorOptions.experimentalOverloadResult!.runWithActivation(() => { + const result = callback(); + if (evaluatorOptions.experimentalOverloadResult!.hasPendingActivation) { + finishOverloadActivation(); + } + return result; + }); + } + // Determines the type of the specified node by evaluating it in // context, logging any errors in the process. This may require the // type of surrounding statements to be evaluated. function getType(node: ExpressionNode): Type | undefined { + if (evaluatorOptions.experimentalOverloadResult?.needsActivationBoundary()) { + return runWithOverloadActivation(() => getType(node)); + } + const canonical = + operationRouter?.route?.({ kind: 'query', node })?.result ?? + evaluatorOptions.testOnlyExpression?.query?.(node); + if (canonical) { + return canonical.type; + } initializePrefetchedTypes(node); let type = evaluateContextualTypeForSubnode(node, () => { @@ -1074,6 +1142,15 @@ export function createTypeEvaluator( } function getTypeResult(node: ExpressionNode): TypeResult | undefined { + if (evaluatorOptions.experimentalOverloadResult?.needsActivationBoundary()) { + return runWithOverloadActivation(() => getTypeResult(node)); + } + const canonical = + operationRouter?.route?.({ kind: 'query', node })?.result ?? + evaluatorOptions.testOnlyExpression?.query?.(node); + if (canonical) { + return canonical; + } return evaluateContextualTypeForSubnode(node, () => { evaluateTypesForExpressionInContext(node); }); @@ -1244,6 +1321,9 @@ export function createTypeEvaluator( flags = EvalFlags.None, inferenceContext?: InferenceContext ): TypeResult { + if (evaluatorOptions.experimentalOverloadResult?.needsActivationBoundary()) { + return runWithOverloadActivation(() => getTypeOfExpression(node, flags, inferenceContext)); + } let useTypeFormCache = (flags & EvalFlags.TypeFormArg) !== 0; if (inferenceContext) { inferenceContext.expectedType = transformPossibleRecursiveTypeAlias(inferenceContext.expectedType); @@ -1258,6 +1338,17 @@ export function createTypeEvaluator( flags |= EvalFlags.NoParamSpec | EvalFlags.NoTypeVarTuple; } + const testResult = + operationRouter?.route?.({ + kind: 'expression', + node, + flags, + context: inferenceContext, + })?.result ?? operationRouter?.dispatch(node, flags, inferenceContext); + if (testResult) { + return testResult; + } + // Is this type already cached? const cacheEntry = useTypeFormCache ? readTypeFormTypeCacheEntry(node, inferenceContext?.expectedType) @@ -1318,6 +1409,12 @@ export function createTypeEvaluator( initializePrefetchedTypes(node); let typeResult = getTypeOfExpressionCore(node, flags, inferenceContext); + if (evaluatorOptions.experimentalOverloadResult?.hasPendingActivation) { + finishOverloadActivation(); + } + if (operationRouter) { + typeResult = operationRouter.project(node, typeResult); + } // Should we disable type promotions for bytes? if ( @@ -2158,6 +2255,9 @@ export function createTypeEvaluator( recursionCount++; switch (type.category) { + case TypeCategory.OverloadResult: + return true; + case TypeCategory.Unbound: case TypeCategory.Unknown: case TypeCategory.Any: @@ -2276,6 +2376,9 @@ export function createTypeEvaluator( recursionCount++; switch (type.category) { + case TypeCategory.OverloadResult: + return true; + case TypeCategory.Unknown: case TypeCategory.Function: case TypeCategory.Overloaded: @@ -3868,7 +3971,9 @@ export function createTypeEvaluator( if (isNodeReachable(node)) { const fileInfo = nodeInfo.getFileInfo(node); - return fileInfo.diagnosticSink.addDiagnosticWithTextRange(diagLevel, message, range ?? node); + const sink = + evaluatorOptions.experimentalOverloadResult?.getDiagnosticSink(node) ?? fileInfo.diagnosticSink; + return sink.addDiagnosticWithTextRange(diagLevel, message, range ?? node); } return undefined; @@ -5270,7 +5375,7 @@ export function createTypeEvaluator( type = addTypeFormForSymbol(node, type, flags, !!effectiveTypeInfo.includesVariableDecl); } else { // Handle the special case of "reveal_type" and "reveal_locals". - if (name === 'reveal_type' || name === 'reveal_locals') { + if (ParseTreeUtils.isImplicitRevealTypeName(node) || name === 'reveal_locals') { type = AnyType.create(); } else { addDiagnostic( @@ -6524,6 +6629,9 @@ export function createTypeEvaluator( break; } + case TypeCategory.OverloadResult: + throw new Error('Overload-result member dispatch is not enabled'); + default: assertNever(baseType); } @@ -9037,6 +9145,8 @@ export function createTypeEvaluator( ); } + operationRouter?.beforeCall(node); + const argList = ParseTreeUtils.getArgsByRuntimeOrder(node).map((arg) => { const functionArg: Arg = { valueExpression: arg.d.valueExpr, @@ -9057,8 +9167,7 @@ export function createTypeEvaluator( typeResult = getTypeOfSuperCall(node); } else if ( isAnyOrUnknown(baseTypeResult.type) && - node.d.leftExpr.nodeType === ParseNodeType.Name && - node.d.leftExpr.d.value === 'reveal_type' + ParseTreeUtils.isImplicitRevealTypeName(node.d.leftExpr) ) { // Handle the implicit "reveal_type" call. typeResult = getTypeOfRevealType(node, inferenceContext); @@ -10074,6 +10183,25 @@ export function createTypeEvaluator( } if (!callResult.argumentErrors && callResult.returnType) { + if ( + evaluatorOptions.experimentalOverloadResult && + !callResult.isTypeIncomplete && + !overload.priv.boundToType && + !overload.shared.methodClass && + !overload.shared.declaration?.isMethod && + errorNode.nodeType === ParseNodeType.Call && + errorNode.d.leftExpr.nodeType === ParseNodeType.Name && + errorNode.parent?.nodeType === ParseNodeType.Assignment && + errorNode.parent.d.rightExpr === errorNode && + expandedArgTypes.length === 1 && + !hasArgTypeOverride && + callResult.argResults + ) { + evaluatorOptions.experimentalOverloadResult.considerActivation( + errorNode, + callResult.argResults + ); + } overloadsUsedForCall.push(overload); matchedOverload = overload; @@ -10357,6 +10485,141 @@ export function createTypeEvaluator( return winningOverloadIndex === undefined ? undefined : matches[winningOverloadIndex].overload; } + function selectAutomaticOverloadResult( + node: CallNode, + limit: number, + proofUnits: number + ): AutomaticOverloadSelection { + const budget = new OverloadSelectionBudget(proofUnits); + const coverage: OverloadCoverage[] = []; + const matches: number[] = []; + const decline = (reason: string): AutomaticOverloadSelection => ({ + candidates: [], + reason: budget.exceeded ? 'proof-limit' : reason, + coverage, + matches, + units: budget.used, + }); + return useSpeculativeMode(node, () => { + const callee = getTypeOfExpression(node.d.leftExpr, EvalFlags.NoSpecialize); + if (callee.isIncomplete || callee.typeErrors || !isOverloaded(callee.type)) { + return decline('not-complete-overloaded-function'); + } + const overloads = OverloadedType.getOverloads(callee.type); + if (overloads.length > limit) { + return decline('candidate-limit'); + } + const args: Arg[] = node.d.args.map((arg) => ({ + argCategory: arg.d.argCategory, + valueExpression: arg.d.valueExpr, + node: arg, + name: arg.d.name, + })); + const sources = node.d.args.map((arg) => getTypeOfExpression(arg.d.valueExpr)); + if ( + sources.some( + (s) => + s.isIncomplete || s.typeErrors || isAny(s.type) || !isFixedOverloadArgumentShape(s.type, budget) + ) + ) { + return decline('unsupported-argument-shape'); + } + if (!sources.some((s) => hasNestedOverloadAny(s.type))) { + return decline('no-invariant-any'); + } + const returns: Type[] = []; + for (let index = 0; index < overloads.length; index++) { + checkForCancellation(); + const overload = overloads[index]; + const declaration = overload.shared.declaration; + const declaredReturnType = overload.shared.declaredReturnType; + const returnType = FunctionType.getEffectiveReturnType(overload); + if ( + !declaration || + declaration.isMethod || + overload.shared.methodClass || + overload.priv.boundToType || + overload.shared.typeParams.length || + !declaredReturnType || + !returnType || + declaration.node.d.decorators.length !== 1 || + overload.shared.parameters.some( + (p, i) => + p.category !== ParamCategory.Simple || + !p.name || + p.defaultExpr || + FunctionType.getParamDefaultType(overload, i) + ) + ) { + return decline('unsupported-signature'); + } + if ( + !isFixedOverloadReturnShape(declaredReturnType, budget) || + (returnType !== declaredReturnType && !isFixedOverloadReturnShape(returnType, budget)) + ) { + return decline('unsupported-return-shape'); + } + const mapping = matchArgsToParams(node, args, { type: overload, isIncomplete: false }, index); + if (mapping.isTypeIncomplete) { + return decline('incomplete-mapping'); + } + if (mapping.argumentErrors) { + continue; + } + if ( + mapping.argParams.length !== args.length || + mapping.argParams.some( + (p, i) => + p.argument !== args[i] || + p.isDefaultArg || + p.requiresTypeVarMatching || + !isFixedOverloadArgumentShape(p.paramType, budget) + ) + ) { + return decline('unsupported-mapping'); + } + // This is the ordinary matcher and solver, not a parallel + // assignability algorithm. Only complete successful matches count. + const matched = useSpeculativeMode(node, () => + validateArgTypesWithContext(node, mapping, new ConstraintTracker(), true, undefined) + ); + if (matched.isTypeIncomplete || !matched.returnType) { + return decline('incomplete-solution'); + } + if (!isFixedOverloadReturnShape(matched.returnType, budget)) { + return decline('unsupported-solved-return-shape'); + } + if (matched.argumentErrors) { + continue; + } + const proof = mapping.argParams.map((p, i) => + getFixedOverloadCoverage(sources[i].type, p.paramType, budget) + ); + const covered = proof.includes('unsupported') + ? 'unsupported' + : proof.includes('not-covered') + ? 'not-covered' + : 'covered'; + coverage.push(covered); + matches.push(index); + if (covered === 'unsupported') { + return decline('unsupported-coverage'); + } + if (!returns.some((type) => isTypeSame(type, matched.returnType!))) { + returns.push(matched.returnType); + } + // A covering overload ends the retained prefix; earlier + // non-covering matches remain live alternatives. + if (covered === 'covered') { + break; + } + } + return returns.length > 1 + ? { candidates: returns, reason: 'selected', coverage, matches, units: budget.used } + : decline('equivalent-or-single-return'); + }); + } + function validateOverloadedArgTypes( errorNode: ExpressionNode, argList: Arg[], @@ -17849,6 +18112,15 @@ export function createTypeEvaluator( } function evaluateTypesForAssignmentStatement(node: AssignmentNode): void { + if ( + operationRouter?.route?.({ + kind: 'statement', + node, + evaluate: () => evaluateTypesForAssignmentStatement(node), + }) + ) { + return; + } const fileInfo = nodeInfo.getFileInfo(node); // If the entire statement has already been evaluated, don't @@ -21794,6 +22066,9 @@ export function createTypeEvaluator( // be evaluated to provide sufficient context for the type. Evaluated types // are written back to the type cache for later retrieval. function evaluateTypesForStatement(node: ParseNode): void { + if (evaluatorOptions.experimentalOverloadResult?.needsActivationBoundary()) { + return runWithOverloadActivation(() => evaluateTypesForStatement(node)); + } initializePrefetchedTypes(node); let curNode: ParseNode | undefined = node; @@ -30131,5 +30406,138 @@ export function createTypeEvaluator( const codeFlowEngine = getCodeFlowEngine(evaluatorInterface, speculativeTypeTracker, nodeInfo); + if (evaluatorOptions.testOnlyMemberAccess) { + const { nodes, install } = evaluatorOptions.testOnlyMemberAccess; + install((node, baseTypeResult) => { + assert(nodes.has(node)); + return getTypeOfMemberAccessWithBaseType(node, baseTypeResult, { method: 'get' }, EvalFlags.NoSpecialize); + }); + } + + evaluatorOptions.experimentalOverloadResult?.installSelector(selectAutomaticOverloadResult); + evaluatorOptions.experimentalOverloadResult?.installActivationBoundary( + () => !returnTypeInferenceTypeCache && !isSpeculativeModeInUse(undefined), + finishOverloadActivation + ); + evaluatorOptions.experimentalOverloadResult?.installCheckerHandoff((root, expression, result, callback) => { + const ids = evaluatorOptions.experimentalOverloadResult!.roots.get(root); + const isComplete = (value: TypeResult) => + !value.isIncomplete && + !value.typeErrors && + !(isFunction(value.type) && FunctionType.isPartiallyEvaluated(value.type)) && + !(isClass(value.type) && ClassType.isPartiallyEvaluated(value.type)); + if ( + !ids || + returnTypeInferenceTypeCache || + isSpeculativeModeInUse(undefined) || + typeCache.get(expression.id)?.typeResult !== result || + typeCache.get(expression.id)?.flags !== EvalFlags.None || + !isComplete(result) + ) { + return false; + } + for (const id of ids) { + const entry = typeCache.get(id); + if ((entry && !isComplete(entry.typeResult)) || typeFormTypeCache.has(id)) { + return false; + } + } + // Continue synchronously before this candidate's isolation unwinds. No + // saved cache set or result is restored, and the checker reads live entries. + checkForCancellation(); + callback(); + return true; + }); + evaluatorOptions.experimentalOverloadResult?.installQueryCache((root, node) => { + const roots = evaluatorOptions.experimentalOverloadResult!.roots; + const ids = roots.get(root); + const isOrdinaryContext = () => !returnTypeInferenceTypeCache && !isSpeculativeModeInUse(undefined); + if (!ids || !isOrdinaryContext()) { + return undefined; + } + const isComplete = (result: TypeResult) => + !result.isIncomplete && + !(isFunction(result.type) && FunctionType.isPartiallyEvaluated(result.type)) && + !(isClass(result.type) && ClassType.isPartiallyEvaluated(result.type)); + const queryEntry = readContextualTypeCacheEntryForNode(node); + if (!queryEntry || !isComplete(queryEntry.typeResult)) { + return undefined; + } + const entries = [...ids].map((id) => ({ + id, + runtime: typeCache.get(id), + expected: expectedTypeCache.get(id), + typeForm: typeFormTypeCache.get(id), + })); + const areComplete = () => + entries.every( + (entry) => + (!entry.runtime || isComplete(entry.runtime.typeResult)) && + (!entry.typeForm || entry.typeForm.every((e) => isComplete(e.typeResult))) + ); + if (!areComplete()) { + return undefined; + } + // This is a validity guard, not a saved public answer. Reuse only while + // the complete, privately restored cache set is still installed. + return () => { + const valid = + isOrdinaryContext() && + roots.get(root) === ids && + readContextualTypeCacheEntryForNode(node) === queryEntry && + areComplete() && + entries.every( + (entry) => + typeCache.get(entry.id) === entry.runtime && + expectedTypeCache.get(entry.id) === entry.expected && + typeFormTypeCache.get(entry.id) === entry.typeForm + ); + if (valid) { + checkForCancellation(); + } + return valid; + }; + }); + + if (operationController) { + const { roots, install } = operationController; + operationController.installEviction?.((root) => { + const ids = roots.get(root); + assert(ids && ids.size <= 256); + assert(!returnTypeInferenceTypeCache && !isSpeculativeModeInUse(undefined)); + ids.forEach((id) => { + typeCache.delete(id); + expectedTypeCache.delete(id); + typeFormTypeCache.delete(id); + }); + speculativeTypeTracker.testOnlyEvict(ids); + }); + install((root, callback, retain = false) => { + const ids = roots.get(root); + assert(ids && ids.size <= 256); + // Native argument speculation can query earlier, disjoint operations + // through code flow. Never isolate a root that overlaps that context. + assert(!returnTypeInferenceTypeCache && speculativeTypeTracker.canUseTestOnlyCacheIsolation(root)); + return useTestOnlyCacheIsolation( + typeCache, + ids, + () => + useTestOnlyCacheIsolation( + expectedTypeCache, + ids, + () => + useTestOnlyCacheIsolation( + typeFormTypeCache, + ids, + () => speculativeTypeTracker.useTestOnlyCacheIsolation(ids, callback, root), + retain + ), + retain + ), + retain + ); + }); + } + return evaluatorInterface; } diff --git a/packages/pyright-internal/src/analyzer/typePrinter.ts b/packages/pyright-internal/src/analyzer/typePrinter.ts index 7ab181242931..d57ba75211a9 100644 --- a/packages/pyright-internal/src/analyzer/typePrinter.ts +++ b/packages/pyright-internal/src/analyzer/typePrinter.ts @@ -517,6 +517,31 @@ function printTypeInternal( return `Overload[${overloads.join(', ')}]`; } + case TypeCategory.OverloadResult: { + if (printTypeFlags & PrintTypeFlags.PythonSyntax) { + return printTypeInternal( + type.priv.baselineType, + printTypeFlags, + returnTypeCallback, + uniqueNameMap, + recursionTypes, + recursionCount + ); + } + const candidates = type.priv.candidates.map((candidate) => + printTypeInternal( + candidate, + printTypeFlags, + returnTypeCallback, + uniqueNameMap, + recursionTypes, + recursionCount + ) + ); + const implicit = type.priv.uncertaintyKind === TypeCategory.Unknown ? '; Unknown' : ''; + return `OverloadResult[${candidates.join(', ')}${implicit}]`; + } + case TypeCategory.Union: { // If this is a value expression that evaluates to a union type but is // not a type alias, simply print the special form ("UnionType"). diff --git a/packages/pyright-internal/src/analyzer/typeUtils.ts b/packages/pyright-internal/src/analyzer/typeUtils.ts index a396b944af8f..14087eefbf70 100644 --- a/packages/pyright-internal/src/analyzer/typeUtils.ts +++ b/packages/pyright-internal/src/analyzer/typeUtils.ts @@ -959,6 +959,9 @@ export function addConditionToType( case TypeCategory.Function: return TypeBase.cloneForCondition(type, TypeCondition.combine(type.props?.condition, condition)); + case TypeCategory.OverloadResult: + return TypeBase.cloneForCondition(type, TypeCondition.combine(type.props?.condition, condition)); + case TypeCategory.Overloaded: return OverloadedType.create( OverloadedType.getOverloads(type).map((t) => addConditionToType(t, condition)) @@ -986,6 +989,7 @@ export function getTypeCondition(type: Type): TypeCondition[] | undefined { case TypeCategory.Class: case TypeCategory.Function: + case TypeCategory.OverloadResult: return type.props?.condition; } } @@ -1261,28 +1265,28 @@ export function isSentinelLiteral(type: Type): boolean { return isClassInstance(type) && type.priv.literalValue instanceof SentinelLiteral; } -export function containsLiteralType(type: Type, includeTypeArgs = false): boolean { - class ContainsLiteralTypeWalker extends TypeWalker { - foundLiteral = false; +class ContainsLiteralTypeWalker extends TypeWalker { + foundLiteral = false; - constructor(private _includeTypeArgs: boolean) { - super(); - } + constructor(private _includeTypeArgs: boolean) { + super(); + } - override visitClass(classType: ClassType): void { - if (isClassInstance(classType)) { - if (isLiteralLikeType(classType)) { - this.foundLiteral = true; - this.cancelWalk(); - } + override visitClass(classType: ClassType): void { + if (isClassInstance(classType)) { + if (isLiteralLikeType(classType)) { + this.foundLiteral = true; + this.cancelWalk(); } + } - if (this._includeTypeArgs) { - super.visitClass(classType); - } + if (this._includeTypeArgs) { + super.visitClass(classType); } } +} +export function containsLiteralType(type: Type, includeTypeArgs = false): boolean { const walker = new ContainsLiteralTypeWalker(includeTypeArgs); walker.walk(type); return walker.foundLiteral; diff --git a/packages/pyright-internal/src/analyzer/typeWalker.ts b/packages/pyright-internal/src/analyzer/typeWalker.ts index 511b7af82086..ecaaf915f52a 100644 --- a/packages/pyright-internal/src/analyzer/typeWalker.ts +++ b/packages/pyright-internal/src/analyzer/typeWalker.ts @@ -16,6 +16,7 @@ import { ModuleType, NeverType, OverloadedType, + OverloadResultType, Type, TypeCategory, TypeVarType, @@ -79,6 +80,10 @@ export class TypeWalker { this.visitOverloaded(type); break; + case TypeCategory.OverloadResult: + this.visitOverloadResult(type); + break; + case TypeCategory.Class: this.visitClass(type); break; @@ -171,6 +176,16 @@ export class TypeWalker { } } + visitOverloadResult(type: OverloadResultType): void { + for (const candidate of type.priv.candidates) { + this.walk(candidate); + if (this._isWalkCanceled) { + return; + } + } + this.walk(type.priv.baselineType); + } + visitClass(type: ClassType): void { if (!ClassType.isPseudoGenericClass(type)) { const typeArgs = type.priv.tupleTypeArgs?.map((t) => t.type) || type.priv.typeArgs; diff --git a/packages/pyright-internal/src/analyzer/types.ts b/packages/pyright-internal/src/analyzer/types.ts index e870854b3d76..99fc9b0d368e 100644 --- a/packages/pyright-internal/src/analyzer/types.ts +++ b/packages/pyright-internal/src/analyzer/types.ts @@ -44,6 +44,9 @@ export const enum TypeCategory { // Type variable TypeVar, + + // Complete return-type witnesses for a materialization-ambiguous call. + OverloadResult, } export const enum TypeFlags { @@ -72,7 +75,8 @@ export type UnionableType = | OverloadedType | ClassType | ModuleType - | TypeVarType; + | TypeVarType + | OverloadResultType; export type Type = UnionableType | NeverType | UnionType; @@ -129,6 +133,8 @@ export interface TypeSameOptions { honorTypeForm?: boolean; honorIsTypeArgExplicit?: boolean; treatAnySameAsUnknown?: boolean; + honorCallBehavior?: boolean; + ignoreOverloadOwner?: boolean; } export interface TypeAliasSharedInfo { @@ -475,6 +481,36 @@ export namespace UnknownType { } } +export interface OverloadResultType extends TypeBase { + priv: { + readonly candidates: readonly Type[]; + readonly baselineType: Type; + readonly uncertaintyKind: TypeCategory.Any | TypeCategory.Unknown; + }; +} + +export namespace OverloadResultType { + export function create( + candidates: readonly Type[], + baselineType: Type, + uncertaintyKind: TypeCategory.Any | TypeCategory.Unknown + ): OverloadResultType { + assert(candidates.length > 0); + return { + category: TypeCategory.OverloadResult, + flags: candidates.reduce((flags, candidate) => flags | candidate.flags, candidates[0].flags), + props: undefined, + cached: undefined, + shared: undefined, + priv: { + candidates: Object.freeze([...candidates]), + baselineType, + uncertaintyKind, + }, + }; + } +} + export interface ModuleDetailsPriv { fields: SymbolTable; docString: string | undefined; @@ -3348,6 +3384,10 @@ export function isOverloaded(type: Type): type is OverloadedType { return type.category === TypeCategory.Overloaded; } +export function isOverloadResult(type: Type): type is OverloadResultType { + return type.category === TypeCategory.OverloadResult; +} + export function isFunctionOrOverloaded(type: Type): type is FunctionType | OverloadedType { return type.category === TypeCategory.Function || type.category === TypeCategory.Overloaded; } @@ -3423,7 +3463,7 @@ export function isTypeSame(type1: Type, type2: Type, options: TypeSameOptions = } if (recursionCount > maxTypeRecursionCount) { - return true; + return !options.honorCallBehavior && !isOverloadResult(type1); } recursionCount++; @@ -3549,6 +3589,55 @@ export function isTypeSame(type1: Type, type2: Type, options: TypeSameOptions = return false; } + if (options.honorCallBehavior) { + const optionalTypesMatch = (left: Type | undefined, right: Type | undefined) => + left && right ? isTypeSame(left, right, recursiveOptions, recursionCount) : left === right; + if ( + type1.shared.name !== functionType2.shared.name || + type1.shared.fullName !== functionType2.shared.fullName || + type1.shared.moduleName !== functionType2.shared.moduleName || + type1.shared.declaration !== functionType2.shared.declaration || + type1.shared.typeVarScopeId !== functionType2.shared.typeVarScopeId || + type1.priv.constructorTypeVarScopeId !== functionType2.priv.constructorTypeVarScopeId || + type1.shared.flags !== functionType2.shared.flags || + type1.priv.preBoundFlags !== functionType2.priv.preBoundFlags || + type1.priv.isCallableWithTypeArgs !== functionType2.priv.isCallableWithTypeArgs || + type1.shared.typeParams.length !== functionType2.shared.typeParams.length || + type1.shared.typeParams.some( + (param, index) => + !isTypeSame(param, functionType2.shared.typeParams[index], recursiveOptions, recursionCount) + ) || + !optionalTypesMatch(type1.priv.boundToType, functionType2.priv.boundToType) || + !optionalTypesMatch(type1.priv.strippedFirstParamType, functionType2.priv.strippedFirstParamType) || + params1.some( + (param, index) => + param.flags !== params2[index].flags || + !optionalTypesMatch( + FunctionType.getParamDefaultType(type1, index), + FunctionType.getParamDefaultType(functionType2, index) + ) + ) + ) { + return false; + } + if (!options.ignoreOverloadOwner) { + const owner1 = type1.priv.overloaded; + const owner2 = functionType2.priv.overloaded; + if ( + owner1 && owner2 + ? !isTypeSame( + owner1, + owner2, + { ...recursiveOptions, ignoreOverloadOwner: true }, + recursionCount + ) + : owner1 !== owner2 + ) { + return false; + } + } + } + const specializedTypes1 = type1.priv.specializedTypes; const specializedTypes2 = functionType2.priv.specializedTypes; @@ -3630,6 +3719,18 @@ export function isTypeSame(type1: Type, type2: Type, options: TypeSameOptions = case TypeCategory.Overloaded: { // Make sure the overload counts match. const functionType2 = type2 as OverloadedType; + + if (options.honorCallBehavior) { + const implementation1 = OverloadedType.getImplementation(type1); + const implementation2 = OverloadedType.getImplementation(functionType2); + if ( + implementation1 && implementation2 + ? !isTypeSame(implementation1, implementation2, recursiveOptions, recursionCount) + : implementation1 !== implementation2 + ) { + return false; + } + } if (type1.priv._overloads === functionType2.priv._overloads) { return true; } @@ -3676,6 +3777,19 @@ export function isTypeSame(type1: Type, type2: Type, options: TypeSameOptions = case TypeCategory.TypeVar: { const type2TypeVar = type2 as TypeVarType; + if ( + options.honorCallBehavior && + (type1.shared.isDefaultExplicit !== type2TypeVar.shared.isDefaultExplicit || + !isTypeSame( + type1.shared.defaultType, + type2TypeVar.shared.defaultType, + recursiveOptions, + recursionCount + )) + ) { + return false; + } + if (type1.priv.scopeId !== type2TypeVar.priv.scopeId) { return false; } @@ -3781,6 +3895,33 @@ export function isTypeSame(type1: Type, type2: Type, options: TypeSameOptions = return type1.priv.isIncomplete === type2Unknown.priv.isIncomplete; } + + case TypeCategory.OverloadResult: { + const other = type2 as OverloadResultType; + const strictOptions: TypeSameOptions = { + ...recursiveOptions, + honorCallBehavior: true, + honorTypeForm: true, + honorIsTypeArgExplicit: true, + treatAnySameAsUnknown: false, + }; + if ( + type1.priv.uncertaintyKind !== other.priv.uncertaintyKind || + type1.priv.candidates.length !== other.priv.candidates.length || + !isTypeSame(type1.priv.baselineType, other.priv.baselineType, strictOptions, recursionCount) + ) { + return false; + } + const unmatched = [...other.priv.candidates]; + return type1.priv.candidates.every((candidate) => { + const index = unmatched.findIndex((item) => isTypeSame(candidate, item, strictOptions, recursionCount)); + if (index < 0) { + return false; + } + unmatched.splice(index, 1); + return true; + }); + } } return true; diff --git a/packages/pyright-internal/src/common/configOptions.ts b/packages/pyright-internal/src/common/configOptions.ts index b79ab9ffebed..fc484744a253 100644 --- a/packages/pyright-internal/src/common/configOptions.ts +++ b/packages/pyright-internal/src/common/configOptions.ts @@ -1087,6 +1087,9 @@ export class ConfigOptions { // Run additional analysis as part of test cases? internalTestMode?: boolean | undefined; + // Internal baseline escape hatch; intentionally not read from project configuration. + experimentalOverloadResults?: boolean = true; + // Run program in index generation mode. indexGenerationMode?: boolean | undefined; diff --git a/packages/pyright-internal/src/languageService/completionProvider.ts b/packages/pyright-internal/src/languageService/completionProvider.ts index 9b3c70825c01..b436c5477d5f 100644 --- a/packages/pyright-internal/src/languageService/completionProvider.ts +++ b/packages/pyright-internal/src/languageService/completionProvider.ts @@ -57,6 +57,7 @@ import { isFunctionOrOverloaded, isInstantiableClass, isModule, + isOverloadResult, isUnknown, Type, TypeBase, @@ -906,7 +907,10 @@ export class CompletionProvider { leftType = this.evaluator.makeTopLevelTypeVarsConcrete(leftType.priv.possibleType); } - doForEachSubtype(leftType, (subtype) => { + // Member availability follows the retained witnesses; keep the + // canonical value intact for subsequent hover and evaluator queries. + const memberType = isOverloadResult(leftType) ? combineTypes([...leftType.priv.candidates]) : leftType; + doForEachSubtype(memberType, (subtype) => { subtype = this.evaluator.makeTopLevelTypeVarsConcrete(subtype); let resolvedClassSubtype: ClassType | undefined; diff --git a/packages/pyright-internal/src/tests/config.test.ts b/packages/pyright-internal/src/tests/config.test.ts index 1d680d57f6f9..0e0ef8c87383 100644 --- a/packages/pyright-internal/src/tests/config.test.ts +++ b/packages/pyright-internal/src/tests/config.test.ts @@ -500,6 +500,57 @@ describe(`config test'}`, () => { assert.ok(deserialized.findExecEnvironment(UriEx.file('foo/bar.py'))); }); + test.each([undefined, true, false])( + 'automatic overload results preserve default/internal override %s through serialization and service cloning', + (override) => { + const fs = new TestFileSystem(/* ignoreCase */ true); + const root = Uri.file(process.cwd(), fs); + const config = new ConfigOptions(root); + if (override !== undefined) { + config.experimentalOverloadResults = override; + } + const sp = createServiceProvider(fs, new NullConsole()); + const host = new TestAccessHost(); + // Project configuration must not expose or override the internal baseline escape hatch. + config.initializeFromJson({ experimentalOverloadResults: !override }, root, sp, host); + const restored = deserialize(serialize(config)); + assert.strictEqual(restored.experimentalOverloadResults, override ?? true); + const service = new AnalyzerService('', sp, { + configOptions: restored, + hostFactory: () => host, + shouldRunAnalysis: () => false, + }); + const cloned = service.clone('', 'overload-clone'); + try { + for (const target of [service, cloned]) { + assert.strictEqual(target.getConfigOptions().experimentalOverloadResults, override ?? true); + assert.strictEqual(!!target.test_program.experimentalOverloadResultController, override !== false); + } + } finally { + cloned.dispose(); + service.dispose(); + sp.dispose(); + } + } + ); + + test('automatic overload results default survives omitted serialized property and configuration reload', () => { + const config = new ConfigOptions(UriEx.file(process.cwd())); + delete config.experimentalOverloadResults; + assert.strictEqual(deserialize(serialize(config)).experimentalOverloadResults, true); + const service = createAnalyzer(); + try { + for (const fromLanguageServer of [false, true]) { + const options = new CommandLineOptions(undefined, fromLanguageServer); + service.setOptions(options); + assert.strictEqual(service.getConfigOptions().experimentalOverloadResults, true); + assert.ok(service.test_program.experimentalOverloadResultController); + } + } finally { + service.dispose(); + } + }); + test('Config watcher reloads when pyproject.toml is created', () => { const fs = new TestFileSystem(/* ignoreCase */ true); const cons = new NullConsole(); diff --git a/packages/pyright-internal/src/tests/languageServer.test.ts b/packages/pyright-internal/src/tests/languageServer.test.ts index 887fd78060e8..bcaf0a0e766d 100644 --- a/packages/pyright-internal/src/tests/languageServer.test.ts +++ b/packages/pyright-internal/src/tests/languageServer.test.ts @@ -14,6 +14,7 @@ import { ConfigurationItem, DidChangeWorkspaceFoldersNotification, DidCloseTextDocumentNotification, + DiagnosticSeverity, DocumentDiagnosticRequest, InitializedNotification, InitializeRequest, @@ -183,6 +184,82 @@ describe(`Basic language server tests`, () => { // Background analysis takes longer than 5 seconds sometimes, so we need to // increase the timeout. jest.setTimeout(200000); + test.each([false, true])( + 'automatic overload results default with background thread=%s', + async (supportsBackgroundThread) => { + const code = ` +// @filename: root/test.py +//// from typing import Any, overload +//// @overload +//// def choose(value: list[int]) -> list[int]: ... +//// @overload +//// def choose(value: list[str]) -> list[str]: ... +//// def choose(value: Any) -> Any: +//// return value +//// def check(value: list[Any]) -> None: +//// [|/*result*/result|] = choose(value) +//// integer: list[int] = result +//// string: list[str] = result +//// result.append(1) +//// result.append("x") +//// bad: list[bytes] = result +//// result.append(3.14) +//// result.nonexistent() +//// def ordinary(value: list[int]) -> None: +//// [|/*concrete*/concrete|] = choose(value) +//// concrete.append("bad") +`; + const info = await runLanguageServer( + DEFAULT_WORKSPACE_ROOT, + code, + true, + [ + { + item: { + scopeUri: `file://${normalizeSlashes(DEFAULT_WORKSPACE_ROOT, '/')}`, + section: 'python.analysis', + }, + value: { typeCheckingMode: 'standard' }, + }, + ], + undefined, + supportsBackgroundThread, + supportsPullDiagnostics + ); + await openFile(info, 'result'); + for (const [marker, type] of [ + ['result', 'OverloadResult[list[int], list[str]]'], + ['concrete', 'list[int]'], + ]) { + const result = await hover(info, marker); + assert(result && MarkupContent.is(result.contents)); + assert.strictEqual( + result.contents.value, + `\`\`\`python\n(variable) ${marker}: ${type}\n\`\`\`` + ); + } + const diagnostics = (await waitForDiagnostics(info)).find((d) => d.uri.endsWith('root/test.py')); + assert(diagnostics); + const marker = info.testData.markerPositions.get('result')!; + const content = info.testData.files.find((f) => f.fileName === marker.fileName)!.content; + const expected = [ + ['bad: list[bytes]', 'reportAssignmentType'], + ['result.append(3.14)', 'reportArgumentType'], + ['result.nonexistent()', 'reportAttributeAccessIssue'], + ['concrete.append("bad")', 'reportArgumentType'], + ]; + assert.deepStrictEqual( + Array.from(diagnostics.diagnostics) + .filter((d) => d.severity === DiagnosticSeverity.Error) + .map((d) => [d.range.start.line, d.code]), + expected.map(([text, rule]) => [ + content.slice(0, content.indexOf(text)).split('\n').length - 1, + rule, + ]) + ); + } + ); + test('background thread diagnostics', async () => { const code = ` // @filename: root/test.py diff --git a/packages/pyright-internal/src/tests/overloadResult.test.ts b/packages/pyright-internal/src/tests/overloadResult.test.ts new file mode 100644 index 000000000000..4055e70ac5d3 --- /dev/null +++ b/packages/pyright-internal/src/tests/overloadResult.test.ts @@ -0,0 +1,247 @@ +/* + * overloadResult.test.ts + * Copyright (c) Microsoft Corporation. + * Licensed under the MIT license. + */ + +import { strict as assert } from 'assert'; + +import { projectOverloadResultBaseline } from '../analyzer/overloadResult'; +import { printType, PrintTypeFlags } from '../analyzer/typePrinter'; +import { + AnyType, + ClassType, + ClassTypeFlags, + combineTypes, + FunctionParam, + FunctionParamFlags, + FunctionType, + isClass, + isFunction, + isOverloaded, + isOverloadResult, + isTypeSame, + isUnion, + NeverType, + OverloadedType, + OverloadResultType, + Type, + TypeBase, + TypeCategory, + UnknownType, +} from '../analyzer/types'; +import { Uri } from '../common/uri/uri'; +import { ParamCategory } from '../parser/parseNodes'; + +function instance(name: string) { + return ClassType.cloneAsInstance( + ClassType.createInstantiable(name, name, '', Uri.empty(), ClassTypeFlags.None, 0, undefined, undefined) + ); +} + +const intType = instance('int'); +const strType = instance('str'); +const listType = instance('list'); +const intList = ClassType.specialize(listType, [intType]); +const strList = ClassType.specialize(listType, [strType]); + +function alternatives(reverse = false) { + return OverloadResultType.create(reverse ? [strList, intList] : [intList, strList], intList, TypeCategory.Any); +} + +function signature(defaultType: Type) { + const fn = FunctionType.createSynthesizedInstance('f'); + FunctionType.addParam( + fn, + FunctionParam.create(ParamCategory.Simple, AnyType.create(), FunctionParamFlags.TypeDeclared, 'x', defaultType) + ); + fn.shared.declaredReturnType = intType; + return fn; +} + +function projected(type: Type): Type { + const result = projectOverloadResultBaseline(type); + assert.strictEqual(result.kind, 'projected'); + assert(result.kind === 'projected'); + return result.type; +} + +test('OverloadResult retains complete payload without pretending to be an ordinary type', () => { + const candidates = [intList, strList]; + const result = OverloadResultType.create(candidates, intList, TypeCategory.Any); + candidates.reverse(); + assert.deepStrictEqual(result.priv.candidates, [intList, strList]); + assert(Object.isFrozen(result.priv.candidates)); + assert.strictEqual(result.priv.baselineType, intList); + assert(isOverloadResult(result)); + assert(!isUnion(result)); + assert(!isClass(result)); + assert(!isTypeSame(result, ClassType.specialize(listType, [AnyType.create()]))); + assert(!isTypeSame(result, combineTypes([intList, strList]))); +}); + +test('OverloadResult comparison retains uncertainty and baseline but ignores candidate order', () => { + assert(isTypeSame(alternatives(), alternatives(true))); + assert(!isTypeSame(alternatives(), OverloadResultType.create([intList, strList], strList, TypeCategory.Any))); + assert(!isTypeSame(alternatives(), OverloadResultType.create([intList, strList], intList, TypeCategory.Unknown))); +}); + +test('OverloadResult uses an honest display and ordinary baseline for Python-only syntax', () => { + const callback = (type: FunctionType) => type.shared.declaredReturnType ?? UnknownType.create(); + assert.strictEqual( + printType(alternatives(), PrintTypeFlags.None, callback), + 'OverloadResult[list[int], list[str]]' + ); + assert.strictEqual(printType(alternatives(), PrintTypeFlags.PythonSyntax, callback), 'list[int]'); +}); + +test('OverloadResult survives ordinary union normalization in both orders', () => { + const ordinary = ClassType.specialize(listType, [AnyType.create()]); + for (const items of [ + [alternatives(), ordinary], + [ordinary, alternatives()], + ]) { + const union = combineTypes(items); + assert(isUnion(union)); + assert.strictEqual(union.priv.subtypes.length, 2); + assert(union.priv.subtypes.some(isOverloadResult)); + assert(isTypeSame(projected(union), combineTypes([intList, ordinary]))); + } +}); + +test('OverloadResult equality does not erase callable defaults, receiver or stripped state', () => { + const first = signature(intType); + const second = signature(strType); + assert(isTypeSame(first, second)); + assert(!isTypeSame(first, second, { honorCallBehavior: true })); + for (const field of ['boundToType', 'strippedFirstParamType'] as const) { + const left = signature(intType); + const right = signature(intType); + left.priv[field] = intList; + right.priv[field] = strList; + assert(isTypeSame(left, right)); + assert(!isTypeSame(left, right, { honorCallBehavior: true })); + } +}); + +test('OverloadResult equality retains ordered overloads, implementations and signature ownership', () => { + const first = signature(intType); + const second = signature(strType); + const implementation1 = signature(intType); + const implementation2 = signature(strType); + const owner1 = OverloadedType.create([first], implementation1); + const owner2 = OverloadedType.create([second], implementation2); + assert(!isTypeSame(owner1, owner2, { honorCallBehavior: true })); + const sameSignature = signature(intType); + OverloadedType.create([sameSignature], signature(strType)); + assert(!isTypeSame(first, sameSignature, { honorCallBehavior: true })); + assert( + !isTypeSame( + OverloadedType.create([signature(intType), signature(strType)]), + OverloadedType.create([signature(strType), signature(intType)]), + { honorCallBehavior: true } + ) + ); +}); + +test('OverloadResult baseline projection preserves ordinary alias and tuple metadata', () => { + const tuple = ClassType.specialize( + ClassType.cloneForTypingAlias(instance('tuple'), 'Tuple'), + [alternatives()], + false, + true, + [{ type: alternatives(), isUnbounded: false, isOptional: true }] + ); + const result = projected(tuple); + assert(isClass(result)); + assert.strictEqual(result.priv.aliasName, 'Tuple'); + assert.strictEqual(result.priv.isTypeArgExplicit, false); + assert.strictEqual(result.priv.includeSubclasses, true); + assert.strictEqual(result.priv.typeArgs![0], intList); + assert.deepStrictEqual(result.priv.tupleTypeArgs, [{ type: intList, isUnbounded: false, isOptional: true }]); + assert(isOverloadResult(tuple.priv.typeArgs![0])); +}); + +test('OverloadResult projects callable parameters, defaults, return and bound payload without reparenting', () => { + const fn = signature(alternatives()); + fn.shared.parameters[0] = FunctionParam.create( + ParamCategory.Simple, + alternatives(), + FunctionParamFlags.TypeDeclared, + 'x', + alternatives() + ); + fn.shared.declaredReturnType = alternatives(); + fn.priv.strippedFirstParamType = alternatives(); + fn.priv.boundToType = ClassType.specialize(listType, [alternatives()]); + const unchanged = signature(intType); + const implementation = signature(intType); + const owner = OverloadedType.create([fn, unchanged], implementation); + const result = projected(owner); + assert(isOverloaded(result)); + const [projectedFn, projectedUnchanged] = OverloadedType.getOverloads(result); + assert.strictEqual(FunctionType.getParamType(projectedFn, 0), intList); + assert.strictEqual(FunctionType.getParamDefaultType(projectedFn, 0), intList); + assert.strictEqual(FunctionType.getEffectiveReturnType(projectedFn), intList); + assert.strictEqual(projectedFn.priv.strippedFirstParamType, intList); + assert.strictEqual(projectedFn.priv.boundToType!.priv.typeArgs![0], intList); + assert.strictEqual(projectedFn.priv.overloaded, result); + assert.strictEqual(projectedUnchanged.priv.overloaded, result); + assert.strictEqual(fn.priv.overloaded, owner); + assert.strictEqual(unchanged.priv.overloaded, owner); + assert.strictEqual(implementation.priv.overloaded, owner); + assert(isFunction(OverloadedType.getImplementation(result)!)); + assert.notStrictEqual(OverloadedType.getImplementation(result), implementation); +}); + +test('OverloadResult does not treat a mixed Never payload as ordinary Never', () => { + const never = NeverType.createNever(); + const result = OverloadResultType.create([never, intList], never, TypeCategory.Any); + assert.notStrictEqual(result.category, TypeCategory.Never); + assert.strictEqual(projected(result), never); +}); + +test('OverloadResult baseline projection retains unchanged inputs by identity', () => { + assert.strictEqual(projected(intList), intList); + assert.strictEqual(projected(signature(intType)).category, TypeCategory.Function); +}); + +test('OverloadResult projection reports depth cutoff without pretending to have a baseline', () => { + let nested: Type = alternatives(); + for (let depth = 0; depth < 21; depth++) { + nested = ClassType.specialize(listType, [nested]); + } + const result = projectOverloadResultBaseline(nested); + assert.strictEqual(result.kind, 'blocked'); + assert(result.kind === 'blocked'); + assert.strictEqual(result.reason, 'depth'); + assert(!('type' in result)); + const largerBudget = projectOverloadResultBaseline(nested, { maxDepth: 24 }); + assert.strictEqual(largerBudget.kind, 'projected'); +}); + +test('OverloadResult projection reports node cutoff even for a shallow ordinary enclosing value', () => { + const args = Array.from({ length: 4096 }, (_, index) => ({ + type: ClassType.cloneWithLiteral(intType, index), + isUnbounded: false, + })); + args.push({ type: ClassType.specialize(listType, [alternatives()]), isUnbounded: false }); + const tuple = ClassType.specialize(instance('tuple'), [AnyType.create()], true, false, args); + const result = projectOverloadResultBaseline(tuple); + assert.strictEqual(result.kind, 'blocked'); + assert(result.kind === 'blocked'); + assert.strictEqual(result.reason, 'nodes'); + assert.strictEqual(result.visitedNodes, 4096); + assert(!('type' in result)); + assert.strictEqual(projectOverloadResultBaseline(tuple, { maxNodes: 4110 }).kind, 'projected'); +}); + +test('OverloadResult projection rejects recursive value edges rather than leaking a wrapper', () => { + const cyclic = TypeBase.cloneType(listType); + cyclic.priv.typeArgs = [cyclic, alternatives()]; + const result = projectOverloadResultBaseline(cyclic); + assert.strictEqual(result.kind, 'blocked'); + assert(result.kind === 'blocked'); + assert.strictEqual(result.reason, 'cycle'); + assert(!('type' in result)); +}); diff --git a/packages/pyright-internal/src/tests/overloadResultController.test.ts b/packages/pyright-internal/src/tests/overloadResultController.test.ts new file mode 100644 index 000000000000..c0ef64c7f272 --- /dev/null +++ b/packages/pyright-internal/src/tests/overloadResultController.test.ts @@ -0,0 +1,2380 @@ +/* + * overloadResultController.test.ts + * Copyright (c) Microsoft Corporation. + * Licensed under the MIT license. + */ + +import * as assert from 'assert'; +import { CancellationToken } from 'vscode-languageserver'; + +import { createAnalyzerNodeInfoAccessor } from '../analyzer/analyzerNodeInfo'; +import { ImportResolver } from '../analyzer/importResolver'; +import { ExperimentalOverloadResultOptions, OverloadResultEvent } from '../analyzer/overloadResultController'; +import { getEnclosingFunction } from '../analyzer/parseTreeUtils'; +import { ParseTreeWalker } from '../analyzer/parseTreeWalker'; +import { Program } from '../analyzer/program'; +import * as TypeEvaluatorModule from '../analyzer/typeEvaluator'; +import { EvalFlags, TypeEvaluator, TypeResult } from '../analyzer/typeEvaluatorTypes'; +import { + AnyType, + FunctionType, + isFunction, + isOverloaded, + isOverloadResult, + OverloadedType, + TypeCategory, +} from '../analyzer/types'; +import { OperationCanceledException } from '../common/cancellationUtils'; +import { ConfigOptions } from '../common/configOptions'; +import { NullConsole } from '../common/console'; +import { Diagnostic, DiagnosticCategory } from '../common/diagnostic'; +import { FullAccessHost } from '../common/fullAccessHost'; +import { pythonVersion3_12 } from '../common/pythonVersion'; +import { createFromRealFileSystem, RealTempFile } from '../common/realFileSystem'; +import { createServiceProvider } from '../common/serviceProviderExtensions'; +import { UriEx } from '../common/uri/uriUtils'; +import { HoverProvider } from '../languageService/hoverProvider'; +import { CompletionProvider } from '../languageService/completionProvider'; +import { CallNode, NameNode, ParseNode, ParseNodeType } from '../parser/parseNodes'; +import { readSampleFile, resolveSampleFilePath } from './testUtils'; + +class Nodes extends ParseTreeWalker { + readonly nodes: ParseNode[] = []; + override visitNode(node: ParseNode) { + this.nodes.push(node); + return super.visitNode(node); + } +} + +const fixture = readSampleFile('overloadResultController1.py'); + +function resultSummary(evaluator: TypeEvaluator, result: TypeResult): object { + const { + type, + bindToSelfType, + unpackedType, + typeList, + inlinedTypeDict, + classType, + narrowedTypeForSet, + overloadsUsedForCall, + expectedTypeDiagAddendum, + ...metadata + } = result; + return { + ...metadata, + type: evaluator.printType(type), + bindToSelfType: bindToSelfType && evaluator.printType(bindToSelfType), + unpackedType: unpackedType && evaluator.printType(unpackedType), + inlinedTypeDict: inlinedTypeDict && evaluator.printType(inlinedTypeDict), + classType: classType && evaluator.printType(classType), + narrowedTypeForSet: narrowedTypeForSet && evaluator.printType(narrowedTypeForSet), + overloadsUsedForCall: overloadsUsedForCall?.map((t) => evaluator.printType(t)), + expectedTypeDiagAddendum: expectedTypeDiagAddendum?.getString(), + typeList: typeList?.map(({ node, ...value }) => ({ + node: { start: node.start, length: node.length, nodeType: node.nodeType }, + value: resultSummary(evaluator, value), + })), + }; +} + +function create( + content = fixture, + options: Partial | false = {}, + automaticConfiguration: boolean | 'default' = false, + observe = true, + checking?: ConstructorParameters[6] +) { + const temp = new RealTempFile(); + const service = createServiceProvider(createFromRealFileSystem(temp), new NullConsole(), temp); + const uri = UriEx.file(resolveSampleFilePath('overloadResultController1.py')); + const config = new ConfigOptions(uri.getDirectory()); + config.defaultPythonVersion = pythonVersion3_12; + config.internalTestMode = true; + if (automaticConfiguration !== 'default') { + config.experimentalOverloadResults = automaticConfiguration; + } + const resolver = new ImportResolver(service, config, new FullAccessHost(service)); + const events: OverloadResultEvent[] = []; + const experiment: ExperimentalOverloadResultOptions = { + seeds(module) { + const info = createAnalyzerNodeInfoAccessor(program.analyzerNodeInfoReader); + if (!info.getFileInfo(module).fileUri.equals(uri)) { + return []; + } + const walker = new Nodes(); + walker.walk(module); + return walker.nodes + .filter( + (n): n is CallNode => + n.nodeType === ParseNodeType.Call && + n.d.leftExpr.nodeType === ParseNodeType.Name && + n.d.leftExpr.d.value === 'choose' + ) + .map((node) => ({ + node, + uncertainty: TypeCategory.Any, + candidates(evaluator) { + const type = evaluator.getTypeOfExpression(node.d.leftExpr).type; + assert.ok(isOverloaded(type)); + return OverloadedType.getOverloads(type).map((f) => FunctionType.getEffectiveReturnType(f)!); + }, + })); + }, + ...(options || {}), + observe: observe + ? (event) => { + events.push(event); + if (options) { + options.observe?.(event); + } + } + : undefined, + }; + const program = new Program( + resolver, + config, + service, + undefined, + undefined, + undefined, + checking, + options === false ? undefined : options.automatic ? { ...experiment, seeds: undefined } : experiment + ); + program.setTrackedFiles([uri]); + program.setFileOpened(uri, 1, content); + program.getBoundSourceFile(uri); + const walker = new Nodes(); + walker.walk(program.getParseResults(uri)!.parserOutput.parseTree); + const module = program.getParseResults(uri)!.parserOutput.parseTree; + const info = createAnalyzerNodeInfoAccessor(program.analyzerNodeInfoReader); + const name = (value: string, fn = 'positive', occurrence = 0) => { + const names = walker.nodes.filter( + (n): n is NameNode => + n.nodeType === ParseNodeType.Name && + n.d.value === value && + getEnclosingFunction(n)?.d.name.d.value === fn + ); + assert.ok(names[occurrence]); + return names[occurrence]; + }; + const analyze = () => { + let count = 0; + while (program.analyze()) { + assert.ok(++count < 100); + } + return program + .getSourceFile(uri)! + .getDiagnostics(config)! + .map((d) => ({ category: d.category, rule: d.getRule(), message: d.message, range: d.range })); + }; + let disposed = false; + return { + program, + uri, + config, + events, + module, + nodes: walker.nodes, + name, + analyze, + info, + dispose() { + if (!disposed) { + program.dispose(); + service.dispose(); + disposed = true; + } + }, + }; +} + +test.each([false, true])('AutomaticOverloadResult live checker handoff queryFirst=%s', (queryFirst) => { + const baseline = create(fixture, { automatic: true, checkerHandoff: false }, false, false); + const automatic = create(fixture, { automatic: true, checkerHandoff: true }, false, false); + try { + if (queryFirst) { + for (const test of [baseline, automatic]) { + for (const node of [...test.nodes].reverse()) { + if (node.nodeType === ParseNodeType.Name && node.d.value === 'ret') { + test.program.evaluator!.getType(node); + } + } + } + } + const expected = baseline.analyze(); + assert.deepStrictEqual(automatic.analyze(), expected); + const before = baseline.program.experimentalOverloadResultController!.getStats(baseline.module)!; + const after = automatic.program.experimentalOverloadResultController!.getStats(automatic.module)!; + assert.equal(after.candidateCalls, before.candidateCalls); + assert.equal(after.candidateTrials, before.candidateTrials); + assert.equal(after.interruptedSlots, before.interruptedSlots); + // Reverse reads prepare earlier records; only the two trailing append + // statements remain fresh checker demands in that order. + assert.equal(after.checkerHandoffs, queryFirst ? 2 : 6); + assert.equal(after.ordinaryCalls, before.ordinaryCalls - (queryFirst ? 2 : 14)); + assert.equal(automatic.program.experimentalOverloadResultController!.getAutomaticStats().selected, 2); + for (const name of ['ret', 'alias', 'scalar', 'integer', 'string']) { + assert.deepStrictEqual( + resultSummary( + automatic.program.evaluator!, + automatic.program.evaluator!.getTypeResult(automatic.name(name))! + ), + resultSummary( + baseline.program.evaluator!, + baseline.program.evaluator!.getTypeResult(baseline.name(name))! + ) + ); + } + for (let i = 0; i < 3; i++) { + for (const test of [baseline, automatic]) { + test.program.experimentalOverloadResultController!.evict(test.module); + test.program.evaluator!.evaluateTypesForStatement(test.name('ret').parent!); + test.program.evaluator!.getTypeOfExpression(test.name('ret', 'positive', 1), EvalFlags.TypeExpression, { + expectedType: AnyType.create(), + }); + assert.deepStrictEqual(test.analyze(), expected); + assert.deepStrictEqual(test.info.getFileInfo(test.module).diagnosticSink.fetchAndClear(), []); + } + } + } finally { + baseline.dispose(); + automatic.dispose(); + } +}); + +test.each([0, 1, 2, 3, 4, 5, 7, 8, 9, 10])( + 'AutomaticOverloadResult checker handoff preserves candidate cutoffs calls=%s', + (consumerCalls) => { + const baseline = create(fixture, { automatic: true, checkerHandoff: false, consumerCalls }, false, false); + const automatic = create(fixture, { automatic: true, checkerHandoff: true, consumerCalls }, false, false); + try { + assert.deepStrictEqual(automatic.analyze(), baseline.analyze()); + const before = baseline.program.experimentalOverloadResultController!.getStats(baseline.module)!; + const after = automatic.program.experimentalOverloadResultController!.getStats(automatic.module)!; + assert.equal(after.candidateCalls, before.candidateCalls); + assert.equal(after.candidateTrials, before.candidateTrials); + assert.equal(after.candidateCallCeiling, before.candidateCallCeiling); + assert.equal(after.interruptedSlots, before.interruptedSlots); + assert.ok(after.ordinaryCalls <= before.ordinaryCalls); + assert.equal(after.checkerHandoffs > 0, consumerCalls >= 2); + assert.deepStrictEqual(automatic.info.getFileInfo(automatic.module).diagnosticSink.fetchAndClear(), []); + } finally { + baseline.dispose(); + automatic.dispose(); + } + } +); + +test('AutomaticOverloadResult checker handoff preserves checker-only unused-call diagnostics', () => { + const source = + '# pyright: reportUnusedCallResult=error\n' + + fixture.slice(0, fixture.indexOf('def project')) + + ` +def check(value: list[Any]): + ret = choose(value) + ret.pop().upper() +`; + const baseline = create(source, { automatic: true, checkerHandoff: false }, false, false); + const automatic = create(source, { automatic: true, checkerHandoff: true }, false, false); + try { + const expected = baseline.analyze(); + assert.ok(expected.some((d) => d.rule === 'reportUnusedCallResult')); + assert.deepStrictEqual(automatic.analyze(), expected); + assert.equal( + automatic.program.experimentalOverloadResultController!.getStats(automatic.module)!.checkerHandoffs, + 1 + ); + } finally { + baseline.dispose(); + automatic.dispose(); + } +}); + +test.each(['replace', 'incomplete', 'in-place'] as const)( + 'AutomaticOverloadResult checker handoff declines changed native result %s', + (change) => { + const factory = TypeEvaluatorModule.createTypeEvaluator; + let declines = 0; + const spy = jest.spyOn(TypeEvaluatorModule, 'createTypeEvaluator').mockImplementation((...args) => { + const controller = args[1].experimentalOverloadResult; + if (controller) { + const install = controller.installCheckerHandoff; + controller.installCheckerHandoff = (handoff) => + install((root, expression, result, callback) => { + const oldErrors = result.typeErrors; + const hadErrors = Object.prototype.hasOwnProperty.call(result, 'typeErrors'); + if (change === 'in-place') { + result.typeErrors = true; + } else { + evaluator.setTypeResultForNode(expression, { + ...result, + ...(change === 'incomplete' ? { isIncomplete: true } : {}), + }); + } + try { + const accepted = handoff(root, expression, result, callback); + assert.equal(accepted, false); + declines++; + return accepted; + } finally { + if (hadErrors) { + result.typeErrors = oldErrors; + } else { + delete result.typeErrors; + } + } + }); + } + const evaluator = factory(...args); + return evaluator; + }); + const test = create(fixture, { automatic: true, checkerHandoff: true }, false, false); + try { + const diagnostics = test.analyze(); + assert.equal(declines, 6); + assert.equal(test.program.experimentalOverloadResultController!.getStats(test.module)!.checkerHandoffs, 0); + assert.deepStrictEqual(test.info.getFileInfo(test.module).diagnosticSink.fetchAndClear(), []); + spy.mockRestore(); + const baseline = create(fixture, { automatic: true, checkerHandoff: false }, false, false); + try { + assert.deepStrictEqual(diagnostics, baseline.analyze()); + } finally { + baseline.dispose(); + } + } finally { + spy.mockRestore(); + test.dispose(); + } + } +); + +test.each(['exception', 'cancel'] as const)( + 'AutomaticOverloadResult live checker transaction unwinds on %s', + (fault) => { + const test = create(fixture, { automatic: true, checkerHandoff: true }, false, false); + try { + const controller = test.program.experimentalOverloadResultController!; + const evaluator = test.program.evaluator!; + const member = test.name('append'); + const call = member.parent!.parent!; + const root = call.parent!; + assert.equal(root.nodeType, ParseNodeType.StatementList); + // Activation now follows a real producer request, not check(module). + evaluator.getType(test.name('ret')); + const owner = controller.check(test.module)!; + const sink = test.info.getFileInfo(test.module).diagnosticSink; + assert.throws( + () => + owner.walk(root, () => { + assert.ok(evaluator.getCachedType(call as CallNode)); + if (fault === 'cancel') { + evaluator.runWithCancellationToken( + CancellationToken.Cancelled, + evaluator.checkForCancellation + ); + } else { + throw new Error('checker interruption'); + } + }), + (error) => + fault === 'cancel' + ? OperationCanceledException.is(error) + : error instanceof Error && error.message === 'checker interruption' + ); + assert.strictEqual(test.info.getFileInfo(test.module).diagnosticSink, sink); + assert.deepStrictEqual(sink.fetchAndClear(), []); + assert.equal(evaluator.getCachedType(call as CallNode), undefined); + assert.ok(!evaluator.isSpeculativeModeInUse(undefined)); + assert.equal(controller.getStats(test.module)!.checkerHandoffs, 0); + owner.walk(root, () => evaluator.getType(call as CallNode)); + assert.equal(controller.getStats(test.module)!.checkerHandoffs, 1); + assert.ok(controller.getStats(test.module)!.retryReservations > 0); + assert.deepStrictEqual(sink.fetchAndClear(), []); + owner.dispose(); + } finally { + test.dispose(); + } + } +); + +test('AutomaticOverloadResult real matching preserves the fixed supported graph', () => { + const fixed = create(); + const automatic = create(fixture, { automatic: true }); + try { + assert.deepStrictEqual(automatic.analyze(), fixed.analyze()); + assert.ok( + automatic.events.some((e) => e.kind === 'selection' && e.selection.reason === 'selected'), + JSON.stringify(automatic.events.flatMap((e) => (e.kind === 'selection' ? [e.selection.reason] : []))) + ); + const evaluator = automatic.program.evaluator!; + assert.equal( + evaluator.printType(evaluator.getType(automatic.name('ret'))!), + 'OverloadResult[list[int], list[str]]' + ); + } finally { + fixed.dispose(); + automatic.dispose(); + } +}); + +test.each([false, true])('AutomaticOverloadResult public consumers and reentry queryFirst=%s', (queryFirst) => { + const test = create(fixture, { automatic: true }); + try { + const evaluator = test.program.evaluator!; + if (queryFirst) { + assert.ok(isOverloadResult(evaluator.getType(test.name('alias'))!)); + } + const diagnostics = test.analyze(); + const alias = test.name('alias'); + const canonical = evaluator.getType(alias); + assert.ok(canonical && isOverloadResult(canonical)); + const position = (offset: number) => { + const prefix = fixture.slice(0, offset); + return { line: prefix.split('\n').length - 1, character: offset - prefix.lastIndexOf('\n') - 1 }; + }; + const hover = new HoverProvider( + test.program, + test.uri, + position(alias.start), + 'markdown', + CancellationToken.None + ).getHover(); + assert.ok(JSON.stringify(hover).includes('OverloadResult[list[int], list[str]]')); + for (const [text, labels] of [ + ['ret.append("x")', ['append', 'clear']], + ['scalar.upper()', ['upper', 'bit_length']], + ] as const) { + const offset = fixture.indexOf(text) + text.indexOf('.') + 1; + const provider = new CompletionProvider( + test.program, + test.uri, + position(offset), + { format: 'markdown', snippet: false, lazyEdit: false }, + CancellationToken.None + ); + const completions = provider.getCompletions(); + for (const label of labels) { + const completion = completions?.items.find((item) => item.label === label); + assert.ok(completion, `${text}: ${label}`); + provider.resolveCompletionItem(completion); + } + } + const controller = test.program.experimentalOverloadResultController!; + const selections = test.events.filter((e) => e.kind === 'selection').length; + const target = test.name('string'); + assert.ok(target.parent?.parent?.nodeType === ParseNodeType.Assignment); + for (let round = 0; round < 3; round++) { + controller.evict(test.module); + evaluator.evaluateTypesForStatement(alias.parent!); + evaluator.evaluateTypesForStatement(target.parent.parent); + assert.equal(evaluator.printType(evaluator.getCachedType(target)!), 'list[str]'); + assert.strictEqual(evaluator.getType(alias), canonical); + assert.strictEqual(evaluator.getTypeResult(alias)!.type, canonical); + assert.strictEqual(evaluator.getTypeOfExpression(alias).type, canonical); + assert.deepStrictEqual(test.info.getFileInfo(test.module).diagnosticSink.fetchAndClear(), []); + } + assert.equal(test.events.filter((e) => e.kind === 'selection').length, selections); + assert.deepStrictEqual(test.analyze(), diagnostics); + } finally { + test.dispose(); + } +}); + +test.each([false, true])( + 'OverloadResultController cold reverse use queries and no-return reentry automatic=%s', + (automatic) => { + const warm = create(fixture, { automatic }); + const cold = create(fixture, { automatic }); + try { + const expected = warm.analyze(); + const targets = cold.nodes.filter( + (node): node is NameNode => + node.nodeType === ParseNodeType.Name && + ['ret', 'alias', 'string', 'integer', 'projected', 'scalar'].includes(node.d.value) + ); + for (const target of [...targets].reverse()) { + cold.program.evaluator!.getType(target); + } + // Queries of declined/ordinary names can emit their ordinary errors + // before the walk. Compare the full multiset, including duplicates. + const ordered = (diagnostics: typeof expected) => + [...diagnostics].sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b))); + assert.deepStrictEqual(ordered(cold.analyze()), ordered(expected)); + for (const target of targets) { + const other = warm.nodes.find( + (node): node is NameNode => node.nodeType === ParseNodeType.Name && node.start === target.start + )!; + assert.deepStrictEqual( + resultSummary(cold.program.evaluator!, cold.program.evaluator!.getTypeOfExpression(target)), + resultSummary(warm.program.evaluator!, warm.program.evaluator!.getTypeOfExpression(other)) + ); + } + assert.deepStrictEqual(cold.info.getFileInfo(cold.module).diagnosticSink.fetchAndClear(), []); + } finally { + warm.dispose(); + cold.dispose(); + } + } +); + +test('AutomaticOverloadResult exact target and declined controls', () => { + const source = readSampleFile('overloadResultAutomatic1.py'); + const ordinary = create(source, false); + const enabled = create(source, { automatic: true }); + try { + // Rejected early-use graphs must remain ordinary even before checking. + const evaluator = enabled.program.evaluator!; + assert.equal(evaluator.printType(evaluator.getType(enabled.name('ret', 'early', 1))!), 'list[int]'); + const expected = ordinary.analyze(); + const actual = enabled.analyze(); + const inFunction = (diagnostics: typeof actual, fn: string) => { + const node = getEnclosingFunction(enabled.name('ret', fn))!; + const start = source.slice(0, node.start).split('\n').length - 1; + const end = source.slice(0, node.start + node.length).split('\n').length - 1; + return diagnostics.filter((d) => d.range.start.line >= start && d.range.start.line <= end); + }; + for (const fn of [ + 'prefix', + 'equal_returns', + 'default_argument', + 'type_variable', + 'unsupported_proof', + 'one_mapping', + 'concrete', + 'unknown', + 'any_argument', + 'union_argument', + 'keywords', + 'unpack', + 'receiver', + 'early', + ]) { + assert.deepStrictEqual(inFunction(actual, fn), inFunction(expected, fn), fn); + assert.deepStrictEqual( + resultSummary(evaluator, evaluator.getTypeOfExpression(enabled.name('ret', fn))), + resultSummary( + ordinary.program.evaluator!, + ordinary.program.evaluator!.getTypeOfExpression(ordinary.name('ret', fn)) + ), + fn + ); + } + for (const [fn, type] of [ + ['original', 'OverloadResult[list[int], list[str]]'], + ['nested_invariant', 'OverloadResult[list[list[int]], list[list[str]]]'], + ['custom_nominal', 'OverloadResult[Invariant[int], Invariant[str]]'], + ['independent', 'OverloadResult[list[int], list[str]]'], + ]) { + assert.equal(evaluator.printType(evaluator.getType(enabled.name('ret', fn))!), type, fn); + } + const selections = enabled.events.flatMap((e) => + e.kind === 'selection' || e.kind === 'activationProbe' ? [e] : [] + ); + const prefix = selections.find((e) => getEnclosingFunction(e.node)?.d.name.d.value === 'prefix')!; + assert.equal(prefix.selection.reason, 'unsupported-return-shape'); + assert.deepStrictEqual(prefix.selection.candidates, []); + const pair = selections.find((e) => getEnclosingFunction(e.node)?.d.name.d.value === 'original')!; + assert.deepStrictEqual(pair.selection.coverage, ['not-covered', 'not-covered']); + assert.equal( + selections.find((e) => getEnclosingFunction(e.node)?.d.name.d.value === 'equal_returns')!.selection.reason, + 'equivalent-or-single-return' + ); + const original = inFunction(actual, 'original'); + assert.deepStrictEqual( + original.filter((d) => d.rule === 'reportInvalidTypeForm'), + inFunction(expected, 'original').filter((d) => d.rule === 'reportInvalidTypeForm') + ); + assert.equal(original.filter((d) => d.rule === 'reportAssertTypeFailure').length, 2); + assert.ok(original.some((d) => d.message.includes('OverloadResult[list[int], list[str]]'))); + for (const text of ['ret.append(3.14)', 'ret.nonexistent()', 'invalid: list[bytes]', 'unrelated: int']) { + const line = source.slice(0, source.indexOf(text)).split('\n').length - 1; + assert.deepStrictEqual( + actual.filter((d) => d.range.start.line === line), + expected.filter((d) => d.range.start.line === line), + text + ); + } + assert.deepStrictEqual(enabled.info.getFileInfo(enabled.module).diagnosticSink.fetchAndClear(), []); + } finally { + ordinary.dispose(); + enabled.dispose(); + } +}); + +test('AutomaticOverloadResult re-enables the explicit-off internal Program control without injected seeds', () => { + const test = create(readSampleFile('overloadResultAutomatic1.py'), false); + try { + test.config.experimentalOverloadResults = true; + test.program.setConfigOptions(test.config); + test.analyze(); + const controller = test.program.experimentalOverloadResultController!; + assert.ok(isOverloadResult(test.program.evaluator!.getType(test.name('ret', 'original'))!)); + assert.ok(controller.getAutomaticStats().selected > 0); + } finally { + test.dispose(); + } +}); + +test('AutomaticOverloadResult omitted default preserves reviewed diagnostics and metadata without handoffs', () => { + const source = readSampleFile('overloadResultAutomatic1.py'); + const normal = create(source, false, 'default', false); + const reviewed = create(source, { automatic: true, checkerHandoff: false }, false, false); + try { + assert.strictEqual(normal.config.experimentalOverloadResults, true); + assert.deepStrictEqual(normal.analyze(), reviewed.analyze()); + for (const fn of ['original', 'nested_invariant', 'custom_nominal', 'concrete', 'unknown', 'early']) { + const actual = normal.program.evaluator!; + const expected = reviewed.program.evaluator!; + assert.deepStrictEqual( + resultSummary(actual, actual.getTypeOfExpression(normal.name('ret', fn))), + resultSummary(expected, expected.getTypeOfExpression(reviewed.name('ret', fn))), + fn + ); + } + const evaluator = normal.program.evaluator!; + assert.strictEqual( + evaluator.printType(evaluator.getType(normal.name('ret', 'original'))!), + 'OverloadResult[list[int], list[str]]' + ); + assert.strictEqual(evaluator.printType(evaluator.getType(normal.name('ret', 'concrete'))!), 'list[str]'); + const controller = normal.program.experimentalOverloadResultController!; + assert.ok(controller.getAutomaticStats().selected > 0); + assert.strictEqual(controller.getStats(normal.module)!.checkerHandoffs, 0); + assert.deepStrictEqual(normal.info.getFileInfo(normal.module).diagnosticSink.fetchAndClear(), []); + } finally { + normal.dispose(); + reviewed.dispose(); + } +}); + +test.each<{ + name: string; + config: boolean | 'default'; + options: Partial | false; + automatic: boolean; + carrier: boolean; +}>([ + { name: 'omitted configuration', config: 'default', options: false, automatic: true, carrier: true }, + { name: 'explicit internal on', config: true, options: false, automatic: true, carrier: true }, + { name: 'explicit internal off', config: false, options: false, automatic: false, carrier: false }, + { + name: 'injected automatic overrides internal off without implying handoff', + config: false, + options: { automatic: true }, + automatic: true, + carrier: true, + }, + { + name: 'injected fixed seeds override automatic default', + config: 'default', + options: { automatic: false }, + automatic: false, + carrier: true, + }, +])('AutomaticOverloadResult option precedence: $name', ({ config, options, automatic, carrier }) => { + const target = create(fixture, options, config, false); + try { + target.analyze(); + const evaluator = target.program.evaluator!; + assert.strictEqual(isOverloadResult(evaluator.getType(target.name('ret'))!), carrier); + if (!carrier) { + assert.strictEqual(evaluator.printType(evaluator.getType(target.name('ret'))!), 'list[int]'); + assert.strictEqual(target.program.experimentalOverloadResultController, undefined); + } else { + const controller = target.program.experimentalOverloadResultController!; + assert.strictEqual(controller.getAutomaticStats().selected > 0, automatic); + assert.strictEqual(controller.getStats(target.module)!.checkerHandoffs, 0); + } + } finally { + target.dispose(); + } +}); + +test('AutomaticOverloadResult explicit test-only checking controller overrides the automatic default', () => { + let expressionFactories = 0; + const target = create(fixture, false, 'default', false, { + expression: () => { + expressionFactories++; + return undefined; + }, + check: () => undefined, + }); + try { + target.analyze(); + assert.ok(expressionFactories > 0); + assert.strictEqual(target.config.experimentalOverloadResults, true); + assert.strictEqual(target.program.experimentalOverloadResultController, undefined); + const evaluator = target.program.evaluator!; + assert.strictEqual(evaluator.printType(evaluator.getType(target.name('ret'))!), 'list[int]'); + } finally { + target.dispose(); + } +}); + +test.each(['check-first', 'value-first', 'use-first'])( + 'AutomaticOverloadResult gradual returns preserve exact ordinary errors: %s', + (order) => { + const source = readSampleFile('overloadResultDynamicReturns1.py'); + const ordinary = create(source, false); + const automatic = create(source, { automatic: true, checkerHandoff: false }); + const expected = [ + { + category: DiagnosticCategory.Error, + rule: 'reportAttributeAccessIssue', + message: + 'Cannot access attribute "missing_member" for class "list[int]"\n' + + '\u00a0\u00a0Attribute "missing_member" is unknown', + range: { start: { line: 13, character: 8 }, end: { line: 13, character: 22 } }, + }, + { + category: DiagnosticCategory.Error, + rule: 'reportAttributeAccessIssue', + message: + 'Cannot access attribute "missing_member" for class "int"\n' + + '\u00a0\u00a0Attribute "missing_member" is unknown', + range: { start: { line: 26, character: 14 }, end: { line: 26, character: 28 } }, + }, + ]; + const uses = (test: ReturnType) => + test.nodes.filter( + (node): node is CallNode => + node.nodeType === ParseNodeType.Call && + node.d.leftExpr.nodeType === ParseNodeType.MemberAccess && + node.d.leftExpr.d.member.d.value === 'missing_member' + ); + try { + for (const test of [ordinary, automatic]) { + if (order === 'value-first') { + for (const fn of ['root_dynamic_return', 'nested_dynamic_return']) { + const type = test.program.evaluator!.getType(test.name('ret', fn))!; + assert.ok(!isOverloadResult(type)); + assert.equal(test.program.evaluator!.printType(type), 'list[int]'); + } + } else if (order === 'use-first') { + uses(test).forEach((node) => test.program.evaluator!.getTypeOfExpression(node)); + } + } + assert.deepStrictEqual(ordinary.analyze(), expected); + assert.deepStrictEqual(automatic.analyze(), expected); + const evaluator = automatic.program.evaluator!; + const controller = automatic.program.experimentalOverloadResultController!; + assert.equal(controller.getAutomaticStats().selected, 0); + const rejected = automatic.events.filter( + (e) => e.kind === 'activationProbe' && e.selection.reason === 'unsupported-return-shape' + ); + assert.equal(new Set(rejected.map((e) => e.kind === 'activationProbe' && e.node)).size, 2); + assert.equal(controller.getAutomaticStats().selections, 0); + assert.equal(controller.getStats(automatic.module), undefined); + const selections = controller.getAutomaticStats().selections; + for (let round = 0; round < 3; round++) { + controller.evict(automatic.module); + for (const fn of ['root_dynamic_return', 'nested_dynamic_return']) { + const target = automatic.name('ret', fn); + evaluator.evaluateTypesForStatement(target.parent!); + assert.equal(evaluator.printType(evaluator.getCachedType(target)!), 'list[int]'); + assert.deepStrictEqual( + resultSummary(evaluator, evaluator.getTypeOfExpression(target)), + resultSummary( + ordinary.program.evaluator!, + ordinary.program.evaluator!.getTypeOfExpression(ordinary.name('ret', fn)) + ) + ); + assert.ok(!isOverloadResult(evaluator.getType(target)!)); + assert.ok(!isOverloadResult(evaluator.getTypeResult(target)!.type)); + } + uses(automatic).forEach((node, index) => + assert.deepStrictEqual( + resultSummary(evaluator, evaluator.getTypeOfExpression(node)), + resultSummary( + ordinary.program.evaluator!, + ordinary.program.evaluator!.getTypeOfExpression(uses(ordinary)[index]) + ) + ) + ); + assert.deepStrictEqual(automatic.info.getFileInfo(automatic.module).diagnosticSink.fetchAndClear(), []); + assert.deepStrictEqual(automatic.analyze(), expected); + assert.equal(controller.getAutomaticStats().selections, selections); + } + } finally { + ordinary.dispose(); + automatic.dispose(); + } + } +); + +test.each(['int', 'str'])('AutomaticOverloadResult static covering prefix retains %s before object', (first) => { + const second = first === 'int' ? 'str' : 'int'; + const source = `from typing import Any, overload +@overload +def choose(value: list[${first}]) -> list[${first}]: ... +@overload +def choose(value: list[Any]) -> object: ... +@overload +def choose(value: list[${second}]) -> list[${second}]: ... +def choose(value: Any) -> Any: + return value +def positive(value: list[Any]): + ret = choose(value) + ret.clear() + ret.append(3.14) +`; + const ordinary = create(source, false); + const automatic = create(source, { automatic: true }); + try { + assert.deepStrictEqual(automatic.analyze(), ordinary.analyze()); + const evaluator = automatic.program.evaluator!; + assert.equal( + evaluator.printType(evaluator.getType(automatic.name('ret'))!), + `OverloadResult[list[${first}], object]` + ); + const selection = automatic.events.find((e) => e.kind === 'selection'); + assert.ok(selection?.kind === 'selection'); + assert.deepStrictEqual(selection.selection.coverage, ['not-covered', 'covered']); + assert.deepStrictEqual(selection.selection.matches, [0, 1]); + assert.equal(selection.selection.candidates.length, 2); + } finally { + ordinary.dispose(); + automatic.dispose(); + } +}); + +test.each([ + 'Any', + 'list[Any]', + 'list[list[Any]]', + 'dict[str, list[Any]]', + 'GradualAlias', + 'list', + 'list[list]', + 'list[int] | Any', + 'tuple[Any]', + 'Callable[..., Any]', +])('AutomaticOverloadResult rejects gradual or unsupported return %s before use-first queries', (returnType) => { + const source = `from typing import Any, Callable, overload +GradualAlias = dict[str, list[Any]] +@overload +def choose(value: list[int]) -> list[int]: ... +@overload +def choose(value: list[str]) -> ${returnType}: ... +def choose(value: Any) -> Any: + return value +def positive(value: list[Any]): + ret = choose(value) + ret.missing_member() +`; + const ordinary = create(source, false); + const automatic = create(source, false, true); + try { + const evaluator = automatic.program.evaluator!; + const use = automatic.nodes.find( + (node): node is CallNode => + node.nodeType === ParseNodeType.Call && + node.d.leftExpr.nodeType === ParseNodeType.MemberAccess && + node.d.leftExpr.d.member.d.value === 'missing_member' + )!; + evaluator.getTypeOfExpression(use); + assert.deepStrictEqual(automatic.analyze(), ordinary.analyze()); + assert.ok(automatic.analyze().some((d) => d.rule === 'reportAttributeAccessIssue')); + assert.equal(automatic.program.experimentalOverloadResultController!.getAutomaticStats().selected, 0); + assert.equal(evaluator.printType(evaluator.getType(automatic.name('ret'))!), 'list[int]'); + assert.deepStrictEqual( + resultSummary(evaluator, evaluator.getTypeOfExpression(automatic.name('ret'))), + resultSummary( + ordinary.program.evaluator!, + ordinary.program.evaluator!.getTypeOfExpression(ordinary.name('ret')) + ) + ); + assert.deepStrictEqual(automatic.info.getFileInfo(automatic.module).diagnosticSink.fetchAndClear(), []); + } finally { + ordinary.dispose(); + automatic.dispose(); + } +}); + +test('AutomaticOverloadResult original implicit reveal reports the actual canonical value', () => { + // A reveal compares the whole carrier, not one successful alternative. + const source = readSampleFile('overloadCall11.py').replace( + 'expected_text="OverloadResult[list[int], list[str]]"', + 'expected_text="list[int]"' + ); + const ordinary = create(source, false); + const automatic = create(source, { automatic: true }); + try { + assert.equal(ordinary.analyze().filter((d) => d.category === DiagnosticCategory.Error).length, 2); + const actual = automatic.analyze(); + const evaluator = automatic.program.evaluator!; + assert.equal( + evaluator.printType(evaluator.getType(automatic.name('result', 'check'))!), + 'OverloadResult[list[int], list[str]]' + ); + assert.ok(actual.some((d) => d.message.includes('OverloadResult[list[int], list[str]]'))); + assert.equal(actual.filter((d) => d.category === DiagnosticCategory.Error).length, 1); + assert.ok(!actual.some((d) => d.rule === 'reportAssignmentType' || d.rule === 'reportArgumentType')); + } finally { + ordinary.dispose(); + automatic.dispose(); + } +}); + +test.each([false, true])('AutomaticOverloadResult public subnode queries queryFirst=%s', (queryFirst) => { + const source = `from typing import Any, overload, reveal_type +@overload +def choose(value: list[int]) -> list[int]: ... +@overload +def choose(value: list[str]) -> list[str]: ... +def choose(value: Any) -> Any: + return value +def ordinary(value: list[int]): + ret = choose(value) + reveal_type(ret) + ret.append(1) +def active(value: list[Any]): + ret = choose(value) + reveal_type(ret) + ret.append(1) +`; + const ordinary = create(source, false); + const automatic = create(source, false, true); + try { + if (!queryFirst) { + ordinary.analyze(); + automatic.analyze(); + } + const evaluator = automatic.program.evaluator!; + const readerArgument = automatic.name('ret', 'ordinary', 1); + const baselineArgument = ordinary.name('ret', 'ordinary', 1); + const readerResult = evaluator.getTypeResult(readerArgument); + const baselineResult = ordinary.program.evaluator!.getTypeResult(baselineArgument); + assert.ok(readerResult && baselineResult); + assert.deepStrictEqual( + resultSummary(evaluator, readerResult), + resultSummary(ordinary.program.evaluator!, baselineResult) + ); + assert.equal(evaluator.printType(evaluator.getType(readerArgument)!), 'list[int]'); + const restored = automatic.program.experimentalOverloadResultController!.getStats(automatic.module)!; + for (let i = 0; i < 3; i++) { + assert.deepStrictEqual( + resultSummary(evaluator, evaluator.getTypeResult(readerArgument)!), + resultSummary(ordinary.program.evaluator!, baselineResult) + ); + assert.equal(evaluator.printType(evaluator.getType(readerArgument)!), 'list[int]'); + } + assert.deepStrictEqual( + automatic.program.experimentalOverloadResultController!.getStats(automatic.module), + restored + ); + const member = automatic.name('append', 'active'); + const parent = member.parent!; + assert.ok(parent.nodeType === ParseNodeType.MemberAccess); + assert.strictEqual(evaluator.getType(member), evaluator.getType(parent)); + assert.strictEqual(evaluator.getTypeResult(member), evaluator.getTypeResult(parent)); + assert.ok(isOverloadResult(evaluator.getType(member)!)); + const position = (node: NameNode) => { + const before = source.slice(0, node.start); + return { line: before.split('\n').length - 1, character: node.start - before.lastIndexOf('\n') - 1 }; + }; + for (const [node, expected] of [ + [readerArgument, 'list[int]'], + [member, 'OverloadResult'], + ] as const) { + const hover = new HoverProvider( + automatic.program, + automatic.uri, + position(node), + 'markdown', + CancellationToken.None + ).getHover(); + assert.ok(JSON.stringify(hover).includes(expected)); + assert.ok(!JSON.stringify(hover).includes('Unknown')); + } + const expectedDiagnostics = automatic.analyze(); + for (let i = 0; i < 3; i++) { + automatic.program.experimentalOverloadResultController!.evict(automatic.module); + const evicted = automatic.program.experimentalOverloadResultController!.getStats(automatic.module)!; + assert.equal(evaluator.printType(evaluator.getType(readerArgument)!), 'list[int]'); + assert.equal( + automatic.program.experimentalOverloadResultController!.getStats(automatic.module)!.ordinaryRequests, + evicted.ordinaryRequests + ); + evaluator.evaluateTypesForStatement(automatic.name('ret', 'ordinary').parent!); + evaluator.evaluateTypesForStatement(automatic.name('ret', 'active').parent!); + assert.equal(evaluator.printType(evaluator.getType(readerArgument)!), 'list[int]'); + assert.strictEqual(evaluator.getType(member), evaluator.getType(parent)); + assert.deepStrictEqual(automatic.analyze(), expectedDiagnostics); + assert.deepStrictEqual(automatic.info.getFileInfo(automatic.module).diagnosticSink.fetchAndClear(), []); + } + assert.equal(automatic.program.experimentalOverloadResultController!.getAutomaticStats().selected, 1); + } finally { + ordinary.dispose(); + automatic.dispose(); + } +}); + +test.each([false, true])( + 'AutomaticOverloadResult activation preserves producer diagnostics queryFirst=%s', + (queryFirst) => { + const source = + fixture.slice(0, fixture.indexOf('def project')) + + `def wrong(value: list[Any]): + ret: list[bytes] = choose(value) + ret.append(b"ok") +def invalid(value: list[Any]): + ret: list[int()] = choose(value) + ret.append("ok") +def good(value: list[Any]): + ret: list[int] = choose(value) + ret.append("bad") +`; + const ordinary = create(source, false); + const automatic = create(source, { automatic: true, checkerHandoff: false }); + try { + if (!queryFirst) { + ordinary.analyze(); + automatic.analyze(); + } + for (const fn of ['good', 'invalid', 'wrong']) { + assert.deepStrictEqual( + resultSummary( + automatic.program.evaluator!, + automatic.program.evaluator!.getTypeResult(automatic.name('ret', fn, 1))! + ), + resultSummary( + ordinary.program.evaluator!, + ordinary.program.evaluator!.getTypeResult(ordinary.name('ret', fn, 1))! + ) + ); + } + const earlyErrors = automatic.info.getFileInfo(automatic.module).diagnosticSink.getErrors(); + assert.equal(earlyErrors.length, queryFirst ? 1 : 0); + if (queryFirst) { + // The annotation is validated before the call in both lifecycles. + const prefix = source.slice(0, source.indexOf('int()')); + const line = prefix.split('\n').length - 1; + const character = prefix.length - prefix.lastIndexOf('\n') - 1; + assert.equal(earlyErrors[0].getRule(), 'reportInvalidTypeForm'); + assert.equal(earlyErrors[0].message, 'Call expression not allowed in type expression'); + assert.deepStrictEqual(earlyErrors[0].range, { + start: { line, character }, + end: { line, character: character + 5 }, + }); + } + assert.deepStrictEqual(automatic.analyze(), ordinary.analyze()); + const errors = automatic.analyze().filter((d) => d.category === DiagnosticCategory.Error); + assert.deepStrictEqual(errors.map((d) => d.rule).sort(), [ + 'reportArgumentType', + 'reportAssignmentType', + 'reportInvalidTypeForm', + ]); + } finally { + ordinary.dispose(); + automatic.dispose(); + } + } +); + +test.each(['analysis-first', 'later-producer-first', 'later-consumer-first', 'later-function-first'] as const)( + 'AutomaticOverloadResult independent activation order and replacement: %s', + (order) => { + const original = readSampleFile('overloadResultActivationOrder1.py'); + const edited = original.replace('first = choose(value)', 'first = [b"edited"]'); + const automatic = create(original, { automatic: true, checkerHandoff: false }); + const reference = create(original, { automatic: true, checkerHandoff: false }); + const ordinary = create(original, false); + const targets = [ + ['first', 'OverloadResult[list[int], list[str]]'], + ['first_alias', 'OverloadResult[list[int], list[str]]'], + ['first_projected', 'OverloadResult[Sequence[int], Sequence[str]]'], + ['first_scalar', 'OverloadResult[int, str]'], + ['first_int', 'list[int]'], + ['first_str', 'list[str]'], + ['second', 'OverloadResult[list[bytes], list[float]]'], + ['second_alias', 'OverloadResult[list[bytes], list[float]]'], + ['second_projected', 'OverloadResult[Sequence[bytes], Sequence[float]]'], + ['second_scalar', 'OverloadResult[bytes, float]'], + ['second_bytes', 'list[bytes]'], + ['second_float', 'list[float]'], + ['remote', 'OverloadResult[list[bytes], list[float]]'], + ['bad_producer', 'list[bytes]'], + ['ordinary', 'list[int]'], + ['escaped', 'list[int]'], + ] as const; + const current = (test: ReturnType) => { + test.program.getBoundSourceFile(test.uri); + const module = test.program.getParseResults(test.uri)!.parserOutput.parseTree; + const nodes = new Nodes(); + nodes.walk(module); + const name = (value: string) => { + const fn = value === 'remote' ? 'independent' : value === 'escaped' ? 'declined' : 'positive'; + const node = nodes.nodes.find( + (n): n is NameNode => + n.nodeType === ParseNodeType.Name && + n.d.value === value && + getEnclosingFunction(n)?.d.name.d.value === fn + ); + assert.ok(node, value); + return node; + }; + return { module, name, evaluator: test.program.evaluator! }; + }; + const ordered = (diagnostics: readonly T[]) => + [...diagnostics].sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b))); + const fullDiagnostics = (test: ReturnType) => + test.program + .getSourceFile(test.uri)! + .getDiagnostics(test.config)! + .map((d) => d.toJsonObj()); + const sink = (test: ReturnType, module: ParseNode) => + test.info + .getFileInfo(module) + .diagnosticSink.getErrors() + .map((d) => d.toJsonObj()); + try { + for (const [generation, source] of [original, edited, original].entries()) { + const previous = current(automatic); + const previousType = generation ? previous.evaluator.getType(previous.name('second')) : undefined; + const eventStart = automatic.events.length; + const activationsBefore = + automatic.program.experimentalOverloadResultController!.getAutomaticStats().activations; + if (generation) { + for (const test of [automatic, reference, ordinary]) { + test.program.setFileOpened(test.uri, generation + 1, source); + } + } + const actual = current(automatic); + const expected = current(reference); + const native = current(ordinary); + const nativeDiagnostics = ordinary.analyze(); + const referenceDiagnostics = reference.analyze(); + const line = (text: string) => source.slice(0, source.indexOf(text)).split('\n').length - 1; + const failures = [ + 'between: int = "bad"', + 'first_invalid: list[bytes]', + 'first.extend([1, "x"])', + 'second.extend([b"x", 1.0])', + 'first.append(3.14)', + 'second.append("neither")', + 'first.nonexistent()', + 'second.nonexistent()', + 'bad_producer: list[bytes]', + 'ordinary.append("bad")', + 'escaped.append("not_supported")', + ] + .filter((text) => generation !== 1 || text !== 'first_invalid: list[bytes]') + .map(line); + // After editing away the first producer, all of its operations + // must have precisely their native (non-alternative) behavior. + const firstLines = source + .split('\n') + .flatMap((text, index) => + generation === 1 && text.trimStart().startsWith('first') ? [index] : [] + ); + const keepDiagnostic = (d: (typeof nativeDiagnostics)[number]) => + d.category !== DiagnosticCategory.Error || + [...failures, ...firstLines].includes(d.range.start.line); + const diagnostics = nativeDiagnostics.filter(keepDiagnostic); + const fullExpected = fullDiagnostics(ordinary).filter((_, index) => + keepDiagnostic(nativeDiagnostics[index]) + ); + for (const failure of failures) { + const errors = diagnostics.filter( + (d) => d.category === DiagnosticCategory.Error && d.range.start.line === failure + ); + assert.equal(errors.length, 1, JSON.stringify({ generation, line: failure, errors })); + } + assert.deepStrictEqual(ordered(referenceDiagnostics), ordered(diagnostics)); + assert.deepStrictEqual(ordered(fullDiagnostics(reference)), ordered(fullExpected)); + + if (order === 'analysis-first') { + assert.deepStrictEqual(ordered(automatic.analyze()), ordered(diagnostics)); + } else { + const firstQuery = + order === 'later-function-first' + ? 'remote' + : order === 'later-consumer-first' + ? 'second_scalar' + : 'second'; + const value = actual.evaluator.getTypeResult(actual.name(firstQuery))!; + assert.ok(isOverloadResult(value.type)); + assert.deepStrictEqual( + resultSummary(actual.evaluator, value), + resultSummary(expected.evaluator, expected.evaluator.getTypeResult(expected.name(firstQuery))!) + ); + if (order === 'later-function-first') { + const probe = automatic.events.slice(eventStart).find((e) => e.kind === 'activationProbe'); + assert.ok(probe?.kind === 'activationProbe'); + assert.equal(getEnclosingFunction(probe.node)?.d.name.d.value, 'independent'); + } + } + for (let repeat = 0; repeat < 3; repeat++) { + for (const [name, type] of [...targets].reverse()) { + const node = actual.name(name); + const other = expected.name(name); + const query = actual.evaluator.getTypeResult(node)!; + const expectedType = + generation === 1 && name.startsWith('first') + ? native.evaluator.printType(native.evaluator.getType(native.name(name))!) + : type; + assert.equal(actual.evaluator.printType(query.type), expectedType, name); + assert.strictEqual(actual.evaluator.getType(node), query.type, name); + assert.deepStrictEqual( + resultSummary(actual.evaluator, query), + resultSummary(expected.evaluator, expected.evaluator.getTypeResult(other)!), + name + ); + assert.deepStrictEqual( + resultSummary(actual.evaluator, actual.evaluator.getTypeOfExpression(node)), + resultSummary(expected.evaluator, expected.evaluator.getTypeOfExpression(other)), + name + ); + } + assert.deepStrictEqual(ordered(automatic.analyze()), ordered(diagnostics)); + assert.deepStrictEqual(ordered(fullDiagnostics(automatic)), ordered(fullExpected)); + assert.deepStrictEqual(sink(automatic, actual.module), sink(reference, expected.module)); + assert.deepStrictEqual( + automatic.info.getFileInfo(actual.module).diagnosticSink.fetchAndClear(), + [] + ); + } + for (const name of ['first_scalar', 'second_scalar']) { + const node = actual.name(name); + const prefix = source.slice(0, node.start); + const position = { + line: prefix.split('\n').length - 1, + character: node.start - prefix.lastIndexOf('\n') - 1, + }; + const hover = (test: ReturnType) => + new HoverProvider( + test.program, + test.uri, + position, + 'markdown', + CancellationToken.None + ).getHover(); + const actualHover = hover(automatic); + assert.deepStrictEqual(actualHover, hover(reference)); + assert.ok( + JSON.stringify(actualHover).includes( + actual.evaluator.printType(actual.evaluator.getType(node)!) + ) + ); + } + const events = automatic.events.slice(eventStart); + const admission = events.find((e) => e.kind === 'admission'); + assert.ok(admission?.kind === 'admission' && admission.admission.complete); + const controller = automatic.program.experimentalOverloadResultController!; + const owners: Extract[] = []; + for (const event of events) { + if (event.kind === 'enter') { + owners.push(event); + } else if (event.kind === 'trial') { + const owner = owners.pop(); + assert.strictEqual(owner?.root, event.root); + assert.equal(owner?.mode, event.mode); + } else if (event.kind === 'activationProbe' && owners.length) { + assert.equal(owners.length, 1); + assert.equal(owners[0].mode, 'baseline'); + assert.equal(controller.roots.get(owners[0].root)!.has(event.node.id), false); + } + } + assert.equal(owners.length, 0); + for (const name of generation === 1 ? ['second', 'remote'] : ['first', 'second', 'remote']) { + const record = admission.admission.definitions.get(actual.name(name)); + assert.ok(record?.admitted && record.seed); + assert.ok(controller.roots.has(record.root)); + assert.ok( + events.some( + (e) => + e.kind === 'selection' && + e.node === record.seed!.node && + e.selection.reason === 'selected' + ) + ); + assert.ok( + events.some( + (e) => e.kind === 'value' && e.root === record.root && isOverloadResult(e.canonical.type) + ) + ); + } + for (const name of ['ordinary', 'escaped']) { + const record = admission.admission.definitions.get(actual.name(name)); + assert.ok(record); + assert.equal(controller.roots.has(record.root), false); + assert.ok(!isOverloadResult(actual.evaluator.getType(actual.name(name))!)); + } + const annotated = admission.admission.definitions.get(actual.name('bad_producer')); + assert.ok(annotated?.admitted && annotated.annotated && annotated.seed); + assert.ok(controller.roots.has(annotated.root)); + assert.ok( + events.some( + (e) => + e.kind === 'activationProbe' && + e.node === annotated.seed!.node && + e.selection.reason === 'selected' + ) + ); + assert.ok( + events.some( + (e) => e.kind === 'value' && e.root === annotated.root && !isOverloadResult(e.canonical.type) + ) + ); + for (const name of generation === 1 ? ['second'] : ['first', 'second']) { + const record = [...admission.admission.records.values()].find( + (r) => r.statement.start === source.indexOf(`${name}.extend(`) + ); + assert.ok(record?.admitted && controller.roots.has(record.root)); + const trials = events.filter( + (e) => e.kind === 'trial' && e.root === record.root && e.mode === 'candidate' + ); + assert.equal(trials.length, 2); + assert.ok(trials.every((e) => e.kind === 'trial' && e.failed && !e.exception)); + } + assert.ok(!events.some((e) => e.kind === 'value' && e.cutoff)); + assert.equal(controller.hasPendingActivation, false); + assert.equal(controller.getAutomaticStats().activations - activationsBefore, generation === 1 ? 3 : 4); + assert.deepStrictEqual(automatic.info.getFileInfo(actual.module).diagnosticSink.fetchAndClear(), []); + if (generation) { + assert.ok(events.some((e) => e.kind === 'retire' && e.module === previous.module)); + assert.notStrictEqual(actual.evaluator.getType(actual.name('second')), previousType); + assert.throws( + () => previous.evaluator.getType(previous.name('second')), + /Retired overload-result controller|Expired overload-result parse generation/ + ); + } + } + } finally { + automatic.dispose(); + reference.dispose(); + ordinary.dispose(); + } + } +); + +test.each(['getType', 'getTypeResult'] as const)( + 'AutomaticOverloadResult external call speculation leaves activation pending for cached %s', + (api) => { + const source = + fixture.slice(0, fixture.indexOf('def project')) + + `def ordinary(value: int): + cached = value + return cached +def positive(value: list[Any]): + ret = choose(value) + ret.append("ok") +`; + const test = create(source, { automatic: true, checkerHandoff: false }); + const reference = create(source, { automatic: true, checkerHandoff: false }); + try { + const evaluator = test.program.evaluator!; + const controller = test.program.experimentalOverloadResultController!; + const cached = test.name('cached', 'ordinary', 1); + assert.equal(evaluator.printType(evaluator.getType(cached)!), 'int'); + const call = test.nodes.find( + (n): n is CallNode => + n.nodeType === ParseNodeType.Call && + n.d.leftExpr.nodeType === ParseNodeType.Name && + n.d.leftExpr.d.value === 'choose' + )!; + evaluator.useSpeculativeMode(call, () => evaluator.getTypeOfExpression(call)); + assert.equal(controller.hasPendingActivation, true); + assert.equal(controller.roots.size, 0); + assert.equal(controller.getAutomaticStats().modules, 0); + evaluator[api](cached); + assert.equal(controller.hasPendingActivation, false); + assert.ok(isOverloadResult(evaluator.getType(test.name('ret'))!)); + assert.deepStrictEqual(test.analyze(), reference.analyze()); + assert.deepStrictEqual( + resultSummary(evaluator, evaluator.getTypeResult(cached)!), + resultSummary( + reference.program.evaluator!, + reference.program.evaluator!.getTypeResult(reference.name('cached', 'ordinary', 1))! + ) + ); + assert.deepStrictEqual(test.info.getFileInfo(test.module).diagnosticSink.fetchAndClear(), []); + } finally { + test.dispose(); + reference.dispose(); + } + } +); + +test.each([false, true])('AutomaticOverloadResult concrete producers stay ordinary queryFirst=%s', (queryFirst) => { + const source = + fixture.slice(0, fixture.indexOf('def project')) + + `def ordinary(value: list[int]): + ret = choose(value) + reveal_type(ret) + ret.append("invalid") +`; + const ordinary = create(source, false); + const automatic = create(source, { automatic: true, checkerHandoff: false }); + const controller = automatic.program.experimentalOverloadResultController!; + const route = controller.route; + const project = controller.project; + let routes = 0; + let projections = 0; + controller.route = (request) => { + routes++; + return route(request); + }; + controller.project = (node, result) => { + projections++; + return project(node, result); + }; + try { + if (!queryFirst) { + ordinary.analyze(); + automatic.analyze(); + } + for (const occurrence of [2, 1, 0]) { + assert.deepStrictEqual( + resultSummary( + automatic.program.evaluator!, + automatic.program.evaluator!.getTypeResult(automatic.name('ret', 'ordinary', occurrence))! + ), + resultSummary( + ordinary.program.evaluator!, + ordinary.program.evaluator!.getTypeResult(ordinary.name('ret', 'ordinary', occurrence))! + ) + ); + } + assert.deepStrictEqual(automatic.analyze(), ordinary.analyze()); + assert.equal(automatic.program.experimentalOverloadResultController!.getStats(automatic.module), undefined); + const stats = automatic.program.experimentalOverloadResultController!.getAutomaticStats(); + assert.equal(stats.modules, 0); + assert.equal(stats.activationProbes, 0); + assert.equal(stats.selections, 0); + assert.equal(routes, 0); + assert.equal(projections, 0); + assert.ok(!automatic.events.some((e) => e.kind === 'trial')); + assert.deepStrictEqual(automatic.info.getFileInfo(automatic.module).diagnosticSink.fetchAndClear(), []); + const evaluator = automatic.program.evaluator!; + automatic.dispose(); + assert.throws( + () => evaluator.getTypeResult(automatic.name('ret', 'ordinary')), + /Retired overload-result controller/ + ); + } finally { + ordinary.dispose(); + automatic.dispose(); + } +}); + +test('OverloadResultController ordinary query reuse requires the owned cache entries and request', () => { + const source = + fixture.slice(0, fixture.indexOf('def project')) + + `def ordinary(value: list[int]): + ret = choose(value) + reveal_type(ret) +`; + // A fixed empty candidate set retains a declined owner. Automatic concrete + // producers no longer acquire one, but owned restoration still needs this guard. + const test = create(source, { + seeds(module) { + const nodes = new Nodes(); + nodes.walk(module); + return nodes.nodes + .filter( + (node): node is CallNode => + node.nodeType === ParseNodeType.Call && + node.d.leftExpr.nodeType === ParseNodeType.Name && + node.d.leftExpr.d.value === 'choose' + ) + .map((node) => ({ node, uncertainty: TypeCategory.Any, candidates: () => [] })); + }, + }); + try { + const original = test.analyze(); + const evaluator = test.program.evaluator!; + const controller = test.program.experimentalOverloadResultController!; + const node = test.name('ret', 'ordinary', 1); + const expected = evaluator.getTypeResult(node)!; + const requests = () => controller.getStats(test.module)!.ordinaryRequests; + const before = requests(); + assert.throws( + () => evaluator.runWithCancellationToken(CancellationToken.Cancelled, () => evaluator.getTypeResult(node)), + (error) => OperationCanceledException.is(error) + ); + assert.deepStrictEqual( + resultSummary(evaluator, evaluator.getTypeResult(node)!), + resultSummary(evaluator, expected) + ); + assert.equal(requests(), before); + + // Equal types do not establish that an entry was restored by this owner. + for (const replacement of [expected, { ...expected, isIncomplete: true }]) { + evaluator.setTypeResultForNode(node, replacement); + const replaced = requests(); + assert.deepStrictEqual( + resultSummary(evaluator, evaluator.getTypeResult(node)!), + resultSummary(evaluator, expected) + ); + assert.ok(requests() > replaced); + } + + const expression = requests(); + evaluator.getTypeOfExpression(node); + assert.ok(requests() > expression); + const contextual = controller.getStats(test.module)!.contextualRequests; + evaluator.getTypeOfExpression(node, EvalFlags.TypeExpression, { expectedType: AnyType.create() }); + assert.equal(controller.getStats(test.module)!.contextualRequests, contextual + 1); + assert.deepStrictEqual( + resultSummary(evaluator, evaluator.getTypeResult(node)!), + resultSummary(evaluator, expected) + ); + assert.deepStrictEqual(test.info.getFileInfo(test.module).diagnosticSink.fetchAndClear(), []); + assert.deepStrictEqual(test.analyze(), original); + } finally { + test.dispose(); + } +}); + +test.each([false, true])('AutomaticOverloadResult implicit reader type roles queryFirst=%s', (queryFirst) => { + const header = fixture.slice(0, fixture.indexOf('def project')); + const source = + header.replace(', reveal_type', '') + + ` +def positive(value: list[Any]): + ret = choose(value) + reveal_type(ret, expected_type=list[int]) + reveal_type(1, expected_type=ret) + ret.append("x") +`; + const ordinary = create(source, false); + const automatic = create(source, { automatic: true }); + try { + if (queryFirst) { + automatic.program.evaluator!.getType(automatic.name('ret')); + } + const expected = ordinary.analyze(); + const actual = automatic.analyze(); + const invalid = expected.filter((d) => d.rule === 'reportInvalidTypeForm'); + assert.equal(invalid.length, 1); + assert.deepStrictEqual( + actual.filter((d) => d.rule === 'reportInvalidTypeForm'), + invalid + ); + assert.ok(actual.some((d) => d.message.includes('OverloadResult[list[int], list[str]]'))); + assert.ok(!actual.some((d) => d.rule === 'reportArgumentType')); + } finally { + ordinary.dispose(); + automatic.dispose(); + } +}); + +test.each([ + 'constructor12.py', + 'constructor34.py', + 'protocol54.py', + 'typeVarDefaultClass5.py', + 'typeVarDefaultClass6.py', + 'overloadCall6.py', + 'overloadCall12.py', +])('AutomaticOverloadResult preserves complete ordinary source diagnostics for %s', (sample) => { + const source = readSampleFile(sample); + const ordinary = create(source, false); + const automatic = create(source, { automatic: true }); + try { + assert.deepStrictEqual(automatic.analyze(), ordinary.analyze()); + assert.equal(automatic.program.experimentalOverloadResultController!.getAutomaticStats().selected, 0); + if (sample === 'constructor34.py') { + assert.equal(automatic.analyze().filter((d) => d.category === DiagnosticCategory.Error).length, 6); + } + } finally { + ordinary.dispose(); + automatic.dispose(); + } +}); + +test('AutomaticOverloadResult proof cutoff and zero consumer discovery preserve ordinary contracts', () => { + const source = readSampleFile('overloadResultAutomatic1.py'); + const ordinary = create(source, false); + const cutoff = create(source, { automatic: true, limits: { proofUnits: 1 } }); + const zero = create(source, { automatic: true, consumerCalls: 0 }); + try { + assert.deepStrictEqual(cutoff.analyze(), ordinary.analyze()); + assert.equal(cutoff.program.experimentalOverloadResultController!.getAutomaticStats().activationProbes, 0); + assert.equal(cutoff.program.experimentalOverloadResultController!.getStats(cutoff.module), undefined); + const expectedTypeName = (test: ReturnType) => + test.nodes.find( + (node): node is NameNode => + node.nodeType === ParseNodeType.Name && + node.start === source.indexOf('assert_type(1, ret)') + 'assert_type(1, '.length + )!; + const ordinaryResult = ordinary.program.evaluator!.getTypeOfExpression(expectedTypeName(ordinary)); + const cutoffResult = cutoff.program.evaluator!.getTypeOfExpression(expectedTypeName(cutoff)); + assert.equal(ordinary.program.evaluator!.printType(ordinaryResult.type), 'Unknown'); + assert.deepStrictEqual( + resultSummary(cutoff.program.evaluator!, cutoffResult), + resultSummary(ordinary.program.evaluator!, ordinaryResult) + ); + for (let i = 0; i < 3; i++) { + cutoff.program.experimentalOverloadResultController!.evict(cutoff.module); + cutoff.program.evaluator!.evaluateTypesForStatement(cutoff.name('ret', 'original').parent!); + assert.deepStrictEqual( + resultSummary( + cutoff.program.evaluator!, + cutoff.program.evaluator!.getTypeOfExpression(expectedTypeName(cutoff)) + ), + resultSummary(ordinary.program.evaluator!, ordinaryResult) + ); + assert.equal( + cutoff.program.evaluator!.printType(cutoff.program.evaluator!.getType(expectedTypeName(cutoff))!), + 'Unknown' + ); + assert.deepStrictEqual(cutoff.info.getFileInfo(cutoff.module).diagnosticSink.fetchAndClear(), []); + } + zero.analyze(); + const evaluator = zero.program.evaluator!; + const ret = evaluator.getType(zero.name('ret', 'original')); + assert.ok(ret && isOverloadResult(ret)); + assert.strictEqual(evaluator.getType(zero.name('alias', 'original')), ret); + } finally { + ordinary.dispose(); + cutoff.dispose(); + zero.dispose(); + } +}); + +test('AutomaticOverloadResult reversed overloads retain generic projection and both scalar member witnesses', () => { + const source = fixture + .replace(/\r\n/g, '\n') + .replace( + 'def choose(value: list[int]) -> list[int]: ...\n@overload\ndef choose(value: list[str]) -> list[str]: ...', + 'def choose(value: list[str]) -> list[str]: ...\n@overload\ndef choose(value: list[int]) -> list[int]: ...' + ); + const fixed = create(source); + const automatic = create(source, { automatic: true }); + try { + assert.deepStrictEqual(automatic.analyze(), fixed.analyze()); + const evaluator = automatic.program.evaluator!; + for (const [name, type] of [ + ['ret', 'OverloadResult[list[str], list[int]]'], + ['projected', 'OverloadResult[Sequence[str], Sequence[int]]'], + ['scalar', 'OverloadResult[str, int]'], + ]) { + assert.equal(evaluator.printType(evaluator.getType(automatic.name(name))!), type); + } + } finally { + fixed.dispose(); + automatic.dispose(); + } +}); + +test('OverloadResultController installed without seeds preserves complete ordinary diagnostics and types', () => { + const ordinary = create(fixture, false); + const empty = create(fixture, { seeds: () => [] }); + try { + assert.deepStrictEqual(empty.analyze(), ordinary.analyze()); + for (const fn of ['positive', 'concrete', 'gradual', 'unknown', 'failed']) { + const a = ordinary.program.evaluator!.getTypeOfExpression(ordinary.name('ret', fn)); + const b = empty.program.evaluator!.getTypeOfExpression(empty.name('ret', fn)); + assert.deepStrictEqual( + resultSummary(empty.program.evaluator!, b), + resultSummary(ordinary.program.evaluator!, a) + ); + } + assert.equal(empty.events.length, 0); + assert.equal(empty.program.experimentalOverloadResultController!.getStats(empty.module), undefined); + } finally { + ordinary.dispose(); + empty.dispose(); + } +}); + +test('OverloadResultController canonical queries, aliases and actual assignment reentry', () => { + const test = create(); + try { + const diagnostics = test.analyze(); + const evaluator = test.program.evaluator!; + const alias = test.name('alias'); + const canonical = evaluator.getTypeOfExpression(alias).type; + assert.ok(isOverloadResult(canonical)); + assert.strictEqual(evaluator.getType(test.name('ret')), canonical); + assert.strictEqual(evaluator.getType(alias), canonical); + assert.strictEqual(evaluator.getTypeResult(alias)!.type, canonical); + assert.equal(evaluator.printType(evaluator.getType(test.name('string'))!), 'list[str]'); + assert.ok(isOverloadResult(evaluator.getType(test.name('projected'))!)); + assert.ok(isOverloadResult(evaluator.getType(test.name('scalar'))!)); + const offset = fixture.slice(0, alias.start); + const hover = new HoverProvider( + test.program, + test.uri, + { + line: offset.split('\n').length - 1, + character: offset.length - offset.lastIndexOf('\n') - 1, + }, + 'markdown', + CancellationToken.None + ).getHover(); + assert.ok(JSON.stringify(hover).includes('OverloadResult')); + const controller = test.program.experimentalOverloadResultController!; + const target = test.name('string'); + assert.ok(target.parent?.nodeType === ParseNodeType.TypeAnnotation); + const assignment = target.parent.parent; + assert.ok(assignment?.nodeType === ParseNodeType.Assignment); + const before = controller.getStats(test.module)!; + for (let round = 0; round < 3; round++) { + controller.evict(test.module); + assert.equal(evaluator.getCachedType(target), undefined); + evaluator.evaluateTypesForStatement(alias.parent!); + evaluator.evaluateTypesForStatement(assignment); + assert.equal(evaluator.printType(evaluator.getCachedType(target)!), 'list[str]'); + assert.ok(!isOverloadResult(evaluator.getCachedType(alias)!)); + assert.strictEqual(evaluator.getType(alias), canonical); + } + assert.deepStrictEqual(test.info.getFileInfo(test.module).diagnosticSink.fetchAndClear(), []); + const after = controller.getStats(test.module)!; + assert.equal(before.candidateCalls, after.candidateCalls); + assert.equal(before.outcomes, after.outcomes); + assert.ok(after.ordinaryRequests > before.ordinaryRequests); + assert.deepStrictEqual(test.analyze(), diagnostics); + assert.ok(diagnostics.some((d) => d.rule === 'reportAssignmentType')); + assert.ok(diagnostics.some((d) => d.rule === 'reportAttributeAccessIssue')); + } finally { + test.dispose(); + } +}); + +test('OverloadResultController declines unsupported uses before early queries in both orders', () => { + const normal = create(fixture, false); + const forward = create(); + const reverse = create(); + try { + const unsupported = ['repeated', 'escaped', 'captured', 'joined', 'returned', 'attribute', 'distinct']; + for (const [test, names] of [ + [forward, unsupported], + [reverse, [...unsupported].reverse()], + ] as const) { + for (const fn of names) { + const node = test.name('ret', fn, 1); + const result = test.program.evaluator!.getTypeOfExpression(node); + const baseline = normal.program.evaluator!.getTypeOfExpression(normal.name('ret', fn, 1)); + assert.ok(!isOverloadResult(result.type)); + assert.deepStrictEqual( + resultSummary(test.program.evaluator!, result), + resultSummary(normal.program.evaluator!, baseline) + ); + } + assert.ok(isOverloadResult(test.program.evaluator!.getType(test.name('independent', 'repeated'))!)); + } + assert.deepStrictEqual(forward.analyze(), reverse.analyze()); + assert.equal(forward.events.filter((e) => e.kind === 'admission').length, 1); + const base = normal.analyze(); + const enabled = forward.analyze(); + for (const d of enabled.filter((d) => d.range.start.line >= 46)) { + assert.ok(base.some((b) => JSON.stringify(b) === JSON.stringify(d))); + } + } finally { + normal.dispose(); + forward.dispose(); + reverse.dispose(); + } +}); + +test.each([false, true])('OverloadResultController genuine expected contexts automatic=%s', (automatic) => { + const test = create(fixture, { automatic }); + try { + const initial = test.analyze(); + const evaluator = test.program.evaluator!; + const alias = test.name('alias'); + const canonical = evaluator.getType(alias)!; + const expected = evaluator.getType(test.name('string'))!; + const call = test.nodes.find( + (n): n is CallNode => + n.nodeType === ParseNodeType.Call && + n.d.leftExpr.nodeType === ParseNodeType.Name && + n.d.leftExpr.d.value === 'project' && + getEnclosingFunction(n)?.d.name.d.value === 'positive' + )!; + const stableCall = evaluator.getTypeOfExpression(call); + const controller = test.program.experimentalOverloadResultController!; + const before = controller.getStats(test.module)!; + for (let round = 0; round < 8; round++) { + for (const flags of [EvalFlags.None, EvalFlags.NoSpecialize]) { + assert.strictEqual( + evaluator.getTypeOfExpression(alias, flags, { expectedType: expected }).type, + canonical + ); + const result = evaluator.getTypeOfExpression(call, flags, { + expectedType: expected, + isTypeIncomplete: true, + }); + assert.ok(!isOverloadResult(result.type)); + assert.strictEqual(evaluator.getTypeOfExpression(call).type, stableCall.type); + } + } + const after = controller.getStats(test.module)!; + assert.equal(after.outcomes, before.outcomes); + assert.equal(after.slots, before.slots); + assert.equal(after.candidateCalls, before.candidateCalls); + assert.equal(after.contextualRequests - before.contextualRequests, 16); + assert.deepStrictEqual(test.info.getFileInfo(test.module).diagnosticSink.fetchAndClear(), []); + assert.deepStrictEqual(test.analyze(), initial); + const contextFirst = create(fixture, { automatic }); + try { + const coldCall = contextFirst.nodes.find( + (n): n is CallNode => n.nodeType === ParseNodeType.Call && n.start === call.start + )!; + const cold = contextFirst.program.evaluator!; + const contextual = cold.getTypeOfExpression(coldCall, EvalFlags.NoSpecialize, { + expectedType: AnyType.create(), + isTypeIncomplete: true, + }); + assert.ok(!isOverloadResult(contextual.type)); + assert.deepStrictEqual( + contextFirst.info.getFileInfo(contextFirst.module).diagnosticSink.fetchAndClear(), + [] + ); + assert.deepStrictEqual(contextFirst.analyze(), initial); + assert.equal(cold.printType(cold.getTypeOfExpression(coldCall).type), evaluator.printType(stableCall.type)); + } finally { + contextFirst.dispose(); + } + } finally { + test.dispose(); + } +}); + +test('OverloadResultController zero limits preserve ordinary errors and zero consumer budget preserves aliases', () => { + const normal = create(fixture, false); + const zero = create(fixture, { limits: { candidates: 0 } }); + const alias = create(fixture, { consumerCalls: 0 }); + try { + assert.deepStrictEqual(zero.analyze(), normal.analyze()); + alias.analyze(); + const ev = alias.program.evaluator!; + assert.ok(isOverloadResult(ev.getType(alias.name('ret'))!)); + assert.strictEqual(ev.getType(alias.name('ret')), ev.getType(alias.name('alias'))); + const cutoffs = zero.events.filter((e) => e.kind === 'value' && e.cutoff); + assert.ok(cutoffs.length); + for (const event of cutoffs) { + if (event.kind === 'value') { + assert.strictEqual(event.canonical, event.baseline); + } + } + } finally { + normal.dispose(); + zero.dispose(); + alias.dispose(); + } +}); + +test('OverloadResultController unsupported flags and root alternative contexts use complete ordinary operations', () => { + const enabled = create(); + try { + const initial = enabled.analyze(); + const evaluator = enabled.program.evaluator!; + const canonical = evaluator.getType(enabled.name('alias'))!; + assert.ok(isOverloadResult(canonical)); + const findCall = (nodes: ParseNode[]) => + nodes.find( + (n): n is CallNode => + n.nodeType === ParseNodeType.Call && + n.d.leftExpr.nodeType === ParseNodeType.Name && + n.d.leftExpr.d.value === 'project' && + getEnclosingFunction(n)?.d.name.d.value === 'positive' + )!; + const call = findCall(enabled.nodes); + const controller = enabled.program.experimentalOverloadResultController!; + const before = controller.getStats(enabled.module)!; + for (const flags of [EvalFlags.TypeExpression, EvalFlags.TypeFormArg, EvalFlags.ForwardRefs]) { + for (const node of [enabled.name('alias'), call]) { + controller.evict(enabled.module); + const result = evaluator.getTypeOfExpression(node, flags, { + expectedType: canonical, + returnTypeOverride: canonical, + isTypeIncomplete: true, + }); + assert.ok(!isOverloadResult(result.type)); + // An owned contextual operation has fresh selected caches. Compare + // with a cold ordinary request, not a cached context-free answer. + const ordinary = create(fixture, false); + try { + const baselineEvaluator = ordinary.program.evaluator!; + const baseline = baselineEvaluator.getTypeOfExpression( + node === call ? findCall(ordinary.nodes) : ordinary.name('alias'), + flags, + { + expectedType: baselineEvaluator.getType(ordinary.name('ret'))!, + returnTypeOverride: baselineEvaluator.getType(ordinary.name('ret'))!, + isTypeIncomplete: true, + } + ); + assert.deepStrictEqual( + resultSummary(evaluator, result), + resultSummary(baselineEvaluator, baseline) + ); + const trials = enabled.events.filter((e) => e.kind === 'trial' && e.mode === 'context'); + const trial = trials[trials.length - 1]; + assert.ok(trial.kind === 'trial'); + const diagnostic = (d: Diagnostic) => ({ + category: d.category, + message: d.message, + range: d.range, + rule: d.getRule(), + }); + assert.deepStrictEqual( + trial.diagnostics.map(diagnostic), + ordinary.info.getFileInfo(ordinary.module).diagnosticSink.fetchAndClear().map(diagnostic) + ); + } finally { + ordinary.dispose(); + } + assert.strictEqual(evaluator.getType(enabled.name('alias')), canonical); + } + } + const after = controller.getStats(enabled.module)!; + assert.equal(after.outcomes, before.outcomes); + assert.equal(after.candidateCalls, before.candidateCalls); + assert.equal(after.contextualRequests - before.contextualRequests, 6); + assert.deepStrictEqual(enabled.info.getFileInfo(enabled.module).diagnosticSink.fetchAndClear(), []); + assert.deepStrictEqual(enabled.analyze(), initial); + } finally { + enabled.dispose(); + } +}); + +test('OverloadResultController cutoff after successful member lookup preserves the complete ordinary operation', () => { + const content = fixture + .replace('scalar.upper()', 'unwrap(project(ret)).upper()') + .replace('scalar.bit_length()', 'unwrap(project(ret)).bit_length()'); + const test = create(content, { consumerCalls: 3 }); + try { + test.analyze(); + const calls = test.events.filter((e) => e.kind === 'beforeCall' && e.mode === 'candidate'); + const resolved = calls.find( + (e) => + e.kind === 'beforeCall' && + e.node.d.leftExpr.nodeType === ParseNodeType.MemberAccess && + e.node.d.leftExpr.d.member.d.value === 'bit_length' && + isFunction(e.seen.get(e.node.d.leftExpr)!.type) + ); + assert.ok(resolved?.kind === 'beforeCall'); + assert.ok( + test.events.some( + (e) => + e.kind === 'trial' && + e.exception && + e.mode === 'candidate' && + e.root.start <= resolved.node.start && + e.root.start + e.root.length >= resolved.node.start + resolved.node.length + ) + ); + const cutoffs = test.events.filter((e) => e.kind === 'value' && e.cutoff); + assert.ok(cutoffs.length); + for (const e of cutoffs) { + assert.ok(e.kind === 'value'); + assert.strictEqual(e.canonical, e.baseline); + } + assert.deepStrictEqual(test.info.getFileInfo(test.module).diagnosticSink.fetchAndClear(), []); + } finally { + test.dispose(); + } +}); + +test.each(['error', 'cancel'] as const)( + 'AutomaticOverloadResult interrupted activation probe restores state: %s', + (kind) => { + let fault = true; + const reference = create(fixture, { automatic: true, checkerHandoff: false }); + const test = create(fixture, { + automatic: true, + checkerHandoff: false, + observe(event) { + if (fault && event.kind === 'activationProbe' && event.selection.reason === 'selected') { + fault = false; + if (kind === 'cancel') { + throw new OperationCanceledException(); + } + throw new Error('activation interruption'); + } + }, + }); + try { + const evaluator = test.program.evaluator!; + const controller = test.program.experimentalOverloadResultController!; + assert.throws( + () => evaluator.getType(test.name('ret')), + (error) => + kind === 'cancel' + ? OperationCanceledException.is(error) + : error instanceof Error && error.message === 'activation interruption' + ); + assert.equal(controller.hasPendingActivation, false); + assert.equal(controller.getAutomaticStats().modules, 0); + assert.equal(controller.roots.size, 0); + assert.equal(evaluator.isSpeculativeModeInUse(undefined), false); + assert.deepStrictEqual(test.info.getFileInfo(test.module).diagnosticSink.fetchAndClear(), []); + assert.ok(isOverloadResult(evaluator.getType(test.name('ret'))!)); + assert.deepStrictEqual(test.analyze(), reference.analyze()); + } finally { + reference.dispose(); + test.dispose(); + } + } +); + +test.each([false, true])('OverloadResultController cancelled discovery restores state automatic=%s', (automatic) => { + let fault = true; + const test = create(fixture, { + automatic, + observe(event) { + if (fault && event.kind === 'beforeCall' && event.mode === 'candidate') { + fault = false; + throw new OperationCanceledException(); + } + }, + }); + + try { + const evaluator = test.program.evaluator!; + assert.throws( + () => evaluator.getType(test.name('ret')), + (error) => OperationCanceledException.is(error) + ); + assert.equal(evaluator.getCachedType(test.name('ret')), undefined); + assert.deepStrictEqual(test.info.getFileInfo(test.module).diagnosticSink.fetchAndClear(), []); + assert.ok(isOverloadResult(evaluator.getType(test.name('ret'))!)); + assert.ok(test.program.experimentalOverloadResultController!.getStats(test.module)!.retryReservations > 0); + test.analyze(); + const controller = test.program.experimentalOverloadResultController!; + test.dispose(); + assert.equal(controller.getStats(test.module), undefined); + assert.throws(() => evaluator.getType(test.name('ret')), /Retired overload-result controller/); + } finally { + test.dispose(); + } +}); + +test.each([ + ' ret += [1]', + ' while condition:\n ret = [b"changed"]\n ret.append("late")', + ' alias = ret\n print(ret, alias)', + ' second = choose(ret)\n second.append("late")', + ' storage.append(ret)', + ' box.field = ret', +])('OverloadResultController unsupported local path declines before publication: %s', (suffix) => { + const source = + fixture + + '\n\ndef tail(value: list[Any], condition: bool, storage: list[list[int]], box: Box):\n' + + ' ret = choose(value)\n ret.append("early")\n' + + suffix + + '\n'; + const ordinary = create(source, false); + const enabled = create(source); + try { + const result = enabled.program.evaluator!.getTypeOfExpression(enabled.name('ret', 'tail', 1)); + assert.ok(!isOverloadResult(result.type)); + const startLine = source.slice(0, source.indexOf('def tail')).split('\n').length - 1; + const baseline = ordinary.analyze().filter((d) => d.range.start.line >= startLine); + const actual = enabled.analyze().filter((d) => d.range.start.line >= startLine); + assert.deepStrictEqual(actual, baseline); + assert.ok(isOverloadResult(enabled.program.evaluator!.getType(enabled.name('independent', 'repeated'))!)); + } finally { + ordinary.dispose(); + enabled.dispose(); + } +}); + +test('OverloadResultController materialization faults restore selected caches and live diagnostic ownership', () => { + let fault: 'error' | 'cancel' | undefined; + const test = create(fixture, { + observe(event) { + if (fault && event.kind === 'project' && event.mode === 'materialize-witness') { + const current = fault; + fault = undefined; + if (current === 'cancel') { + throw new OperationCanceledException(); + } + throw new Error('materialization interruption'); + } + }, + }); + try { + const original = test.analyze(); + const evaluator = test.program.evaluator!; + const target = test.name('string'); + assert.ok(target.parent?.nodeType === ParseNodeType.TypeAnnotation); + const statement = target.parent.parent; + assert.ok(statement?.nodeType === ParseNodeType.Assignment); + const canonical = evaluator.getType(test.name('alias')); + const sink = test.info.getFileInfo(test.module).diagnosticSink; + for (const kind of ['error', 'cancel'] as const) { + test.program.experimentalOverloadResultController!.evict(test.module); + fault = kind; + assert.throws( + () => evaluator.evaluateTypesForStatement(statement), + (error) => + kind === 'cancel' + ? OperationCanceledException.is(error) + : error instanceof Error && error.message === 'materialization interruption' + ); + assert.strictEqual(test.info.getFileInfo(test.module).diagnosticSink, sink); + assert.equal(evaluator.getCachedType(target), undefined); + assert.deepStrictEqual(sink.fetchAndClear(), []); + assert.ok(!evaluator.isSpeculativeModeInUse(undefined)); + evaluator.evaluateTypesForStatement(statement); + assert.equal(evaluator.printType(evaluator.getCachedType(target)!), 'list[str]'); + assert.strictEqual(evaluator.getType(test.name('alias')), canonical); + } + assert.deepStrictEqual(test.analyze(), original); + } finally { + test.dispose(); + } +}); + +test.each([false, true])('OverloadResultController discovery limits automatic=%s', (automatic) => { + const exact = create(fixture, { automatic, limits: { candidates: 2 } }); + const over = create(fixture, { automatic, limits: { candidates: 1 } }); + const zeroGraph = create(fixture, { automatic, limits: { graphUnits: 0 } }); + const ordinary = create(fixture, false); + try { + assert.ok(isOverloadResult(exact.program.evaluator!.getType(exact.name('ret'))!)); + const baseline = ordinary.analyze(); + assert.deepStrictEqual(over.analyze(), baseline); + assert.deepStrictEqual(zeroGraph.analyze(), baseline); + assert.ok(!isOverloadResult(over.program.evaluator!.getType(over.name('ret'))!)); + assert.equal( + zeroGraph.program.experimentalOverloadResultController!.getStats(zeroGraph.module)!.candidateCalls, + 0 + ); + } finally { + exact.dispose(); + over.dispose(); + zeroGraph.dispose(); + ordinary.dispose(); + } +}); + +test.each([false, true])('OverloadResultController source replacement automatic=%s', (automatic) => { + const test = create(fixture, { automatic }); + try { + const diagnostics = test.analyze(); + const old = test.program.evaluator!; + const alias = test.name('alias'); + const value = old.getType(alias); + test.program.setFileOpened(test.uri, 2, fixture.replace('ret = choose(value)', 'ret = [b"edited"]')); + test.analyze(); + assert.ok(test.events.some((event) => event.kind === 'retire')); + test.program.setFileOpened(test.uri, 3, fixture); + assert.deepStrictEqual(test.analyze(), diagnostics); + const nodes = new Nodes(); + nodes.walk(test.program.getParseResults(test.uri)!.parserOutput.parseTree); + const restored = nodes.nodes.find( + (n): n is NameNode => + n.nodeType === ParseNodeType.Name && + n.d.value === 'alias' && + getEnclosingFunction(n)?.d.name.d.value === 'positive' + )!; + const current = test.program.evaluator!.getType(restored); + assert.ok(current && isOverloadResult(current)); + assert.notStrictEqual(current, value); + assert.throws( + () => old.getType(alias), + /Retired overload-result controller|Expired overload-result parse generation/ + ); + } finally { + test.dispose(); + } +}); + +const readerFixture = readSampleFile('overloadResultReader1.py'); +const readerOrders = ['check-first', 'value-first', 'reader-first'] as const; + +test.each(readerOrders.flatMap((order) => [false, true].map((automatic) => ({ order, automatic }))))( + 'OverloadResultController reader type argument preserves ordinary errors: %j', + ({ order, automatic }) => { + const ordinary = create(readerFixture, false); + const enabled = create(readerFixture, { automatic }); + const expected = [ + { + category: DiagnosticCategory.Error, + rule: 'reportInvalidTypeForm', + message: 'Variable not allowed in type expression', + range: { start: { line: 13, character: 19 }, end: { line: 13, character: 22 } }, + }, + { + category: DiagnosticCategory.Error, + rule: 'reportAssertTypeFailure', + message: '"assert_type" mismatch: expected "Unknown" but received "Literal[1]"', + range: { start: { line: 13, character: 16 }, end: { line: 13, character: 17 } }, + }, + ]; + const reader = (test: ReturnType) => + test.nodes.find( + (n): n is CallNode => + n.nodeType === ParseNodeType.Call && n.start === readerFixture.indexOf('assert_type(1, ret)') + )!; + try { + for (const test of [ordinary, enabled]) { + if (order === 'value-first') { + test.program.evaluator!.getType(test.name('ret', 'invalid_expected_type')); + } else if (order === 'reader-first') { + test.program.evaluator!.getTypeOfExpression(reader(test)); + } + } + assert.deepStrictEqual(ordinary.analyze(), expected); + assert.deepStrictEqual(enabled.analyze(), expected); + const evaluator = enabled.program.evaluator!; + const value = evaluator.getType(enabled.name('ret', 'invalid_expected_type')); + assert.ok(value && isOverloadResult(value)); + assert.deepStrictEqual( + resultSummary(evaluator, evaluator.getTypeOfExpression(reader(enabled))), + resultSummary( + ordinary.program.evaluator!, + ordinary.program.evaluator!.getTypeOfExpression(reader(ordinary)) + ) + ); + assert.ok( + enabled.events.some( + (e) => + e.kind === 'trial' && + e.mode === 'reader' && + e.diagnostics.some((d) => d.getRule() === 'reportInvalidTypeForm') + ) + ); + enabled.program.experimentalOverloadResultController!.evict(enabled.module); + evaluator.evaluateTypesForStatement(reader(enabled)); + assert.strictEqual(evaluator.getType(enabled.name('ret', 'invalid_expected_type')), value); + assert.deepStrictEqual(enabled.info.getFileInfo(enabled.module).diagnosticSink.fetchAndClear(), []); + assert.deepStrictEqual(enabled.analyze(), expected); + } finally { + ordinary.dispose(); + enabled.dispose(); + } + } +); + +test.each([ + { statement: 'check_type(1, ret)', declined: false }, + { statement: 'show_type(1, expected_type=ret)', declined: false }, + { statement: 'alias = ret\n check_type(1, alias)', declined: false }, + { statement: 'check_type(1, "ret")', declined: false }, + { statement: 'check_type(1, ret.__class__)', declined: true }, + { statement: 'check_type(ret.copy(), list[int])', declined: true }, + { statement: 'show_type(ret.copy())', declined: true }, + { statement: 'check_type(1, list[ret])', declined: true }, + { statement: 'check_type(ret, ret)', declined: true }, + { statement: 'reader = check_type\n reader(1, ret)', declined: true }, + { statement: 'reader = check_type\n reader(ret, list[int])', declined: true }, + { statement: 'reader = show_type\n reader(ret)', declined: true }, + { statement: 'print(check_type(ret, list[int]))', declined: true }, + { statement: 'check_type(ret, list[int]).append(1)', declined: true }, +])('OverloadResultController reader roles and aliases: $statement', ({ statement, declined }) => { + const content = readerFixture + .replace( + 'from typing import Any, assert_type, overload', + 'from typing import Any, assert_type as check_type, reveal_type as show_type, overload' + ) + .replace('assert_type(1, ret)', statement); + for (const order of readerOrders) { + const ordinary = create(content, false); + const enabled = create(content); + const root = (test: ReturnType) => + test.nodes.find( + (n): n is CallNode => + n.nodeType === ParseNodeType.Call && + n.start === content.lastIndexOf(statement.split('\n').pop()!.trim()) + )!; + try { + if (order === 'reader-first') { + ordinary.program.evaluator!.getTypeOfExpression(root(ordinary)); + enabled.program.evaluator!.getTypeOfExpression(root(enabled)); + } else if (order === 'value-first') { + ordinary.program.evaluator!.getType(ordinary.name('ret', 'invalid_expected_type')); + const early = enabled.program.evaluator!.getType(enabled.name('ret', 'invalid_expected_type')); + assert.ok(early); + assert.equal(isOverloadResult(early), !declined); + } + assert.deepStrictEqual(enabled.analyze(), ordinary.analyze()); + assert.deepStrictEqual( + resultSummary( + enabled.program.evaluator!, + enabled.program.evaluator!.getTypeOfExpression(root(enabled)) + ), + resultSummary( + ordinary.program.evaluator!, + ordinary.program.evaluator!.getTypeOfExpression(root(ordinary)) + ) + ); + const value = enabled.program.evaluator!.getType(enabled.name('ret', 'invalid_expected_type')); + assert.ok(value); + assert.equal(isOverloadResult(value), !declined); + assert.deepStrictEqual(enabled.info.getFileInfo(enabled.module).diagnosticSink.fetchAndClear(), []); + } finally { + ordinary.dispose(); + enabled.dispose(); + } + } +}); + +test.each([false, true])( + 'OverloadResultController canonical reader value arguments remain exact, aliased=%s', + (aliased) => { + const assertName = aliased ? 'check_type' : 'assert_type'; + const revealName = aliased ? 'show_type' : 'reveal_type'; + const content = readerFixture + .replace( + 'from typing import Any, assert_type, overload', + `from typing import Any, assert_type${aliased ? ' as check_type' : ''}, reveal_type${ + aliased ? ' as show_type' : '' + }, overload` + ) + .replace( + 'assert_type(1, ret)', + `${assertName}(ret, list[int])\n ${assertName}(ret, list[Any])\n ${revealName}(ret)` + ); + for (const order of readerOrders) { + const ordinary = create(content, false); + const enabled = create(content); + try { + if (order === 'value-first') { + enabled.program.evaluator!.getType(enabled.name('ret', 'invalid_expected_type')); + } else if (order === 'reader-first') { + const calls = enabled.nodes.filter( + (n): n is CallNode => + n.nodeType === ParseNodeType.Call && + n.d.leftExpr.nodeType === ParseNodeType.Name && + [assertName, revealName].includes(n.d.leftExpr.d.value) + ); + for (const call of [...calls].reverse()) { + enabled.program.evaluator!.getTypeOfExpression(call); + } + } + const baseline = ordinary.analyze(); + const diagnostics = enabled.analyze(); + const value = enabled.program.evaluator!.getType(enabled.name('ret', 'invalid_expected_type')); + assert.ok(value && isOverloadResult(value)); + assert.equal(diagnostics.length, 3); + assert.equal(baseline.length, 2); + assert.ok(diagnostics.every((d) => d.rule !== 'reportInvalidTypeForm')); + assert.equal( + diagnostics[0].message, + '"assert_type" mismatch: expected "list[int]" but received "OverloadResult[list[int], list[str]]"' + ); + assert.equal( + diagnostics[1].message, + '"assert_type" mismatch: expected "list[Any]" but received "OverloadResult[list[int], list[str]]"' + ); + assert.equal(diagnostics[2].message, 'Type of "ret" is "OverloadResult[list[int], list[str]]"'); + const readers = enabled.nodes.filter( + (n): n is CallNode => + n.nodeType === ParseNodeType.Call && + n.d.leftExpr.nodeType === ParseNodeType.Name && + [assertName, revealName].includes(n.d.leftExpr.d.value) + ); + for (const call of readers) { + assert.strictEqual( + enabled.program.evaluator!.getTypeOfExpression(call, EvalFlags.NoSpecialize).type, + value + ); + } + assert.deepStrictEqual(enabled.info.getFileInfo(enabled.module).diagnosticSink.fetchAndClear(), []); + } finally { + ordinary.dispose(); + enabled.dispose(); + } + } + } +); diff --git a/packages/pyright-internal/src/tests/samples/overloadCall11.py b/packages/pyright-internal/src/tests/samples/overloadCall11.py index b0c9714f4ed3..3adf126546b5 100644 --- a/packages/pyright-internal/src/tests/samples/overloadCall11.py +++ b/packages/pyright-internal/src/tests/samples/overloadCall11.py @@ -1,5 +1,5 @@ -# This sample records the known conformance limitation restored by rolling back -# #11601/#11732, not the required semantics of an ambiguous invariant-container call. +# Ambiguous invariant arguments retain both usable overload results without +# erasing their containers or merging them into an invariant-incompatible union. from typing import Any, overload @@ -20,13 +20,11 @@ def overloaded(value: Any) -> list[Any]: def check(value: list[Any]) -> None: result = overloaded(value) - reveal_type(result, expected_text="list[int]") + reveal_type(result, expected_text="OverloadResult[list[int], list[str]]") # Conformance requires assignability to BOTH retained return types and permits - # operations supported by either. First-match inference currently fails these checks. + # operations supported by either. int_result: list[int] = result - # This should generate an error: the known rollback limitation rejects list[str]. str_result: list[str] = result result.append(1) - # This should generate an error: string append is valid for a retained materialization. result.append("value") diff --git a/packages/pyright-internal/src/tests/samples/overloadResultActivationOrder1.py b/packages/pyright-internal/src/tests/samples/overloadResultActivationOrder1.py new file mode 100644 index 000000000000..9b927bc995ed --- /dev/null +++ b/packages/pyright-internal/src/tests/samples/overloadResultActivationOrder1.py @@ -0,0 +1,84 @@ +from typing import Any, Sequence, TypeVar, overload + +T = TypeVar("T") + +@overload +def choose(value: list[int]) -> list[int]: ... +@overload +def choose(value: list[str]) -> list[str]: ... +def choose(value: Any) -> Any: + return value + +@overload +def choose_other(value: list[bytes]) -> list[bytes]: ... +@overload +def choose_other(value: list[float]) -> list[float]: ... +def choose_other(value: Any) -> Any: + return value + +def project(value: list[T]) -> Sequence[T]: + return value + +@overload +def unwrap(value: Sequence[int]) -> int: ... +@overload +def unwrap(value: Sequence[str]) -> str: ... +def unwrap(value: Any) -> Any: + return value[0] + +@overload +def unwrap_other(value: Sequence[bytes]) -> bytes: ... +@overload +def unwrap_other(value: Sequence[float]) -> float: ... +def unwrap_other(value: Any) -> Any: + return value[0] + +def positive(value: list[Any], other: list[Any], concrete: list[int]): + first = choose(value) + first_alias = first + first_projected = project(first_alias) + first_scalar = unwrap(first_projected) + first_scalar.upper() + first_scalar.bit_length() + first_int: list[int] = first + first_str: list[str] = first + first.append(1) + first.append("ok") + + between: int = "bad" + first_invalid: list[bytes] = first + + second = choose_other(other) + second_alias = second + second_projected = project(second_alias) + second_scalar = unwrap_other(second_projected) + second_scalar.decode() + second_scalar.hex() + second_bytes: list[bytes] = second + second_float: list[float] = second + second.append(b"ok") + second.append(1.0) + + first_scalar.upper() + first_scalar.bit_length() + # No single alternative accepts the whole iterable. + first.extend([1, "x"]) + second.extend([b"x", 1.0]) + first.append(3.14) + second.append("neither") + first.nonexistent() + second.nonexistent() + + bad_producer: list[bytes] = choose(value) + ordinary = choose(concrete) + ordinary.append("bad") + +def independent(value: list[Any]): + remote = choose_other(value) + remote.append(b"ok") + remote.append(1.0) + +def declined(value: list[Any]): + escaped = choose(value) + escaped.append("not_supported") + return escaped diff --git a/packages/pyright-internal/src/tests/samples/overloadResultAutomatic1.py b/packages/pyright-internal/src/tests/samples/overloadResultAutomatic1.py new file mode 100644 index 000000000000..c457f576b976 --- /dev/null +++ b/packages/pyright-internal/src/tests/samples/overloadResultAutomatic1.py @@ -0,0 +1,171 @@ +from typing import Any, Generic, Sequence, TypeVar, assert_type, overload, reveal_type + +T = TypeVar("T") + +@overload +def choose(value: list[int]) -> list[int]: ... +@overload +def choose(value: list[str]) -> list[str]: ... +def choose(value: Any) -> Any: + return value + +def original(value: list[Any]): + ret = choose(value) + alias = ret + integer: list[int] = alias + string: list[str] = alias + ret.append(1) + ret.append("x") + ret.append(3.14) + ret.nonexistent() + invalid: list[bytes] = ret + unrelated: int = "bad" + reveal_type(alias) + assert_type(alias, list[int]) + assert_type(1, ret) + print(integer, string, invalid, unrelated) + +@overload +def covered(value: list[int]) -> list[int]: ... +@overload +def covered(value: list[Any]) -> list[Any]: ... +@overload +def covered(value: list[str]) -> list[str]: ... +def covered(value: Any) -> Any: + return value + +def prefix(value: list[Any]): + ret = covered(value) + ret.append("x") + +@overload +def equivalent(value: list[int]) -> list[int]: ... +@overload +def equivalent(value: list[str]) -> list[int]: ... +def equivalent(value: Any) -> list[int]: + return [] + +def equal_returns(value: list[Any]): + ret = equivalent(value) + ret.append("x") + +@overload +def defaults(value: list[int], second: int = 0) -> list[int]: ... +@overload +def defaults(value: list[str], second: int = 0) -> list[str]: ... +def defaults(value: Any, second: int = 0) -> Any: + return value + +def default_argument(value: list[Any]): + ret = defaults(value) + ret.append("x") + +@overload +def generic(value: list[T]) -> list[T]: ... +@overload +def generic(value: Sequence[T]) -> list[T]: ... +def generic(value: Sequence[T]) -> list[T]: + return list(value) + +def type_variable(value: list[Any]): + ret = generic(value) + ret.nonexistent() + +@overload +def unsupported(value: list[int]) -> list[int]: ... +@overload +def unsupported(value: Sequence[str]) -> list[str]: ... +def unsupported(value: Any) -> Any: + return value + +def unsupported_proof(value: list[Any]): + ret = unsupported(value) + ret.append("x") + +@overload +def arity(value: list[int], second: int) -> list[int]: ... +@overload +def arity(value: list[str]) -> list[str]: ... +def arity(value: Any, second: int = 0) -> Any: + return value + +def one_mapping(value: list[Any]): + ret = arity(value) + ret.append(1) + +@overload +def nested(value: list[list[int]]) -> list[list[int]]: ... +@overload +def nested(value: list[list[str]]) -> list[list[str]]: ... +def nested(value: Any) -> Any: + return value + +def nested_invariant(value: list[list[Any]]): + ret = nested(value) + integer: list[list[int]] = ret + string: list[list[str]] = ret + print(integer, string) + +def concrete(value: list[str]): + ret = choose(value) + ret.append(1) + +def unknown(value: list): + ret = choose(value) + ret.append("x") + +def any_argument(value: Any): + ret = choose(value) + ret.nonexistent() + +def union_argument(value: list[int] | list[str]): + ret = choose(value) + ret.append("x") + +def keywords(value: list[Any]): + ret = choose(value=value) + ret.append("x") + +def unpack(value: tuple[list[Any]]): + ret = choose(*value) + ret.append("x") + +class Receiver: + @overload + def method(self, value: list[int]) -> list[int]: ... + @overload + def method(self, value: list[str]) -> list[str]: ... + def method(self, value: Any) -> Any: + return value + +def receiver(value: list[Any], obj: Receiver): + ret = obj.method(value) + ret.append("x") + other = Receiver.method(obj, value) + other.append("x") + +def early(value: list[Any]): + ret = choose(value) + ret.append("x") + container = [ret] + print(container) + +def independent(value: list[Any]): + ret = choose(value) + ret.append("x") + +class Invariant(Generic[T]): + pass + +@overload +def nominal(value: Invariant[int]) -> Invariant[int]: ... +@overload +def nominal(value: Invariant[str]) -> Invariant[str]: ... +def nominal(value: Any) -> Any: + return value + +def custom_nominal(value: Invariant[Any]): + ret = nominal(value) + integer: Invariant[int] = ret + string: Invariant[str] = ret + print(integer, string) diff --git a/packages/pyright-internal/src/tests/samples/overloadResultController1.py b/packages/pyright-internal/src/tests/samples/overloadResultController1.py new file mode 100644 index 000000000000..897c4953d20a --- /dev/null +++ b/packages/pyright-internal/src/tests/samples/overloadResultController1.py @@ -0,0 +1,112 @@ +from typing import Any, Sequence, TypeVar, overload, assert_type, reveal_type + +T = TypeVar("T") + +@overload +def choose(value: list[int]) -> list[int]: ... +@overload +def choose(value: list[str]) -> list[str]: ... +def choose(value: Any) -> Any: + return value + +def project(value: list[T]) -> Sequence[T]: + return value + +@overload +def unwrap(value: Sequence[int]) -> int: ... +@overload +def unwrap(value: Sequence[str]) -> str: ... +def unwrap(value: Any) -> Any: + return value[0] + +def positive(value: list[Any], plain: Sequence[Any], condition: bool): + ret = choose(value) + alias = ret + projected = project(alias) + scalar = unwrap(projected) + ret.append("x") + scalar.upper() + scalar.bit_length() + integer: list[int] = ret + string: list[str] = ret + string.append("x") + string.append(1) + bad: list[bytes] = ret + ret.append(3.14) + ret.nonexistent() + adjacent: int = "bad" + unwrap(project(ret) if condition else plain).upper() + unwrap(plain if condition else project(ret)).upper() + reveal_type(alias) + assert_type(alias, list[int]) + assert_type(alias, list[Any]) + print(integer, string, bad, adjacent) + ret = [b"new"] + alias.append("old") + ret.append(1) + +def repeated(value: list[Any]): + ret = choose(value) + ret.append("early") + print(ret, ret) + independent = choose(value) + independent.append("ok") + +def escaped(value: list[Any]): + ret = choose(value) + ret.append("early") + alias = ret + storage = [alias] + print(storage) + +def captured(value: list[Any]): + ret = choose(value) + ret.append("early") + def closure(): + ret.append("inner") + closure() + +def joined(value: list[Any], condition: bool): + ret = choose(value) + ret.append("early") + if condition: + ret = [b"new"] + ret.append("late") + +def returned(value: list[Any]): + ret = choose(value) + ret.append("early") + return ret + +class Box: + field: list[int] + +def attribute(value: list[Any], box: Box): + ret = choose(value) + ret.append("early") + box.field = ret + +def distinct(value: list[Any]): + ret = choose(value) + ret.append("early") + second = choose(value) + print(ret, second) + +def concrete(value: list[str]): + ret = value + ret.append(1) + ret.nonexistent() + +def gradual(value: list[Any]): + ret = value + ret.append(1) + ret.nonexistent() + +def unknown(value: list): + ret = value + ret.append("ordinary") + ret.nonexistent() + +def failed(value: list[Any]): + ret = choose(missing_producer) + ret.append("x") diff --git a/packages/pyright-internal/src/tests/samples/overloadResultDynamicReturns1.py b/packages/pyright-internal/src/tests/samples/overloadResultDynamicReturns1.py new file mode 100644 index 000000000000..326d94e4adf1 --- /dev/null +++ b/packages/pyright-internal/src/tests/samples/overloadResultDynamicReturns1.py @@ -0,0 +1,27 @@ +from typing import Any, overload + + +@overload +def choose(value: list[int]) -> list[int]: ... +@overload +def choose(value: list[str]) -> Any: ... +def choose(value: Any) -> Any: + return value + + +def root_dynamic_return(value: list[Any]): + ret = choose(value) + ret.missing_member() + + +@overload +def choose_nested(value: list[int]) -> list[int]: ... +@overload +def choose_nested(value: list[str]) -> list[Any]: ... +def choose_nested(value: Any) -> Any: + return value + + +def nested_dynamic_return(value: list[Any]): + ret = choose_nested(value) + ret.pop().missing_member() diff --git a/packages/pyright-internal/src/tests/samples/overloadResultReader1.py b/packages/pyright-internal/src/tests/samples/overloadResultReader1.py new file mode 100644 index 000000000000..2c379df113eb --- /dev/null +++ b/packages/pyright-internal/src/tests/samples/overloadResultReader1.py @@ -0,0 +1,14 @@ +from typing import Any, assert_type, overload + + +@overload +def choose(value: list[int]) -> list[int]: ... +@overload +def choose(value: list[str]) -> list[str]: ... +def choose(value: Any) -> Any: + return value + + +def invalid_expected_type(value: list[Any]): + ret = choose(value) + assert_type(1, ret) diff --git a/packages/pyright-internal/src/tests/testOnlyCacheTransaction.test.ts b/packages/pyright-internal/src/tests/testOnlyCacheTransaction.test.ts new file mode 100644 index 000000000000..ce242be7c145 --- /dev/null +++ b/packages/pyright-internal/src/tests/testOnlyCacheTransaction.test.ts @@ -0,0 +1,70 @@ +/* + * testOnlyCacheTransaction.test.ts + * Copyright (c) Microsoft Corporation. + * Licensed under the MIT license. + * + * Tests for the opt-in exact-node experiment transaction. + */ + +import * as assert from 'assert'; + +import { useTestOnlyCacheIsolation } from '../analyzer/typeCacheUtils'; +import { OperationCanceledException } from '../common/cancellationUtils'; + +test('TestOnlyCacheTransactionCommitsSelectedEntries', () => { + const original = {}; + const unrelated = {}; + const replacement = {}; + const cache = new Map([ + [1, original], + [2, unrelated], + [4, undefined], + ]); + const result = useTestOnlyCacheIsolation( + cache, + new Set([1, 3, 4]), + () => { + assert.ok(!cache.has(1) && !cache.has(3) && !cache.has(4)); + assert.strictEqual(cache.get(2), unrelated); + cache.set(1, replacement); + cache.set(3, undefined); + return replacement; + }, + true + ); + assert.strictEqual(result, replacement); + assert.strictEqual(cache.get(1), replacement); + assert.strictEqual(cache.get(2), unrelated); + assert.ok(cache.has(3) && !cache.has(4)); +}); + +test('TestOnlyCacheTransactionRollsBackFailures', () => { + for (const error of [new Error('failure'), new OperationCanceledException()]) { + const original = {}; + const unrelated = {}; + const cache = new Map([ + [1, original], + [2, unrelated], + [4, undefined], + ]); + assert.throws( + () => + useTestOnlyCacheIsolation( + cache, + new Set([1, 3, 4]), + () => { + cache.set(1, {}); + cache.set(3, {}); + cache.set(4, {}); + throw error; + }, + true + ), + (caught) => caught === error + ); + assert.strictEqual(cache.get(1), original); + assert.strictEqual(cache.get(2), unrelated); + assert.ok(!cache.has(3) && cache.has(4)); + assert.strictEqual(cache.get(4), undefined); + } +}); diff --git a/packages/pyright-internal/src/tests/testUtils.ts b/packages/pyright-internal/src/tests/testUtils.ts index e12970d9be25..5b21de8768f1 100644 --- a/packages/pyright-internal/src/tests/testUtils.ts +++ b/packages/pyright-internal/src/tests/testUtils.ts @@ -13,6 +13,7 @@ import * as path from 'path'; import * as AnalyzerNodeInfo from '../analyzer/analyzerNodeInfo'; import { ImportResolver } from '../analyzer/importResolver'; +import { ExperimentalOverloadResultOptions } from '../analyzer/overloadResultController'; import { Program } from '../analyzer/program'; import { Scope } from '../analyzer/scope'; import { NameTypeWalker } from '../analyzer/testWalker'; @@ -47,6 +48,11 @@ export interface FileAnalysisResult { deprecateds: Diagnostic[]; } +export interface SampleAnalysisObserver { + overloadResults?: ExperimentalOverloadResultOptions; + onComplete: (program: Program, fileUris: Uri[], configOptions: ConfigOptions) => void; +} + export function resolveSampleFilePath(fileName: string): string { return path.resolve(path.dirname(module.filename), `./samples/${fileName}`); } @@ -95,7 +101,8 @@ export function parseSampleFile( export function typeAnalyzeSampleFiles( fileNames: string[], configOptions = new ConfigOptions(Uri.empty()), - console?: ConsoleWithLogLevel + console?: ConsoleWithLogLevel, + observer?: SampleAnalysisObserver ): FileAnalysisResult[] { // Always enable "test mode". configOptions.internalTestMode = true; @@ -105,7 +112,16 @@ export function typeAnalyzeSampleFiles( const serviceProvider = createServiceProvider(fs, console || new NullConsole(), tempFile); const importResolver = new ImportResolver(serviceProvider, configOptions, new FullAccessHost(serviceProvider)); - const program = new Program(importResolver, configOptions, serviceProvider); + const program = new Program( + importResolver, + configOptions, + serviceProvider, + undefined, + undefined, + undefined, + undefined, + observer?.overloadResults + ); const fileUris = fileNames.map((name) => UriEx.file(resolveSampleFilePath(name))); program.setTrackedFiles(fileUris); @@ -117,12 +133,14 @@ export function typeAnalyzeSampleFiles( nameTypeWalker.walk(parserOutput.parseTree); }); - const results = getAnalysisResults(program, fileUris, configOptions); - - program.dispose(); - serviceProvider.dispose(); - - return results; + try { + const results = getAnalysisResults(program, fileUris, configOptions); + observer?.onComplete(program, fileUris, configOptions); + return results; + } finally { + program.dispose(); + serviceProvider.dispose(); + } } export function getAnalysisResults( diff --git a/packages/pyright-internal/src/tests/typeCacheUtils.test.ts b/packages/pyright-internal/src/tests/typeCacheUtils.test.ts index 2be1dde4a1cd..de8b9036dd30 100644 --- a/packages/pyright-internal/src/tests/typeCacheUtils.test.ts +++ b/packages/pyright-internal/src/tests/typeCacheUtils.test.ts @@ -14,13 +14,81 @@ import { ContextualTypeCacheEntry, contextualTypeCacheEntryMatches, findContextualTypeCacheEntry, + SpeculativeTypeTracker, } from '../analyzer/typeCacheUtils'; import { Type, TypeVarType } from '../analyzer/types'; +import { OperationCanceledException } from '../common/cancellationUtils'; +import { DiagnosticSink } from '../common/diagnosticSink'; +import { ParseNodeType } from '../parser/parseNodes'; +import { parseText } from './testUtils'; interface TestCacheEntry extends ContextualTypeCacheEntry { value: number; } +test.each(['complete', 'error', 'cancel'] as const)( + 'Exact-node isolation preserves disjoint native speculative contexts: %s', + (outcome) => { + const module = parseText('first\nsecond\nthird\n', new DiagnosticSink()).parserOutput.parseTree; + const [first, second, third] = module.d.statements; + assert.ok(first.nodeType === ParseNodeType.StatementList && second.nodeType === ParseNodeType.StatementList); + const firstValue = first.d.statements[0]; + const secondValue = second.d.statements[0]; + assert.strictEqual(firstValue.parent, first); + const tracker = new SpeculativeTypeTracker(); + const type = TypeVarType.createInstance('T'); + const cache = new Map([[secondValue.id, type]]); + tracker.enterSpeculativeContext(secondValue, { dependentType: type }); + tracker.trackEntry(cache, secondValue.id); + tracker.addSpeculativeType(secondValue, { type }, 0, undefined); + const original = tracker.getSpeculativeType(secondValue, undefined); + assert.ok(original); + assert.equal(tracker.canUseTestOnlyCacheIsolation(first), true); + assert.equal(tracker.canUseTestOnlyCacheIsolation(second), false); + assert.equal(tracker.canUseTestOnlyCacheIsolation(secondValue), false); + assert.equal(tracker.canUseTestOnlyCacheIsolation(module), false); + assert.throws(() => tracker.useTestOnlyCacheIsolation(new Set([secondValue.id]), () => {}, second)); + const error = outcome === 'cancel' ? new OperationCanceledException() : new Error('trial interruption'); + const isolated = () => + tracker.useTestOnlyCacheIsolation( + new Set([first.id, firstValue.id]), + () => { + assert.strictEqual(tracker.getSpeculativeType(secondValue, undefined), original); + tracker.enterSpeculativeContext(firstValue); + try { + assert.equal(tracker.canUseTestOnlyCacheIsolation(first), false); + assert.equal(tracker.canUseTestOnlyCacheIsolation(third), true); + tracker.addSpeculativeType(firstValue, { type }, 0, undefined); + if (outcome !== 'complete') { + throw error; + } + return type; + } finally { + tracker.leaveSpeculativeContext(); + } + }, + first + ); + if (outcome === 'complete') { + assert.strictEqual(isolated(), type); + } else { + assert.throws(isolated, (caught) => caught === error); + } + assert.strictEqual(tracker.getSpeculativeType(secondValue, undefined), original); + assert.strictEqual(cache.get(secondValue.id), type); + tracker.enterSpeculativeContext(firstValue); + assert.equal(tracker.getSpeculativeType(firstValue, undefined), undefined); + tracker.leaveSpeculativeContext(); + tracker.leaveSpeculativeContext(); + assert.equal(cache.has(secondValue.id), false); + assert.equal(tracker.isSpeculative(undefined), false); + tracker.enterSpeculativeContext(second); + assert.equal(tracker.canUseTestOnlyCacheIsolation(secondValue), false); + assert.throws(() => tracker.useTestOnlyCacheIsolation(new Set([secondValue.id]), () => {}, secondValue)); + tracker.leaveSpeculativeContext(); + } +); + test('ContextualTypeCacheEntryMatching', () => { const expectedType = TypeVarType.createInstance('T'); const otherExpectedType = TypeVarType.createInstance('U'); diff --git a/packages/pyright-internal/src/tests/typeEvaluator6.test.ts b/packages/pyright-internal/src/tests/typeEvaluator6.test.ts index 1d8641a98e96..31c7f894bf79 100644 --- a/packages/pyright-internal/src/tests/typeEvaluator6.test.ts +++ b/packages/pyright-internal/src/tests/typeEvaluator6.test.ts @@ -97,33 +97,8 @@ test('OverloadCall10', () => { test('OverloadCall11', () => { const analysisResults = TestUtils.typeAnalyzeSampleFiles(['overloadCall11.py']); - TestUtils.validateResults(analysisResults, 2); - // These diagnostics record the restored conformance gap, not the required overload semantics. - assert.deepStrictEqual( - analysisResults[0].errors.map((diagnostic) => ({ - rule: diagnostic.getRule(), - range: diagnostic.range, - message: diagnostic.message, - })), - [ - { - rule: DiagnosticRule.reportAssignmentType, - range: { start: { line: 28, character: 28 }, end: { line: 28, character: 34 } }, - message: - 'Type "list[int]" is not assignable to declared type "list[str]"\n' + - '\u00a0\u00a0"list[int]" is not assignable to "list[str]"\n' + - '\u00a0\u00a0\u00a0\u00a0Type parameter "_T@list" is invariant, but "int" is not the same as "str"\n' + - '\u00a0\u00a0\u00a0\u00a0Consider switching from "list" to "Sequence" which is covariant', - }, - { - rule: DiagnosticRule.reportArgumentType, - range: { start: { line: 31, character: 18 }, end: { line: 31, character: 25 } }, - message: - 'Argument of type "Literal[\'value\']" cannot be assigned to parameter "object" of type "int" in function "append"\n' + - '\u00a0\u00a0"Literal[\'value\']" is not assignable to "int"', - }, - ] - ); + // Both invariant assignments and both append operations are valid witnesses. + TestUtils.validateResults(analysisResults, 0); }); test('OverloadCall12', () => { diff --git a/packages/pyright-internal/src/tests/typeUtils.test.ts b/packages/pyright-internal/src/tests/typeUtils.test.ts index 6af05857530d..2717074f471c 100644 --- a/packages/pyright-internal/src/tests/typeUtils.test.ts +++ b/packages/pyright-internal/src/tests/typeUtils.test.ts @@ -8,7 +8,7 @@ import * as assert from 'assert'; -import { mapSignatures, transformTypePair } from '../analyzer/typeUtils'; +import { containsLiteralType, mapSignatures, transformTypePair } from '../analyzer/typeUtils'; import { AnyType, ClassType, @@ -21,7 +21,9 @@ import { isOverloaded, isTypeVar, OverloadedType, + OverloadResultType, Type, + TypeCategory, TypeVarScopeType, TypeVarType, UnionType, @@ -31,6 +33,48 @@ import { import { Uri } from '../common/uri/uri'; import { ParamCategory } from '../parser/parseNodes'; +test('Literal detection keeps cancellation and type-argument options local to each walk', () => { + const plain = ClassType.cloneAsInstance(createClass('Value')); + const literal = ClassType.cloneWithLiteral(plain, 1); + const nested = ClassType.specialize(ClassType.cloneAsInstance(createClass('Container')), [literal]); + + for (let i = 0; i < 3; i++) { + assert.strictEqual(containsLiteralType(literal), true); + assert.strictEqual(containsLiteralType(plain), false); + assert.strictEqual(containsLiteralType(nested, true), true); + assert.strictEqual(containsLiteralType(nested), false); + assert.strictEqual(containsLiteralType(nested, false), false); + } + assert.strictEqual(nested.priv.typeArgs?.[0], literal); + assert.strictEqual(plain.priv.literalValue, undefined); +}); + +test('Literal detection visits callable, union and overload-result components', () => { + const plain = ClassType.cloneAsInstance(createClass('Value')); + const literal = ClassType.cloneWithLiteral(plain, 'value'); + const first = createFunction(plain, plain, plain); + const second = createFunction(plain, plain, literal); + const overloads = OverloadedType.create([first, second]); + const union = UnionType.create(); + union.priv.subtypes = [plain, literal]; + const result = OverloadResultType.create([plain, literal], plain, TypeCategory.Any); + + for (const type of [second, overloads, union, result]) { + assert.strictEqual(containsLiteralType(type), true); + assert.strictEqual(containsLiteralType(first), false); + } +}); + +test('Literal detection does not retain recursion-limit state between walks', () => { + const recursive = ClassType.cloneAsInstance(createClass('Recursive')); + recursive.priv.typeArgs = [recursive]; + const literal = ClassType.cloneWithLiteral(ClassType.cloneAsInstance(createClass('Value')), 1); + + assert.strictEqual(containsLiteralType(recursive, true), false); + assert.strictEqual(containsLiteralType(literal), true); + assert.strictEqual(containsLiteralType(recursive, true), false); +}); + test('Map signatures preserves replaced overloads and their order', () => { const unknown = UnknownType.create(); const any = AnyType.create(); diff --git a/packages/pyright-internal/src/typeServer/protocol/typeServerProtocol.ts b/packages/pyright-internal/src/typeServer/protocol/typeServerProtocol.ts index ccd8e52f4112..b9c5b4df8ac7 100644 --- a/packages/pyright-internal/src/typeServer/protocol/typeServerProtocol.ts +++ b/packages/pyright-internal/src/typeServer/protocol/typeServerProtocol.ts @@ -453,6 +453,7 @@ export namespace TypeServerProtocol { Overloaded, // Functions with multiple @overload signatures Synthesized, // Types that are synthesized by the type checker TypeReference, // Reference by ID for deduplication + OverloadResult, // Complete witnesses for a materialization-ambiguous call } /** @@ -1255,6 +1256,12 @@ export namespace TypeServerProtocol { readonly typeReferenceId: number; } + export interface OverloadResultType extends TypeBase { + readonly candidates: Type[]; + readonly baselineType: Type; + readonly uncertaintyKind: 'any' | 'unknown'; + } + export type Type = | BuiltInType | DeclaredType @@ -1264,6 +1271,7 @@ export namespace TypeServerProtocol { | ModuleType | TypeVarType | OverloadedType + | OverloadResultType | SynthesizedType | TypeReferenceType; diff --git a/packages/pyright-internal/src/typeServer/typeServerConversionUtils.ts b/packages/pyright-internal/src/typeServer/typeServerConversionUtils.ts index 4e902db1a760..8a7d58734945 100644 --- a/packages/pyright-internal/src/typeServer/typeServerConversionUtils.ts +++ b/packages/pyright-internal/src/typeServer/typeServerConversionUtils.ts @@ -198,6 +198,16 @@ function toProtocolTypeAliasInfo(info: PyrightTypes.TypeAliasInfo | undefined, f // Otherwise, cycle detection won't work. function toProtocolType(id: number, type: PyrightTypes.Type, factory: ProtocolTypeFactory): TypeServerProtocol.Type { switch (type.category) { + case PyrightTypes.TypeCategory.OverloadResult: + return { + id, + kind: TypeServerProtocol.TypeKind.OverloadResult, + flags: toProtocolTypeFlags(type), + candidates: toProtocolTypes([...type.priv.candidates], factory), + baselineType: toProtocolTypeOrUndefined(type.priv.baselineType, factory)!, + uncertaintyKind: type.priv.uncertaintyKind === PyrightTypes.TypeCategory.Any ? 'any' : 'unknown', + } satisfies TypeServerProtocol.OverloadResultType; + case PyrightTypes.TypeCategory.Class: { const decl = type.shared.declaration; if (decl) { From 9be56a986dacb53469694417461328bbf3918803 Mon Sep 17 00:00:00 2001 From: Rich Chiodo Date: Fri, 18 Sep 2026 16:14:15 -0700 Subject: [PATCH 2/3] Fix evaluator lifetime during memory-pressure eviction Defer cache eviction until the owning file check or outer request unwinds, bind controllers to their own evaluator, and prevent late asynchronous cleanup from recreating a disposed Program. Add regressions for diagnostic completeness, precise overload alternatives, cancellation, and shutdown. Classification: B (Pyright evaluator lifecycle limitation). No typeshed changes or type-precision reductions. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- docs/overloadResultExperiment.md | 11 + .../pyright-internal/src/analyzer/program.ts | 140 ++++++----- .../tests/overloadResultController.test.ts | 226 +++++++++++++++++- .../samples/overloadResultCachePressure1.py | 33 +++ 4 files changed, 353 insertions(+), 57 deletions(-) create mode 100644 packages/pyright-internal/src/tests/samples/overloadResultCachePressure1.py diff --git a/docs/overloadResultExperiment.md b/docs/overloadResultExperiment.md index 5aa1ed30b23f..f2e0ae24b17f 100644 --- a/docs/overloadResultExperiment.md +++ b/docs/overloadResultExperiment.md @@ -81,6 +81,17 @@ Activation state is local to an evaluator/controller and keyed by parse-node identity. Replacement parse generations do not inherit activations or pending probes. Disposal clears pending work and retires owned outcomes. +Memory-pressure eviction requested by lazy import lookup is deferred until the +owning Program invocation unwinds. A checker retains one evaluator/controller +for the complete file; analysis releases that ownership between files, not at +the end of the entire project. Language-service requests retain ownership across +nested and asynchronous work. Pending eviction runs when the outermost invocation +finishes, including cancellation or failure, without changing the memory threshold. +Program disposal cancels pending eviction; a late async completion or rejection +must not recreate an evaluator after shutdown or replace the request's outcome. +Each controller's discovery callback is bound to its own evaluator, never a +replacement evaluator's caches. Direct use of a retired controller remains an error. + Ordinary public queries do not become experimental operations merely because another function in the module activated a producer. Records that already own ordinary or failed outcomes still require their private restoration and diff --git a/packages/pyright-internal/src/analyzer/program.ts b/packages/pyright-internal/src/analyzer/program.ts index b40447b8a748..6864436ed037 100644 --- a/packages/pyright-internal/src/analyzer/program.ts +++ b/packages/pyright-internal/src/analyzer/program.ts @@ -176,6 +176,8 @@ export class Program { private _importResolver: ImportResolver; private _evaluator: TypeEvaluator | undefined; private _experimentalOverloadResult?: ExperimentalOverloadResultController; + private _evaluatorInvocationCount = 0; + private _cacheClearPending = false; private _disposed = false; private _parsedFileCount = 0; private _preCheckCallback: PreCheckCallback | undefined; @@ -266,6 +268,7 @@ export class Program { } dispose() { + this._cacheClearPending = false; if ((this._testOnlyChecking || this._experimentalOverloadResult) && !this._disposed) { this._evaluator?.disposeEvaluator(); } @@ -783,52 +786,48 @@ export class Program { // analysis. In interactive mode, the timeout is always limited // to the smaller value to maintain responsiveness. analyze(maxTime?: MaxAnalysisTime, token: CancellationToken = CancellationToken.None): boolean { - return this._runEvaluatorWithCancellationToken(token, () => { - const elapsedTime = new Duration(); + const elapsedTime = new Duration(); - const openFiles = this._sourceFileList.filter( - (sf) => sf.isOpenByClient && sf.sourceFile.isCheckingRequired() - ); + const openFiles = this._sourceFileList.filter((sf) => sf.isOpenByClient && sf.sourceFile.isCheckingRequired()); - if (openFiles.length > 0) { - const effectiveMaxTime = maxTime ? maxTime.openFilesTimeInMs : Number.MAX_VALUE; + if (openFiles.length > 0) { + const effectiveMaxTime = maxTime ? maxTime.openFilesTimeInMs : Number.MAX_VALUE; - // Check the open files. - for (const sourceFileInfo of openFiles) { - if (this._checkTypes(sourceFileInfo)) { - if (elapsedTime.getDurationInMilliseconds() > effectiveMaxTime) { - return true; - } + // Check the open files. + for (const sourceFileInfo of openFiles) { + if (this._runEvaluatorWithCancellationToken(token, () => this._checkTypes(sourceFileInfo))) { + if (elapsedTime.getDurationInMilliseconds() > effectiveMaxTime) { + return true; } } + } - // If the caller specified a maxTime, return at this point - // since we've finalized all open files. We want to get - // the results to the user as quickly as possible. - if (maxTime !== undefined) { - return true; - } + // If the caller specified a maxTime, return at this point + // since we've finalized all open files. We want to get + // the results to the user as quickly as possible. + if (maxTime !== undefined) { + return true; } + } - if (!this._configOptions.checkOnlyOpenFiles) { - const effectiveMaxTime = maxTime ? maxTime.noOpenFilesTimeInMs : Number.MAX_VALUE; + if (!this._configOptions.checkOnlyOpenFiles) { + const effectiveMaxTime = maxTime ? maxTime.noOpenFilesTimeInMs : Number.MAX_VALUE; - // Now do type parsing and analysis of the remaining. - for (const sourceFileInfo of this._sourceFileList) { - if (!isUserCode(sourceFileInfo)) { - continue; - } + // Now do type parsing and analysis of the remaining. + for (const sourceFileInfo of this._sourceFileList) { + if (!isUserCode(sourceFileInfo)) { + continue; + } - if (this._checkTypes(sourceFileInfo)) { - if (elapsedTime.getDurationInMilliseconds() > effectiveMaxTime) { - return true; - } + if (this._runEvaluatorWithCancellationToken(token, () => this._checkTypes(sourceFileInfo))) { + if (elapsedTime.getDurationInMilliseconds() > effectiveMaxTime) { + return true; } } } + } - return false; - }); + return false; } // Performs parsing and analysis of a single file in the program. If the file is not part of @@ -988,10 +987,9 @@ export class Program { } getTypeOfSymbol(symbol: Symbol) { - this._handleMemoryHighUsage(); - - const evaluator = this._evaluator || this._createNewEvaluator(); - return evaluator.getEffectiveTypeOfSymbol(symbol); + return this._runEvaluatorWithCancellationToken(undefined, () => + this._evaluator!.getEffectiveTypeOfSymbol(symbol) + ); } printType(type: Type, options?: PrintTypeOptions): string { @@ -1173,6 +1171,13 @@ export class Program { // Discards any cached information associated with this program. emptyCache() { + // Lazy import lookup can request eviction while a checker or language-service + // request still owns this evaluator, its controller, and private cache scopes. + if (this._evaluatorInvocationCount > 0) { + this._cacheClearPending = true; + return; + } + this._cacheClearPending = false; this._createNewEvaluator(); this._discardCachedParseResults(); this._parsedFileCount = 0; @@ -1242,6 +1247,9 @@ export class Program { } private _handleMemoryHighUsage() { + if (this._cacheClearPending) { + return; + } const cacheUsage = this._cacheManager.getCacheUsage(); const usedHeapRatio = this._cacheManager.getUsedHeapRatio( this._configOptions.verboseOutput ? this._console : undefined @@ -1289,31 +1297,53 @@ export class Program { token: CancellationToken | undefined, callback: () => T | Promise ): T | Promise { + if (this._evaluatorInvocationCount === 0) { + this._handleMemoryHighUsage(); + } + let isAsync = false; + this._evaluatorInvocationCount++; + const release = () => { + this._evaluatorInvocationCount--; + if (!this._disposed && this._evaluatorInvocationCount === 0 && this._cacheClearPending) { + this.emptyCache(); + } + }; try { const result = token ? this._evaluator!.runWithCancellationToken(token, callback) : callback(); if (!isThenable(result)) { return result; } - return result.catch((e) => { - if ( - !OperationCanceledException.is(e) || - e.isTypeCacheInvalid || - e.code === LSPErrorCodes.ServerCancelled - ) { - this._createNewEvaluator(); - } + isAsync = true; + return result + .catch((e) => { + if ( + !this._disposed && + (!OperationCanceledException.is(e) || + e.isTypeCacheInvalid || + e.code === LSPErrorCodes.ServerCancelled) + ) { + this._createNewEvaluator(); + } - throw e; - }); + throw e; + }) + .finally(release); } catch (e: any) { // An unexpected exception occurred, potentially leaving the current evaluator // in an inconsistent state. Discard it and replace it with a fresh one. It is // Cancellation exceptions are known to handle this correctly. - if (!OperationCanceledException.is(e) || e.isTypeCacheInvalid || e.code === LSPErrorCodes.ServerCancelled) { + if ( + !this._disposed && + (!OperationCanceledException.is(e) || e.isTypeCacheInvalid || e.code === LSPErrorCodes.ServerCancelled) + ) { this._createNewEvaluator(); } throw e; + } finally { + if (!isAsync) { + release(); + } } } @@ -2006,14 +2036,14 @@ export class Program { ? { automatic: true, checkerHandoff: false } : undefined); assert(!this._testOnlyChecking || !overloadResultOptions); - this._experimentalOverloadResult = overloadResultOptions + const overloadResultController = overloadResultOptions ? new ExperimentalOverloadResultController( overloadResultOptions, - () => this._evaluator!, + () => evaluator, this._analyzerNodeInfoContext ) : undefined; - this._evaluator = createTypeEvaluatorWithTracker( + const evaluator: TypeEvaluator = createTypeEvaluatorWithTracker( this._lookUpImport, { printTypeFlags: getPrintTypeFlags(this._configOptions), @@ -2024,7 +2054,7 @@ export class Program { nodeInfoReader: this._analyzerNodeInfoContext, maxCodeComplexity: this._configOptions.maxCodeComplexity, testOnlyExpression: this._testOnlyChecking?.expression(), - experimentalOverloadResult: this._experimentalOverloadResult, + experimentalOverloadResult: overloadResultController, }, this._logTracker, this._configOptions.logTypeEvaluationTime @@ -2037,7 +2067,9 @@ export class Program { : undefined ); - return this._evaluator; + this._experimentalOverloadResult = overloadResultController; + this._evaluator = evaluator; + return evaluator; } private _parseFile(fileToParse: SourceFileInfo, content?: string, skipFileNeededCheck?: boolean) { @@ -2309,10 +2341,6 @@ export class Program { fileToCheck: SourceFileInfo, options?: { chainedByList?: SourceFileInfo[]; skipFileNeededCheck?: boolean } ) { - // For very large programs, we may need to discard the evaluator and - // its cached types to avoid running out of heap space. - this._handleMemoryHighUsage(); - return this._logTracker.log(`analyzing: ${fileToCheck.uri}`, (logState) => { // If the file isn't needed because it was eliminated from the // transitive closure or deleted, skip the file rather than wasting diff --git a/packages/pyright-internal/src/tests/overloadResultController.test.ts b/packages/pyright-internal/src/tests/overloadResultController.test.ts index c0ef64c7f272..4bf0174c2d2d 100644 --- a/packages/pyright-internal/src/tests/overloadResultController.test.ts +++ b/packages/pyright-internal/src/tests/overloadResultController.test.ts @@ -8,6 +8,7 @@ import * as assert from 'assert'; import { CancellationToken } from 'vscode-languageserver'; import { createAnalyzerNodeInfoAccessor } from '../analyzer/analyzerNodeInfo'; +import { CacheManager } from '../analyzer/cacheManager'; import { ImportResolver } from '../analyzer/importResolver'; import { ExperimentalOverloadResultOptions, OverloadResultEvent } from '../analyzer/overloadResultController'; import { getEnclosingFunction } from '../analyzer/parseTreeUtils'; @@ -27,6 +28,7 @@ import { import { OperationCanceledException } from '../common/cancellationUtils'; import { ConfigOptions } from '../common/configOptions'; import { NullConsole } from '../common/console'; +import { createDeferred } from '../common/deferred'; import { Diagnostic, DiagnosticCategory } from '../common/diagnostic'; import { FullAccessHost } from '../common/fullAccessHost'; import { pythonVersion3_12 } from '../common/pythonVersion'; @@ -86,7 +88,8 @@ function create( checking?: ConstructorParameters[6] ) { const temp = new RealTempFile(); - const service = createServiceProvider(createFromRealFileSystem(temp), new NullConsole(), temp); + const cacheManager = new CacheManager(); + const service = createServiceProvider(createFromRealFileSystem(temp), new NullConsole(), temp, cacheManager); const uri = UriEx.file(resolveSampleFilePath('overloadResultController1.py')); const config = new ConfigOptions(uri.getDirectory()); config.defaultPythonVersion = pythonVersion3_12; @@ -171,6 +174,7 @@ function create( let disposed = false; return { program, + cacheManager, uri, config, events, @@ -189,6 +193,226 @@ function create( }; } +test.each(['default', false] as const)( + 'AutomaticOverloadResult memory pressure during lazy import preserves checking config=%s', + (config) => { + const source = 'from os import name\nbefore: int = "bad"\nafter: str = 1\n'; + const reference = create(source, false, config); + const test = create(source, false, config); + const nextUri = test.uri.getDirectory().combinePaths('overloadResultCachePressureNext.py'); + test.program.setTrackedFiles([test.uri, nextUri]); + test.program.setFileOpened(nextUri, 1, 'next_error: int = "bad"\n'); + let pressure = false; + const heap = jest.spyOn(test.cacheManager, 'getUsedHeapRatio').mockImplementation(() => { + if (!pressure) { + return 0; + } + pressure = false; + return 1; + }); + const empty = jest.spyOn(test.program, 'emptyCache'); + const evaluator = test.program.evaluator!; + const name = test.nodes.find((node): node is NameNode => node.nodeType === ParseNodeType.Name)!; + let nextEvaluator: TypeEvaluator | undefined; + test.program.setPreCheckCallback((parsed, current) => { + if (parsed.parseTree === test.module) { + pressure = true; + } else { + nextEvaluator = current; + } + }); + try { + const expected = reference.analyze(); + assert.equal(expected.filter((d) => d.category === DiagnosticCategory.Error).length, 2); + assert.deepStrictEqual(test.analyze(), expected); + assert.equal(empty.mock.calls.length, 2); + assert.notStrictEqual(test.program.evaluator, evaluator); + assert.strictEqual(nextEvaluator, test.program.evaluator); + const nextDiagnostics = test.program.getSourceFile(nextUri)!.getDiagnostics(test.config)!; + assert.equal(nextDiagnostics.length, 1); + assert.equal(nextDiagnostics[0].getRule(), 'reportAssignmentType'); + if (config === 'default') { + assert.throws(() => evaluator.getTypeOfExpression(name), /Retired overload-result controller/); + } + } finally { + heap.mockRestore(); + empty.mockRestore(); + reference.dispose(); + test.dispose(); + } + } +); + +test.each([false, true])( + 'AutomaticOverloadResult memory pressure inside an owned operation queryFirst=%s', + (queryFirst) => { + const source = readSampleFile('overloadResultCachePressure1.py'); + const reference = create(source, { automatic: true }); + let pressure = false; + let triggered = false; + const test = create(source, { + automatic: true, + observe(event) { + if ( + !triggered && + event.kind === 'beforeCall' && + event.node.d.leftExpr.nodeType === ParseNodeType.Name && + event.node.d.leftExpr.d.value === 'cold' + ) { + pressure = true; + triggered = true; + } + }, + }); + const original = test.program.evaluator!; + const controller = test.program.experimentalOverloadResultController!; + const heap = jest.spyOn(test.cacheManager, 'getUsedHeapRatio').mockImplementation(() => { + if (!pressure) { + return 0; + } + pressure = false; + return 1; + }); + const empty = jest.spyOn(test.cacheManager, 'emptyCache'); + try { + if (queryFirst) { + test.program.run((program) => { + const call = test.nodes.find( + (node): node is CallNode => + node.nodeType === ParseNodeType.Call && + node.d.leftExpr.nodeType === ParseNodeType.MemberAccess && + node.d.leftExpr.d.member.d.value === 'append' && + source.substring(node.start, node.start + node.length).includes('cold()') + )!; + program.evaluator!.getType(call); + assert.strictEqual(program.evaluator, original); + assert.ok(isOverloadResult(program.evaluator!.getType(test.name('ret'))!)); + }, CancellationToken.None); + } + const expected = reference.analyze(); + assert.deepStrictEqual( + expected + .filter((d) => d.category === DiagnosticCategory.Error) + .map((d) => [d.range.start.line, d.rule]), + [ + ['before: int', 'reportAssignmentType'], + ['ret.append(cold())', 'reportArgumentType'], + ['ret.extend([1, "mixed"])', 'reportArgumentType'], + ['bad: list[bytes]', 'reportAssignmentType'], + ['ret.append(3.14)', 'reportArgumentType'], + ['ret.nonexistent()', 'reportAttributeAccessIssue'], + ['after: int', 'reportAssignmentType'], + ['after_file: str', 'reportAssignmentType'], + ].map(([text, rule]) => [source.slice(0, source.indexOf(text)).split('\n').length - 1, rule]) + ); + assert.ok(expected.some((d) => d.message.includes('OverloadResult[list[int], list[str]]'))); + assert.deepStrictEqual(test.analyze(), expected); + assert.equal(empty.mock.calls.length, 1); + assert.notStrictEqual(test.program.evaluator, original); + assert.equal(controller.getStats(test.module), undefined); + assert.throws(() => original.getType(test.name('ret')), /Retired overload-result controller/); + const nodes = new Nodes(); + nodes.walk(test.program.getParseResults(test.uri)!.parserOutput.parseTree); + const alias = nodes.nodes.find( + (node): node is NameNode => node.nodeType === ParseNodeType.Name && node.d.value === 'alias' + )!; + test.program.run((program) => { + assert.deepStrictEqual( + resultSummary(program.evaluator!, program.evaluator!.getTypeOfExpression(alias)), + resultSummary( + reference.program.evaluator!, + reference.program.evaluator!.getTypeOfExpression(reference.name('alias')) + ) + ); + }, CancellationToken.None); + assert.deepStrictEqual(test.analyze(), expected); + } finally { + heap.mockRestore(); + empty.mockRestore(); + reference.dispose(); + test.dispose(); + } + } +); + +test.each(['complete', 'cancel', 'error'] as const)( + 'AutomaticOverloadResult deferred cache cleanup unwinds nested async requests: %s', + async (outcome) => { + const source = 'def positive():\n bad: int = "wrong"\n'; + const reference = create(source, false, 'default'); + const test = create(source, false, 'default'); + const evaluator = test.program.evaluator!; + const node = test.name('bad'); + try { + const request = test.program.run(async (program) => { + test.cacheManager.emptyCache(); + test.program.run(() => { + test.cacheManager.emptyCache(); + }, CancellationToken.None); + await Promise.resolve(); + assert.strictEqual(program.evaluator, evaluator); + if (outcome === 'cancel') { + throw new OperationCanceledException(); + } + if (outcome === 'error') { + throw new Error('cache pressure interruption'); + } + }, CancellationToken.None); + if (outcome === 'complete') { + await request; + } else { + await assert.rejects(request, (error) => + outcome === 'cancel' + ? OperationCanceledException.is(error) + : error instanceof Error && error.message === 'cache pressure interruption' + ); + } + assert.notStrictEqual(test.program.evaluator, evaluator); + assert.throws(() => evaluator.getType(node), /Retired overload-result controller/); + assert.deepStrictEqual(test.analyze(), reference.analyze()); + const current = test.program.evaluator; + test.cacheManager.emptyCache(); + assert.notStrictEqual(test.program.evaluator, current); + } finally { + reference.dispose(); + test.dispose(); + } + } +); + +test.each(['complete', 'cancel', 'error'] as const)( + 'AutomaticOverloadResult pending cache cleanup cannot recreate a disposed program: %s', + async (outcome) => { + const test = create('def positive():\n bad: int = "wrong"\n', false, 'default'); + const evaluator = test.program.evaluator!; + const node = test.name('bad'); + const resume = createDeferred(); + const error = outcome === 'cancel' ? new OperationCanceledException() : new Error('shutdown interruption'); + try { + const request = test.program.run(async () => { + test.cacheManager.emptyCache(); + await resume.promise; + if (outcome !== 'complete') { + throw error; + } + return 42; + }, CancellationToken.None); + test.dispose(); + resume.resolve(); + if (outcome === 'complete') { + assert.equal(await request, 42); + } else { + await assert.rejects(request, (actual) => actual === error); + } + assert.strictEqual(test.program.evaluator, evaluator); + assert.throws(() => evaluator.getType(node), /Retired overload-result controller/); + } finally { + resume.resolve(); + test.dispose(); + } + } +); + test.each([false, true])('AutomaticOverloadResult live checker handoff queryFirst=%s', (queryFirst) => { const baseline = create(fixture, { automatic: true, checkerHandoff: false }, false, false); const automatic = create(fixture, { automatic: true, checkerHandoff: true }, false, false); diff --git a/packages/pyright-internal/src/tests/samples/overloadResultCachePressure1.py b/packages/pyright-internal/src/tests/samples/overloadResultCachePressure1.py new file mode 100644 index 000000000000..3b8d9a4eaf6e --- /dev/null +++ b/packages/pyright-internal/src/tests/samples/overloadResultCachePressure1.py @@ -0,0 +1,33 @@ +from typing import Any, overload, reveal_type + + +@overload +def choose(value: list[int]) -> list[int]: ... +@overload +def choose(value: list[str]) -> list[str]: ... +def choose(value: Any) -> Any: + return value + + +def positive(value: list[Any]): + before: int = "bad" + ret = choose(value) + alias = ret + strings: list[str] = alias + ret.append("ok") + ret.append(cold()) + ret.extend([1, "mixed"]) + bad: list[bytes] = ret + ret.append(3.14) + ret.nonexistent() + after: int = "bad" + reveal_type(alias) + + +def cold(): + from os import getpid + + return getpid() / 2 + + +after_file: str = 1 From ea2db303b8ec8d502c0d9d54f60b81c7cde34180 Mon Sep 17 00:00:00 2001 From: Rich Chiodo Date: Fri, 18 Sep 2026 16:43:06 -0700 Subject: [PATCH 3/3] Formalize production cache isolation and cover protocol conversion Define the evaluator-owned isolation contract and rename production cache APIs without changing transaction, speculation, or diagnostic semantics. Preserve all existing type expectations and the evaluator lifetime fix. Classification: D (runtime coverage gap). Add real JSON-RPC carrier conversion and nested factory/reference coverage, plus exact-node restoration coverage. No typeshed changes or type-precision regressions. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- docs/overloadResultExperiment.md | 16 +++ .../src/analyzer/overloadResultController.ts | 35 +++--- .../src/analyzer/typeCacheUtils.ts | 15 +-- .../src/analyzer/typeEvaluator.ts | 96 ++++++++-------- .../src/analyzer/typeEvaluatorTypes.ts | 31 ++++++ ...ion.test.ts => typeCacheIsolation.test.ts} | 36 ++++-- .../src/tests/typeCacheUtils.test.ts | 20 ++-- .../typeServer/typeServer.inProc.test.ts | 104 ++++++++++++++++++ .../src/trace/nativeAdapter.ts | 6 +- 9 files changed, 259 insertions(+), 100 deletions(-) rename packages/pyright-internal/src/tests/{testOnlyCacheTransaction.test.ts => typeCacheIsolation.test.ts} (62%) diff --git a/docs/overloadResultExperiment.md b/docs/overloadResultExperiment.md index f2e0ae24b17f..8d9c5d93a4c9 100644 --- a/docs/overloadResultExperiment.md +++ b/docs/overloadResultExperiment.md @@ -77,6 +77,18 @@ Rejected concrete calls no longer perform the old charged discovery operation. ## Ownership and invalidation +`EvaluationOperationController` and `EvaluationCacheIsolation` define the production +evaluator/controller boundary. The evaluator installs isolation and eviction together, +bound to its own lifetime. `useNodeCacheIsolation` restores the exact registered +node-ID entries on ordinary exit, error, or cancellation; explicit successful retention +keeps only non-speculative results. Unowned cache writes survive. Speculative isolation +requires disjoint native roots and never replaces the native speculation stack; +eviction requires no active speculation, and both operations forbid return inference. +The controller separately owns private diagnostic capture and complete-witness +publication, while ordinary unowned diagnostics retain their surrounding native sink. +`testOnlyExpression` and `testOnlyMemberAccess` remain test-only injection options, +not names for these required production guarantees. + Activation state is local to an evaluator/controller and keyed by parse-node identity. Replacement parse generations do not inherit activations or pending probes. Disposal clears pending work and retires owned outcomes. @@ -121,6 +133,10 @@ native diagnostics and TypeResult metadata, proves actual carrier selection, and requires a single complete witness for mixed-element `extend` operations. Separate tests preserve pending activation across external call speculation and exercise disjoint/overlapping cache roots under completion, error and cancellation. +Type-server tests request a normally activated carrier over the real in-process +JSON-RPC connection, checking both candidates, nested generic arguments, baseline +references, uncertainty and ordinary concrete-type compatibility. A separate factory +case covers nested Unknown-origin carriers without broadening automatic admission. Timing is measured separately from observer instrumentation; no timing threshold is asserted in CI. Earlier V3 residual measurements are historical evidence, diff --git a/packages/pyright-internal/src/analyzer/overloadResultController.ts b/packages/pyright-internal/src/analyzer/overloadResultController.ts index 473a3b753b9f..4635fe04dc3a 100644 --- a/packages/pyright-internal/src/analyzer/overloadResultController.ts +++ b/packages/pyright-internal/src/analyzer/overloadResultController.ts @@ -36,7 +36,14 @@ import { isFixedOverloadArgumentShape, OverloadSelectionBudget, } from './overloadResultSelector'; -import { EvalFlags, TypeEvaluator, TypeResult } from './typeEvaluatorTypes'; +import { + EvalFlags, + EvaluationCacheIsolation, + EvaluationOperationController, + EvaluationOperationRequest, + TypeEvaluator, + TypeResult, +} from './typeEvaluatorTypes'; import { isClassInstance, isOverloadResult, OverloadResultType, Type, TypeCategory } from './types'; import { InferenceContext } from './typeUtils'; @@ -111,11 +118,6 @@ export interface LocalOverloadOperation { context?: InferenceContext; } -export type OverloadOperationRequest = - | { kind: 'query'; node: ExpressionNode } - | { kind: 'expression'; node: ExpressionNode; flags: EvalFlags; context?: InferenceContext } - | { kind: 'statement'; node: AssignmentNode; evaluate: () => void }; - type OperationMode = | 'baseline' | 'candidate' @@ -197,7 +199,7 @@ const defaultLimits: Readonly = { class CandidateCutoff extends Error {} class ActivationRestart extends Error {} -export class ExperimentalOverloadResultController { +export class ExperimentalOverloadResultController implements EvaluationOperationController { readonly roots = new Map>(); private readonly _limits: OverloadResultLimits; private readonly _info: AnalyzerNodeInfoAccessor; @@ -214,8 +216,7 @@ export class ExperimentalOverloadResultController { private readonly _activationChecked = new WeakSet(); private readonly _activated = new WeakSet(); private readonly _pendingActivation = new Set(); - private _isolate?: (root: ParseNode, callback: () => TypeResult, retain?: boolean) => TypeResult; - private _evict?: (root: ParseNode) => void; + private _cacheIsolation?: EvaluationCacheIsolation; private _select?: (node: CallNode, limit: number, proofUnits: number) => AutomaticOverloadSelection; private _captureQueryCache?: (root: ParseNode, node: NameNode) => (() => boolean) | undefined; private _checkerHandoff?: ( @@ -265,12 +266,9 @@ export class ExperimentalOverloadResultController { return this._pendingActivation.size > 0; } - install = (isolate: (root: ParseNode, callback: () => TypeResult, retain?: boolean) => TypeResult) => { - this._isolate = isolate; - }; - - installEviction = (evict: (root: ParseNode) => void) => { - this._evict = evict; + installCacheIsolation = (isolation: EvaluationCacheIsolation) => { + assert(!this._cacheIsolation && !this._disposed); + this._cacheIsolation = isolation; }; installSelector = (select: (node: CallNode, limit: number, proofUnits: number) => AutomaticOverloadSelection) => { @@ -432,11 +430,10 @@ export class ExperimentalOverloadResultController { this._disposed = true; }; - // Kept structurally compatible with the original default-off cache seam. dispatch = (_node: ExpressionNode, _flags: EvalFlags, _context?: InferenceContext): TypeResult | undefined => undefined; - route = (request: OverloadOperationRequest): { result?: TypeResult } | undefined => { + route = (request: EvaluationOperationRequest): { result?: TypeResult } | undefined => { assert(!this._disposed, 'Retired overload-result controller'); if (request.kind !== 'statement') { const result = this._query( @@ -591,7 +588,7 @@ export class ExperimentalOverloadResultController { const generation = this._ensure(module); generation?.admission.records.forEach((record) => { if (this._isEnabled(generation, record)) { - this._evict?.(record.root); + this._cacheIsolation!.evict(record.root); } }); } @@ -882,7 +879,7 @@ export class ExperimentalOverloadResultController { result = nestedSpeculative || liveCheckerCache ? evaluateAndContinue() - : this._isolate!(record.root, evaluateAndContinue, retain); + : this._cacheIsolation!.isolate(record.root, evaluateAndContinue, retain); completed = true; } finally { diagnostics.push(...sink.fetchAndClear()); diff --git a/packages/pyright-internal/src/analyzer/typeCacheUtils.ts b/packages/pyright-internal/src/analyzer/typeCacheUtils.ts index 85ce1f006bc5..50a043875d0f 100644 --- a/packages/pyright-internal/src/analyzer/typeCacheUtils.ts +++ b/packages/pyright-internal/src/analyzer/typeCacheUtils.ts @@ -119,8 +119,9 @@ export function addContextualTypeCacheEntry( return newCacheEntries; } -// Exact-node experiment support. Unrelated cache entries are never cleared. -export function useTestOnlyCacheIsolation( +// Isolate exactly these node IDs for a synchronous operation. Restore entry presence +// and values on exit unless successful results are retained; unrelated writes survive. +export function useNodeCacheIsolation( cache: Map, nodeIds: ReadonlySet, callback: () => T, @@ -162,12 +163,12 @@ export class SpeculativeTypeTracker { private _speculativeTypeCache = new Map(); private _activeDependentTypes: DependentType[] = []; - testOnlyEvict(nodeIds: ReadonlySet) { + evictCacheEntries(nodeIds: ReadonlySet) { assert(this._speculativeContextStack.length === 0); nodeIds.forEach((id) => this._speculativeTypeCache.delete(id)); } - canUseTestOnlyCacheIsolation(root: ParseNode) { + canUseNodeCacheIsolation(root: ParseNode) { return this._speculativeContextStack.every( (context) => !ParseTreeUtils.isNodeContainedWithin(root, context.speculativeRootNode) && @@ -175,9 +176,9 @@ export class SpeculativeTypeTracker { ); } - useTestOnlyCacheIsolation(nodeIds: ReadonlySet, callback: () => T, root?: ParseNode): T { - assert(root ? this.canUseTestOnlyCacheIsolation(root) : this._speculativeContextStack.length === 0); - return useTestOnlyCacheIsolation(this._speculativeTypeCache, nodeIds, callback); + useNodeCacheIsolation(nodeIds: ReadonlySet, callback: () => T, root?: ParseNode): T { + assert(root ? this.canUseNodeCacheIsolation(root) : this._speculativeContextStack.length === 0); + return useNodeCacheIsolation(this._speculativeTypeCache, nodeIds, callback); } enterSpeculativeContext(speculativeRootNode: ParseNode, options?: SpeculativeModeOptions) { diff --git a/packages/pyright-internal/src/analyzer/typeEvaluator.ts b/packages/pyright-internal/src/analyzer/typeEvaluator.ts index cf44979af4bb..d7ffeedf9030 100644 --- a/packages/pyright-internal/src/analyzer/typeEvaluator.ts +++ b/packages/pyright-internal/src/analyzer/typeEvaluator.ts @@ -186,7 +186,7 @@ import { findContextualTypeCacheEntry, SpeculativeModeOptions, SpeculativeTypeTracker, - useTestOnlyCacheIsolation, + useNodeCacheIsolation, } from './typeCacheUtils'; import { assignToTypedDict, @@ -215,6 +215,7 @@ import { EffectiveTypeResult, ensureExpectedTypeCandidates, EvalFlags, + EvaluationOperationController, EvaluatorUsage, ExpectedTypeOptions, ExpectedTypeResult, @@ -632,21 +633,9 @@ export interface EvaluatorOptions { nodes: ReadonlySet; install: (getMember: (node: MemberAccessNode, baseTypeResult: TypeResult) => TypeResult) => void; }; - testOnlyExpression?: { - roots: ReadonlyMap>; - install: (isolate: (root: ParseNode, callback: () => TypeResult, retain?: boolean) => TypeResult) => void; - route?: ( - request: - | { kind: 'query'; node: ExpressionNode } - | { kind: 'expression'; node: ExpressionNode; flags: EvalFlags; context?: InferenceContext } - | { kind: 'statement'; node: AssignmentNode; evaluate: () => void } - ) => { result?: TypeResult } | undefined; + // Test-only injection uses the same production cache-ownership contract. + testOnlyExpression?: EvaluationOperationController & { query?: (node: ExpressionNode) => TypeResult | undefined; - dispose?: () => void; - installEviction?: (evict: (root: ParseNode) => void) => void; - dispatch: (node: ExpressionNode, flags: EvalFlags, context?: InferenceContext) => TypeResult | undefined; - project: (node: ExpressionNode, result: TypeResult) => TypeResult; - beforeCall: (node: CallNode) => void; }; } @@ -695,7 +684,8 @@ export function createTypeEvaluator( ): TypeEvaluator { const nodeInfo = AnalyzerNodeInfo.createAnalyzerNodeInfoAccessor(evaluatorOptions.nodeInfoReader); const maxCodeComplexity = evaluatorOptions.maxCodeComplexity; - const operationController = evaluatorOptions.experimentalOverloadResult ?? evaluatorOptions.testOnlyExpression; + const operationController: EvaluationOperationController | undefined = + evaluatorOptions.experimentalOverloadResult ?? evaluatorOptions.testOnlyExpression; let operationRouter = evaluatorOptions.experimentalOverloadResult?.isAutomatic ? undefined : operationController; const symbolResolutionStack: SymbolResolutionStackEntry[] = []; const speculativeTypeTracker = new SpeculativeTypeTracker(); @@ -30500,42 +30490,44 @@ export function createTypeEvaluator( }); if (operationController) { - const { roots, install } = operationController; - operationController.installEviction?.((root) => { - const ids = roots.get(root); - assert(ids && ids.size <= 256); - assert(!returnTypeInferenceTypeCache && !isSpeculativeModeInUse(undefined)); - ids.forEach((id) => { - typeCache.delete(id); - expectedTypeCache.delete(id); - typeFormTypeCache.delete(id); - }); - speculativeTypeTracker.testOnlyEvict(ids); - }); - install((root, callback, retain = false) => { - const ids = roots.get(root); - assert(ids && ids.size <= 256); - // Native argument speculation can query earlier, disjoint operations - // through code flow. Never isolate a root that overlaps that context. - assert(!returnTypeInferenceTypeCache && speculativeTypeTracker.canUseTestOnlyCacheIsolation(root)); - return useTestOnlyCacheIsolation( - typeCache, - ids, - () => - useTestOnlyCacheIsolation( - expectedTypeCache, - ids, - () => - useTestOnlyCacheIsolation( - typeFormTypeCache, - ids, - () => speculativeTypeTracker.useTestOnlyCacheIsolation(ids, callback, root), - retain - ), - retain - ), - retain - ); + const { roots } = operationController; + operationController.installCacheIsolation({ + evict(root) { + const ids = roots.get(root); + assert(ids && ids.size <= 256); + assert(!returnTypeInferenceTypeCache && !isSpeculativeModeInUse(undefined)); + ids.forEach((id) => { + typeCache.delete(id); + expectedTypeCache.delete(id); + typeFormTypeCache.delete(id); + }); + speculativeTypeTracker.evictCacheEntries(ids); + }, + isolate(root, callback, retain = false) { + const ids = roots.get(root); + assert(ids && ids.size <= 256); + // Native argument speculation can query earlier, disjoint operations + // through code flow. Never isolate a root that overlaps that context. + assert(!returnTypeInferenceTypeCache && speculativeTypeTracker.canUseNodeCacheIsolation(root)); + return useNodeCacheIsolation( + typeCache, + ids, + () => + useNodeCacheIsolation( + expectedTypeCache, + ids, + () => + useNodeCacheIsolation( + typeFormTypeCache, + ids, + () => speculativeTypeTracker.useNodeCacheIsolation(ids, callback, root), + retain + ), + retain + ), + retain + ); + }, }); } diff --git a/packages/pyright-internal/src/analyzer/typeEvaluatorTypes.ts b/packages/pyright-internal/src/analyzer/typeEvaluatorTypes.ts index f13a986eeb48..236289229c06 100644 --- a/packages/pyright-internal/src/analyzer/typeEvaluatorTypes.ts +++ b/packages/pyright-internal/src/analyzer/typeEvaluatorTypes.ts @@ -16,6 +16,7 @@ import { TextRange } from '../common/textRange'; import { ArgCategory, ArgumentNode, + AssignmentNode, CallNode, CaseNode, ClassNode, @@ -290,6 +291,36 @@ export interface TypeResultWithNode extends TypeResult { node: ParseNode; } +export type EvaluationOperationRequest = + | { kind: 'query'; node: ExpressionNode } + | { kind: 'expression'; node: ExpressionNode; flags: EvalFlags; context?: InferenceContext } + | { kind: 'statement'; node: AssignmentNode; evaluate: () => void }; + +// Installed once by the owning evaluator; valid only during that evaluator's lifetime. +export interface EvaluationCacheIsolation { + // Isolate the registered root's runtime, expected, TypeForm, and speculative caches. + // Always restore on failure/cancellation. Retain successful non-speculative entries + // only when requested. Native speculative scopes must be disjoint; return inference + // is forbidden. Unowned entries and their diagnostics remain ordinary evaluator work. + // The controller owns diagnostic capture/publication, not this cache transaction. + isolate(root: ParseNode, callback: () => TypeResult, retain?: boolean): TypeResult; + + // Evict only the registered root's entries, outside speculation and return inference. + evict(root: ParseNode): void; +} + +// Shared production boundary between evaluator caches and whole-operation control. +export interface EvaluationOperationController { + // Each bounded root owns an exact, stable set of node IDs during an operation. + readonly roots: ReadonlyMap>; + installCacheIsolation(isolation: EvaluationCacheIsolation): void; + route?(request: EvaluationOperationRequest): { result?: TypeResult } | undefined; + dispatch(node: ExpressionNode, flags: EvalFlags, context?: InferenceContext): TypeResult | undefined; + project(node: ExpressionNode, result: TypeResult): TypeResult; + beforeCall(node: CallNode): void; + dispose?(): void; +} + // Describes deprecation details about a symbol accessed via a member // access expression, perhaps through a property or descriptor accessor // method. diff --git a/packages/pyright-internal/src/tests/testOnlyCacheTransaction.test.ts b/packages/pyright-internal/src/tests/typeCacheIsolation.test.ts similarity index 62% rename from packages/pyright-internal/src/tests/testOnlyCacheTransaction.test.ts rename to packages/pyright-internal/src/tests/typeCacheIsolation.test.ts index ce242be7c145..087e33118ada 100644 --- a/packages/pyright-internal/src/tests/testOnlyCacheTransaction.test.ts +++ b/packages/pyright-internal/src/tests/typeCacheIsolation.test.ts @@ -1,17 +1,39 @@ /* - * testOnlyCacheTransaction.test.ts + * typeCacheIsolation.test.ts * Copyright (c) Microsoft Corporation. * Licensed under the MIT license. * - * Tests for the opt-in exact-node experiment transaction. + * Tests for exact-node cache isolation and rollback. */ import * as assert from 'assert'; -import { useTestOnlyCacheIsolation } from '../analyzer/typeCacheUtils'; +import { useNodeCacheIsolation } from '../analyzer/typeCacheUtils'; import { OperationCanceledException } from '../common/cancellationUtils'; -test('TestOnlyCacheTransactionCommitsSelectedEntries', () => { +test('CacheIsolationRestoresSelectedEntriesAndPreservesUnownedWrites', () => { + const original = {}; + const replacement = {}; + const cache = new Map([ + [1, original], + [4, undefined], + ]); + const result = useNodeCacheIsolation(cache, new Set([1, 3, 4]), () => { + assert.ok(!cache.has(1) && !cache.has(3) && !cache.has(4)); + cache.set(1, replacement); + cache.set(2, replacement); + cache.set(3, undefined); + cache.set(4, replacement); + return replacement; + }); + assert.strictEqual(result, replacement); + assert.strictEqual(cache.get(1), original); + assert.strictEqual(cache.get(2), replacement); + assert.ok(!cache.has(3) && cache.has(4)); + assert.strictEqual(cache.get(4), undefined); +}); + +test('CacheIsolationCommitsSelectedEntries', () => { const original = {}; const unrelated = {}; const replacement = {}; @@ -20,7 +42,7 @@ test('TestOnlyCacheTransactionCommitsSelectedEntries', () => { [2, unrelated], [4, undefined], ]); - const result = useTestOnlyCacheIsolation( + const result = useNodeCacheIsolation( cache, new Set([1, 3, 4]), () => { @@ -38,7 +60,7 @@ test('TestOnlyCacheTransactionCommitsSelectedEntries', () => { assert.ok(cache.has(3) && !cache.has(4)); }); -test('TestOnlyCacheTransactionRollsBackFailures', () => { +test('CacheIsolationRollsBackFailures', () => { for (const error of [new Error('failure'), new OperationCanceledException()]) { const original = {}; const unrelated = {}; @@ -49,7 +71,7 @@ test('TestOnlyCacheTransactionRollsBackFailures', () => { ]); assert.throws( () => - useTestOnlyCacheIsolation( + useNodeCacheIsolation( cache, new Set([1, 3, 4]), () => { diff --git a/packages/pyright-internal/src/tests/typeCacheUtils.test.ts b/packages/pyright-internal/src/tests/typeCacheUtils.test.ts index de8b9036dd30..6fa4b048021a 100644 --- a/packages/pyright-internal/src/tests/typeCacheUtils.test.ts +++ b/packages/pyright-internal/src/tests/typeCacheUtils.test.ts @@ -43,21 +43,21 @@ test.each(['complete', 'error', 'cancel'] as const)( tracker.addSpeculativeType(secondValue, { type }, 0, undefined); const original = tracker.getSpeculativeType(secondValue, undefined); assert.ok(original); - assert.equal(tracker.canUseTestOnlyCacheIsolation(first), true); - assert.equal(tracker.canUseTestOnlyCacheIsolation(second), false); - assert.equal(tracker.canUseTestOnlyCacheIsolation(secondValue), false); - assert.equal(tracker.canUseTestOnlyCacheIsolation(module), false); - assert.throws(() => tracker.useTestOnlyCacheIsolation(new Set([secondValue.id]), () => {}, second)); + assert.equal(tracker.canUseNodeCacheIsolation(first), true); + assert.equal(tracker.canUseNodeCacheIsolation(second), false); + assert.equal(tracker.canUseNodeCacheIsolation(secondValue), false); + assert.equal(tracker.canUseNodeCacheIsolation(module), false); + assert.throws(() => tracker.useNodeCacheIsolation(new Set([secondValue.id]), () => {}, second)); const error = outcome === 'cancel' ? new OperationCanceledException() : new Error('trial interruption'); const isolated = () => - tracker.useTestOnlyCacheIsolation( + tracker.useNodeCacheIsolation( new Set([first.id, firstValue.id]), () => { assert.strictEqual(tracker.getSpeculativeType(secondValue, undefined), original); tracker.enterSpeculativeContext(firstValue); try { - assert.equal(tracker.canUseTestOnlyCacheIsolation(first), false); - assert.equal(tracker.canUseTestOnlyCacheIsolation(third), true); + assert.equal(tracker.canUseNodeCacheIsolation(first), false); + assert.equal(tracker.canUseNodeCacheIsolation(third), true); tracker.addSpeculativeType(firstValue, { type }, 0, undefined); if (outcome !== 'complete') { throw error; @@ -83,8 +83,8 @@ test.each(['complete', 'error', 'cancel'] as const)( assert.equal(cache.has(secondValue.id), false); assert.equal(tracker.isSpeculative(undefined), false); tracker.enterSpeculativeContext(second); - assert.equal(tracker.canUseTestOnlyCacheIsolation(secondValue), false); - assert.throws(() => tracker.useTestOnlyCacheIsolation(new Set([secondValue.id]), () => {}, secondValue)); + assert.equal(tracker.canUseNodeCacheIsolation(secondValue), false); + assert.throws(() => tracker.useNodeCacheIsolation(new Set([secondValue.id]), () => {}, secondValue)); tracker.leaveSpeculativeContext(); } ); diff --git a/packages/pyright-internal/src/tests/typeServer/typeServer.inProc.test.ts b/packages/pyright-internal/src/tests/typeServer/typeServer.inProc.test.ts index 752c50ea9782..5ea74d43e753 100644 --- a/packages/pyright-internal/src/tests/typeServer/typeServer.inProc.test.ts +++ b/packages/pyright-internal/src/tests/typeServer/typeServer.inProc.test.ts @@ -17,7 +17,12 @@ import assert from 'assert'; +import { ClassType, isClass, OverloadResultType, TypeCategory } from '../../analyzer/types'; +import { isExpressionNode } from '../../parser/parseNodes'; +import { makeProgram } from '../../typeServer/programWrapper'; import { TypeServerProtocol } from '../../typeServer/protocol/typeServerProtocol'; +import { ProtocolTypeFactory } from '../../typeServer/typeServerConversionUtils'; +import { getNodeAtMarker, parseAndGetTestState } from '../harness/fourslash/testState'; import { initializeDependenciesForInProcTests, withInProcTypeServer } from './inProcTypeServerTestUtils'; jest.setTimeout(120000); @@ -34,6 +39,14 @@ function getClassTypeName(type: TypeServerProtocol.Type | undefined): string | u return undefined; } +function getListElement(type: TypeServerProtocol.Type): TypeServerProtocol.Type { + assert(type.kind === TypeServerProtocol.TypeKind.Class); + assert.strictEqual(getClassTypeName(type), 'list'); + assert.strictEqual(type.flags, TypeServerProtocol.TypeFlags.Instance | TypeServerProtocol.TypeFlags.Generic); + assert(type.typeArgs && type.typeArgs.length === 1); + return type.typeArgs[0]; +} + describe('TypeServer in-proc protocol', () => { beforeAll(async () => { await initializeDependenciesForInProcTests(); @@ -198,6 +211,97 @@ describe('TypeServer in-proc protocol', () => { }); }); + test('getComputedType serializes automatically retained overload alternatives', async () => { + const code = ` +// @filename: main.py +//// from typing import Any, overload +//// +//// @overload +//// def choose(value: list[int]) -> list[list[int]]: ... +//// @overload +//// def choose(value: list[str]) -> list[list[str]]: ... +//// def choose(value: Any) -> Any: ... +//// +//// def ambiguous(value: list[Any]): +//// [|/*ambiguous*/result|] = choose(value) +//// +//// def concrete(value: list[int]): +//// [|/*concrete*/result|] = choose(value) +`; + + await withInProcTypeServer(code, async (context) => { + await context.openFileForMarker('ambiguous'); + await context.refreshSnapshot(); + const type = await context.sendRequestWithSnapshot(TypeServerProtocol.GetComputedTypeRequest.type, { + arg: context.getNodeForMarker('ambiguous'), + }); + + assert(type?.kind === TypeServerProtocol.TypeKind.OverloadResult, JSON.stringify(type)); + assert.strictEqual(type.flags, TypeServerProtocol.TypeFlags.Instance); + assert.strictEqual(type.uncertaintyKind, 'any'); + assert.strictEqual(type.candidates.length, 2); + assert.deepStrictEqual( + type.candidates.map((candidate) => getClassTypeName(getListElement(getListElement(candidate)))), + ['int', 'str'] + ); + assert(type.baselineType.kind === TypeServerProtocol.TypeKind.TypeReference); + assert.strictEqual(type.baselineType.typeReferenceId, type.candidates[0].id); + + const concrete = await context.sendRequestWithSnapshot(TypeServerProtocol.GetComputedTypeRequest.type, { + arg: context.getNodeForMarker('concrete'), + }); + assert(concrete); + assert.strictEqual(getClassTypeName(getListElement(getListElement(concrete))), 'int'); + }); + }); + + test('ProtocolTypeFactory preserves nested unknown alternatives and shared references', () => { + const state = parseAndGetTestState(` +// @filename: main.py +//// def example(integers: list[int], strings: list[str]): +//// /*integers*/integers +//// /*strings*/strings +`).state; + + try { + state.program.analyze(); + const integersNode = getNodeAtMarker(state, 'integers'); + const stringsNode = getNodeAtMarker(state, 'strings'); + assert(isExpressionNode(integersNode) && isExpressionNode(stringsNode)); + const evaluator = state.program.evaluator; + assert(evaluator); + const integers = evaluator.getType(integersNode); + const strings = evaluator.getType(stringsNode); + assert(integers && isClass(integers) && strings && isClass(strings)); + + // Unknown ambiguity is transportable without widening automatic Any-only admission. + const carrier = OverloadResultType.create([integers, strings], integers, TypeCategory.Unknown); + const nested = ClassType.specialize(integers, [carrier]); + const factory = new ProtocolTypeFactory( + makeProgram(state.program), + state.configOptions.getDefaultExecEnvironment().pythonVersion, + integersNode + ); + const type = getListElement(factory.getType(nested)); + assert(type.kind === TypeServerProtocol.TypeKind.OverloadResult); + assert.strictEqual(type.flags, TypeServerProtocol.TypeFlags.Instance); + assert.strictEqual(type.uncertaintyKind, 'unknown'); + assert.strictEqual(type.candidates.length, 2); + assert.deepStrictEqual( + type.candidates.map((candidate) => getClassTypeName(getListElement(candidate))), + ['int', 'str'] + ); + assert(type.baselineType.kind === TypeServerProtocol.TypeKind.TypeReference); + assert.strictEqual(type.baselineType.typeReferenceId, type.candidates[0].id); + + const repeated = factory.getType(carrier); + assert(repeated.kind === TypeServerProtocol.TypeKind.TypeReference); + assert.strictEqual(repeated.typeReferenceId, type.id); + } finally { + state.dispose(); + } + }); + test('getExpectedType returns a type for a node', async () => { const code = ` // @filename: main.py diff --git a/packages/pyright-internal/src/trace/nativeAdapter.ts b/packages/pyright-internal/src/trace/nativeAdapter.ts index 2edabf146bb8..0d5e9cefd10e 100644 --- a/packages/pyright-internal/src/trace/nativeAdapter.ts +++ b/packages/pyright-internal/src/trace/nativeAdapter.ts @@ -147,11 +147,7 @@ export function instrument( if (experimental && file === 'analyzer/typeCacheUtils.js') { spec = { ...spec, - functions: [ - ...spec.functions, - 'useTestOnlyCacheIsolation', - 'SpeculativeTypeTracker.useTestOnlyCacheIsolation', - ], + functions: [...spec.functions, 'useNodeCacheIsolation', 'SpeculativeTypeTracker.useNodeCacheIsolation'], }; } const source = ts.createSourceFile(file, code, ts.ScriptTarget.ES2020, true, ts.ScriptKind.JS);