Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions docs/type-evaluator-architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,11 @@ and type guard tests using those primitives. Scalar string evaluation and
forward-annotation parsing belong to `expressionEvaluation.ts`, while callable
parameter packing is private to `typeCompatibility.ts`.

IntFlag callable return refinement runs in `typeCompatibility.ts` after parameter
compatibility succeeds. The main evaluator retains validated call-argument
refinement and correlated packed-argument expansion; its original argument list
is carried in the private `MatchArgsToParamsResult` contract in `internals.ts`.

## Where to start

This is a navigation map, not an acyclic dependency graph. Start in
Expand Down
538 changes: 532 additions & 6 deletions packages/pyright-internal/src/analyzer/enums.ts

Large diffs are not rendered by default.

136 changes: 132 additions & 4 deletions packages/pyright-internal/src/analyzer/operations.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,9 @@ import { getTupleComparisonResult, isTupleOrdering } from './tupleComparison';
import { EvalFlags, MagicMethodDeprecationInfo, TypeEvaluator, TypeResult } from './typeEvaluatorTypes';
import {
InferenceContext,
MemberAccessFlags,
convertToInstantiable,
getClassMemberIterator,
getLiteralTypeClassName,
getTypeCondition,
getUnionSubtypeCount,
Expand All @@ -51,6 +53,7 @@ import {
import {
ClassType,
NeverType,
OverloadedType,
Type,
TypeBase,
UnknownType,
Expand All @@ -60,9 +63,11 @@ import {
isAnyOrUnknown,
isClass,
isClassInstance,
isFunction,
isFunctionOrOverloaded,
isInstantiableClass,
isNever,
isOverloaded,
isUnion,
} from './types';

Expand Down Expand Up @@ -108,6 +113,56 @@ interface BinaryOperationOptions {
// cut off the literal union and fall back to the non-literal supertype.
const maxLiteralMathSubtypeCount = 64;

function shouldPrioritizeReflectedMethod(
evaluator: TypeEvaluator,
leftType: Type,
rightType: Type,
methodName: string
): boolean {
if (
!isClassInstance(leftType) ||
!isClassInstance(rightType) ||
// A synthesized narrowing intersection is not evidence of a runtime subclass.
!rightType.shared.declaration ||
ClassType.isSameGenericClass(leftType, rightType) ||
!rightType.shared.mro.some(
(base) =>
isInstantiableClass(base) && ClassType.isSameGenericClass(base, ClassType.cloneAsInstantiable(leftType))
)
) {
return false;
}

const rightMember = getClassMemberIterator(rightType, methodName, MemberAccessFlags.SkipInstanceMembers).next()
.value;
const leftMember = getClassMemberIterator(leftType, methodName, MemberAccessFlags.SkipInstanceMembers).next().value;
if (!rightMember || rightMember.symbol === leftMember?.symbol) {
return false;
}

const rightMethod = evaluator.getTypeOfMember(rightMember);
const leftMethod = leftMember ? evaluator.getTypeOfMember(leftMember) : undefined;
const rightFunction = isOverloaded(rightMethod) ? OverloadedType.getOverloads(rightMethod)[0] : rightMethod;
const leftFunction =
leftMethod && isOverloaded(leftMethod) ? OverloadedType.getOverloads(leftMethod)[0] : leftMethod;

// Compare unbound declarations, not specialized signatures: aliases and
// inherited generic methods can have different signatures but the same implementation.
return (
!!rightFunction &&
isFunction(rightFunction) &&
!!rightFunction.shared.declaration &&
(!leftFunction ||
(isFunction(leftFunction) && rightFunction.shared.declaration !== leftFunction.shared.declaration))
);
}

function removeNotImplemented(type: Type): Type {
return mapSubtypes(type, (subtype) =>
isClassInstance(subtype) && subtype.shared.fullName === 'types.NotImplementedType' ? undefined : subtype
);
}

export function validateBinaryOperation(
evaluator: TypeEvaluator,
operator: OperatorType,
Expand Down Expand Up @@ -1371,8 +1426,48 @@ function validateArithmeticOperation(
}
}

const magicMethodName = binaryOperatorMap[operator][0];
let resultTypeResult = evaluator.getTypeOfMagicMethodCall(
const [magicMethodName, altMagicMethodName] = binaryOperatorMap[operator];
let resultTypeResult: TypeResult | undefined;
let decliningReflectedResult: TypeResult | undefined;
let skipReflectedFallback = false;

// Rich comparisons have different precedence rules. Leave their
// dispatch, and all ordinary forward-first arithmetic, unchanged.
if (
altMagicMethodName.startsWith('__r') &&
shouldPrioritizeReflectedMethod(
evaluator,
leftSubtypeExpanded,
rightSubtypeExpanded,
altMagicMethodName
)
) {
const reflectedResult = evaluator.getTypeOfMagicMethodCall(
rightSubtypeUnexpanded,
altMagicMethodName,
[{ type: leftSubtypeUnexpanded, isIncomplete: leftTypeResult.isIncomplete }],
errorNode,
inferenceContext
);
if (reflectedResult) {
const acceptedType = removeNotImplemented(reflectedResult.type);
if (acceptedType === reflectedResult.type) {
resultTypeResult = reflectedResult;
} else if (
reflectedResult.overloadsUsedForCall?.length &&
reflectedResult.overloadsUsedForCall.every(
(overload) => overload.shared.declaredReturnType !== undefined
)
) {
// Only an explicit return contract establishes a decline;
// an inherited method's inferred sentinel may be contextual.
decliningReflectedResult = { ...reflectedResult, type: acceptedType };
skipReflectedFallback = true;
}
}
}

resultTypeResult ??= evaluator.getTypeOfMagicMethodCall(
convertFunctionToObject(evaluator, leftSubtypeUnexpanded),
magicMethodName,
[{ type: rightSubtypeUnexpanded, isIncomplete: rightTypeResult.isIncomplete }],
Expand Down Expand Up @@ -1402,9 +1497,28 @@ function validateArithmeticOperation(
);
}

if (!resultTypeResult) {
if (skipReflectedFallback && resultTypeResult) {
const acceptedType = removeNotImplemented(resultTypeResult.type);
if (
acceptedType !== resultTypeResult.type &&
!(
resultTypeResult.overloadsUsedForCall?.length &&
resultTypeResult.overloadsUsedForCall.every(
(overload) => overload.shared.declaredReturnType !== undefined
)
)
) {
decliningReflectedResult = undefined;
} else {
resultTypeResult =
acceptedType !== resultTypeResult.type && isNever(acceptedType)
? undefined
: { ...resultTypeResult, type: acceptedType };
}
}

if (!resultTypeResult && !skipReflectedFallback) {
// Try the alternate form (swapping right and left).
const altMagicMethodName = binaryOperatorMap[operator][1];
resultTypeResult = evaluator.getTypeOfMagicMethodCall(
convertFunctionToObject(evaluator, rightSubtypeUnexpanded),
altMagicMethodName,
Expand Down Expand Up @@ -1441,6 +1555,20 @@ function validateArithmeticOperation(
}
}

if (decliningReflectedResult && (resultTypeResult || !isNever(decliningReflectedResult.type))) {
resultTypeResult = {
...decliningReflectedResult,
type: combineTypes([
decliningReflectedResult.type,
resultTypeResult?.type ?? NeverType.createNever(),
]),
isIncomplete: decliningReflectedResult.isIncomplete || resultTypeResult?.isIncomplete,
magicMethodDeprecationInfo:
decliningReflectedResult.magicMethodDeprecationInfo ??
resultTypeResult?.magicMethodDeprecationInfo,
};
}

if (!resultTypeResult) {
if (inferenceContext && !isAnyOrUnknown(inferenceContext.expectedType)) {
diag.addMessage(
Expand Down
106 changes: 105 additions & 1 deletion packages/pyright-internal/src/analyzer/typeEvaluator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -135,12 +135,16 @@ import {
getFunctionInfoFromDecorators,
} from './decorators';
import {
applyIntFlagTransform,
createEnumType,
getEnumAutoValueType,
getTypeOfEnumMember,
getTypeOfIntFlagMember,
isDeclInEnumClass,
isEnumClassWithMembers,
isEnumMetaclass,
isIntFlagBitwiseFunction,
synthesizeEnumBoundary,
} from './enums';
import { applyFunctionTransform } from './functionTransform';
import { createNamedTupleType } from './namedTuples';
Expand Down Expand Up @@ -202,6 +206,7 @@ import {
EvaluatorUsage,
ExpectedTypeOptions,
ExpectedTypeResult,
FunctionResult,
FunctionTypeResult,
MapSubtypesOptions,
MemberAccessDeprecationInfo,
Expand Down Expand Up @@ -1893,7 +1898,10 @@ export function createTypeEvaluator(
}

return {
type: memberInfo.type,
type:
usage.method === 'get'
? getTypeOfIntFlagMember(evaluatorInterface, objectType, memberInfo.type, nodeInfo)
: memberInfo.type,
classType: memberInfo.classType,
isIncomplete: !!memberInfo.isTypeIncomplete,
isAsymmetricAccessor: memberInfo.isAsymmetricAccessor,
Expand Down Expand Up @@ -8509,6 +8517,8 @@ export function createTypeEvaluator(
overloadIndex: number
): MatchArgsToParamsResult {
const overload = typeResult.type;
const unexpandedArgList =
isIntFlagBitwiseFunction(overload) && !overload.priv.strippedFirstParamType ? argList : undefined;
const paramDetails = getParamListDetails(overload, { disallowExtraKwargsForTd: true });
const paramSpec = FunctionType.getParamSpecFromArgsKwargs(overload);

Expand Down Expand Up @@ -9537,6 +9547,7 @@ export function createTypeEvaluator(
argumentErrors: reportedArgError,
isTypeIncomplete,
argParams: validateArgTypeParams,
unexpandedArgList,
paramSpecTarget,
paramSpecArgList,
activeParam,
Expand Down Expand Up @@ -9991,6 +10002,16 @@ export function createTypeEvaluator(

matchResults.argumentMatchScore = argumentMatchScore;

if (!argumentErrors && isIntFlagBitwiseFunction(type)) {
const transformed = applyIntFlagCallTransform(errorNode, matchResults, argResults, {
argumentErrors,
returnType: specializedReturnType,
isTypeIncomplete,
});
specializedReturnType = transformed.returnType;
isTypeIncomplete = transformed.isTypeIncomplete;
}

return {
argumentErrors,
argResults,
Expand All @@ -10003,6 +10024,88 @@ export function createTypeEvaluator(
};
}

function applyIntFlagCallTransform(
errorNode: ExpressionNode,
matchResults: MatchArgsToParamsResult,
argResults: ArgResult[],
result: FunctionResult
): FunctionResult {
const argumentTypes = new Map<string, Type>();
matchResults.argParams.forEach((param, index) => {
if (param.paramName) {
argumentTypes.set(param.paramName, argResults[index].argType);
}
});
result = applyIntFlagTransform(evaluatorInterface, matchResults.overload, argumentTypes, result);

const argList = matchResults.unexpandedArgList;
if (!argList?.some((arg) => arg.argCategory !== ArgCategory.Simple)) {
return result;
}

// Keep correlated packed alternatives together rather than treating their
// independently combined parameter types as a Cartesian product.
const contextFreeArgTypes = argList.map((arg) =>
arg.argCategory === ArgCategory.Simple
? UnknownType.create()
: getTypeOfArg(arg, /* inferenceContext */ undefined).type
);
let expandedArgTypes: (Type | undefined)[][] = [argList.map(() => undefined)];
while (true) {
const expanded = expressionEvaluation.expandArgTypes(contextFreeArgTypes, expandedArgTypes);
if (!expanded) {
break;
}
if (expanded.length > maxTotalOverloadArgTypeExpansionCount) {
return result;
}
expandedArgTypes = expanded;
}
if (expandedArgTypes.length === 1) {
return result;
}

const returnTypes: Type[] = [];
let isTypeIncomplete = result.isTypeIncomplete;
for (const expandedTypes of expandedArgTypes) {
const expandedArgs = argList.map((arg, index): Arg => {
const type = expandedTypes[index];
return type
? {
...arg,
valueExpression: undefined,
typeResult: { type, isIncomplete: result.isTypeIncomplete },
}
: arg;
});
const expandedMatch = useSpeculativeMode(getSpeculativeNodeForCall(errorNode), () =>
matchArgsToParams(
errorNode,
expandedArgs,
{ type: matchResults.overload, isIncomplete: result.isTypeIncomplete },
matchResults.overloadIndex
)
);
isTypeIncomplete ||= expandedMatch.isTypeIncomplete;
if (expandedMatch.argumentErrors) {
return { ...result, isTypeIncomplete };
}

const expandedArgumentTypes = new Map(argumentTypes);
expandedMatch.argParams.forEach((param) => {
const type = param.argType ?? param.argument.typeResult?.type;
if (param.paramName && type) {
expandedArgumentTypes.set(param.paramName, type);
}
});
returnTypes.push(
applyIntFlagTransform(evaluatorInterface, matchResults.overload, expandedArgumentTypes, result)
.returnType
);
}
return { ...result, returnType: combineTypes(returnTypes), isTypeIncomplete };
}

// In general, all in-scope type variables left in a return type should be
// replaced with Unknown. However, if the return type is a callable that uses
// type vars that are found nowhere within the function's input parameters,
Expand Down Expand Up @@ -12788,6 +12891,7 @@ export function createTypeEvaluator(

// Clear the "partially constructed" flag.
classType.shared.flags &= ~ClassTypeFlags.PartiallyEvaluated;
synthesizeEnumBoundary(evaluatorInterface, classType, initSubclassArgs);

// Now determine the decorated type of the class.
let decoratedType: Type = classType;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ export interface MatchArgsToParamsResult {
argumentErrors: boolean;
isTypeIncomplete: boolean;
argParams: ValidateArgTypeParams[];
unexpandedArgList?: Arg[] | undefined;
activeParam?: FunctionParam | undefined;
paramSpecTarget?: ParamSpecType | undefined;
paramSpecArgList?: Arg[] | undefined;
Expand Down
Loading
Loading