From 3245fd7c1083617d145e8656c54e7b1eeef59d15 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Wed, 26 Aug 2026 12:23:23 +0100 Subject: [PATCH 01/15] gh-124697: Represent inlined comprehensions as subscopes in the symbol table --- Doc/library/symtable.rst | 8 + Include/internal/pycore_compile.h | 19 +- Include/internal/pycore_symtable.h | 6 +- Lib/symtable.py | 3 + Lib/test/test_compiler_assemble.py | 3 +- Lib/test/test_symtable.py | 62 +++++- Modules/_testinternalcapi.c | 5 +- Modules/symtablemodule.c | 2 + Python/assemble.c | 12 +- Python/codegen.c | 51 +++-- Python/compile.c | 161 +++------------- Python/symtable.c | 294 ++++++++++++++++++++--------- 12 files changed, 359 insertions(+), 267 deletions(-) diff --git a/Doc/library/symtable.rst b/Doc/library/symtable.rst index 859687340882de..a9dbaae9d4bf2f 100644 --- a/Doc/library/symtable.rst +++ b/Doc/library/symtable.rst @@ -57,6 +57,14 @@ Examining Symbol Tables Used for the symbol table of a class. + .. attribute:: INLINED_COMPREHENSION + :value: "inlined comprehension" + + Used for the symbol table of a list, set or dict comprehension that + is inlined into the enclosing code unit (see :pep:`709`). A symbol + table of this type represents a sub-scope of the enclosing code unit's + scope, and it does not correspond to a separate compilation unit. + The following members refer to different flavors of :ref:`annotation scopes `. diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index 7e248429af8eb8..bf94fc8db5e545 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -69,9 +69,8 @@ typedef struct { PyObject *u_varnames; /* local variables */ PyObject *u_cellvars; /* cell variables */ PyObject *u_freevars; /* free variables */ - PyObject *u_fasthidden; /* dict; keys are names that are fast-locals only - temporarily within an inlined comprehension. When - value is True, treat as fast-local. */ + PyObject *u_fasthidden; /* set of names that are fast-locals only + temporarily within an inlined comprehension. */ Py_ssize_t u_argcount; /* number of arguments for block */ Py_ssize_t u_posonlyargcount; /* number of positional only arguments for block */ @@ -155,7 +154,6 @@ int _PyCompile_ResolveNameop(struct _PyCompiler *c, PyObject *mangled, int scope _PyCompile_optype *optype, Py_ssize_t *arg); int _PyCompile_IsInteractiveTopLevel(struct _PyCompiler *c); -int _PyCompile_IsInInlinedComp(struct _PyCompiler *c); int _PyCompile_ScopeType(struct _PyCompiler *c); int _PyCompile_OptimizationLevel(struct _PyCompiler *c); int _PyCompile_LookupArg(struct _PyCompiler *c, PyCodeObject *co, PyObject *name); @@ -179,16 +177,15 @@ enum { typedef struct { PyObject *pushed_locals; - PyObject *temp_symbols; - PyObject *fast_hidden; _PyJumpTargetLabel cleanup; + PySTEntryObject *saved_ste; } _PyCompile_InlinedComprehensionState; -int _PyCompile_TweakInlinedComprehensionScopes(struct _PyCompiler *c, _Py_SourceLocation loc, - PySTEntryObject *entry, - _PyCompile_InlinedComprehensionState *state); -int _PyCompile_RevertInlinedComprehensionScopes(struct _PyCompiler *c, _Py_SourceLocation loc, - _PyCompile_InlinedComprehensionState *state); +int _PyCompile_EnterInlinedComprehensionScope(struct _PyCompiler *c, _Py_SourceLocation loc, + PySTEntryObject *entry, + _PyCompile_InlinedComprehensionState *state); +int _PyCompile_ExitInlinedComprehensionScope(struct _PyCompiler *c, _Py_SourceLocation loc, + _PyCompile_InlinedComprehensionState *state); int _PyCompile_AddDeferredAnnotation(struct _PyCompiler *c, stmt_ty s, PyObject **conditional_annotation_index); void _PyCompile_EnterConditionalBlock(struct _PyCompiler *c); diff --git a/Include/internal/pycore_symtable.h b/Include/internal/pycore_symtable.h index c650a94a1eab2e..db609243f2f41e 100644 --- a/Include/internal/pycore_symtable.h +++ b/Include/internal/pycore_symtable.h @@ -33,6 +33,10 @@ typedef enum _block_type { // i.e., a TypeVar, a TypeVarTuple or a ParamSpec object (the latter two // do not support a bound or a constraint tuple). TypeVariableBlock, + // Comprehension which is inlined into the enclosing code unit (see PEP 709). + // Represents a sub-scope of the enclosing code unit's scope rather than a + // separate scope. + InlinedComprehensionBlock, } _Py_block_ty; typedef enum _comprehension_type { @@ -119,7 +123,6 @@ typedef struct _symtable_entry { should be created */ unsigned ste_needs_classdict : 1; /* for class scopes, true if a closure over the class dict should be created */ - unsigned ste_comp_inlined : 1; /* true if this comprehension is inlined */ unsigned ste_comp_iter_target : 1; /* true if visiting comprehension target */ unsigned ste_can_see_class_scope : 1; /* true if this block can see names bound in an enclosing class scope */ @@ -132,6 +135,7 @@ typedef struct _symtable_entry { int ste_comp_iter_expr; /* non-zero if visiting a comprehension range expression */ _Py_SourceLocation ste_loc; /* source location of block */ struct _symtable_entry *ste_annotation_block; /* symbol table entry for this entry's annotations */ + struct _symtable_entry *ste_parent; /* st entry for the enclosing block if this entry is a sub-scope, NULL otherwise */ struct symtable *ste_table; } PySTEntryObject; diff --git a/Lib/symtable.py b/Lib/symtable.py index 18bb355d86b09e..3d9c4f6b6ea983 100644 --- a/Lib/symtable.py +++ b/Lib/symtable.py @@ -56,6 +56,7 @@ class SymbolTableType(StrEnum): TYPE_ALIAS = "type alias" TYPE_PARAMETERS = "type parameters" TYPE_VARIABLE = "type variable" + INLINED_COMPREHENSION = "inlined comprehension" class SymbolTable: @@ -98,6 +99,8 @@ def get_type(self): return SymbolTableType.TYPE_PARAMETERS if self._table.type == _symtable.TYPE_TYPE_VARIABLE: return SymbolTableType.TYPE_VARIABLE + if self._table.type == _symtable.TYPE_INLINED_COMPREHENSION: + return SymbolTableType.INLINED_COMPREHENSION assert False, f"unexpected type: {self._table.type}" def get_id(self): diff --git a/Lib/test/test_compiler_assemble.py b/Lib/test/test_compiler_assemble.py index 99a11e99d56485..6e04df99b453ec 100644 --- a/Lib/test/test_compiler_assemble.py +++ b/Lib/test/test_compiler_assemble.py @@ -17,8 +17,9 @@ def complete_metadata(self, metadata, filename="myfile.py"): metadata.setdefault(key, key) for key in ['consts']: metadata.setdefault(key, []) - for key in ['names', 'varnames', 'cellvars', 'freevars', 'fasthidden']: + for key in ['names', 'varnames', 'cellvars', 'freevars']: metadata.setdefault(key, {}) + metadata.setdefault('fasthidden', None) for key in ['argcount', 'posonlyargcount', 'kwonlyargcount']: metadata.setdefault(key, 0) metadata.setdefault('firstlineno', 1) diff --git a/Lib/test/test_symtable.py b/Lib/test/test_symtable.py index ce02b27c599c42..5be898abd2d169 100644 --- a/Lib/test/test_symtable.py +++ b/Lib/test/test_symtable.py @@ -426,11 +426,11 @@ def test_symbol_repr(self): "") st1 = symtable.symtable("[x for x in [1]]", "?", "exec") - self.assertEqual(repr(st1.lookup("x")), + self.assertEqual(repr(st1.get_children()[0].lookup("x")), "") st2 = symtable.symtable("[(lambda: x) for x in [1]]", "?", "exec") - self.assertEqual(repr(st2.lookup("x")), + self.assertEqual(repr(st2.get_children()[0].lookup("x")), "") st3 = symtable.symtable("def f():\n" @@ -502,6 +502,64 @@ def test_nested_genexpr(self): self.assertEqual(sorted(st.get_identifiers()), [".0", "y"]) self.assertEqual(st.get_children(), []) + def test_inlined_comprehension_in_genexpr(self): + st = symtable.symtable("([y for y in x] for x in a)", "?", "exec") + self.assertEqual(len(st.get_children()), 1) + st = st.get_children()[0] + self.assertIs(st.get_type(), symtable.SymbolTableType.FUNCTION) + self.assertEqual(st.get_name(), "") + self.assertFalse(st.is_nested()) + self.assertEqual(sorted(st.get_identifiers()), [".0", "x"]) + children = st.get_children() + self.assertEqual(len(children), 1) + self.check_inlined_listcomp(children[0], ["y"], nested=True) + + def check_inlined_listcomp(self, st, identifiers, *, nested, nchildren=0): + self.assertIs(st.get_type(), symtable.SymbolTableType.INLINED_COMPREHENSION) + self.assertEqual(st.get_name(), "") + self.assertEqual(st.is_nested(), nested) + self.assertEqual(sorted(st.get_identifiers()), identifiers) + children = st.get_children() + self.assertEqual(len(children), nchildren) + return children + + def check_nested_inlined_listcomp(self, outer, hoisted, outer_ids, inner_ids, *, nested): + # Nested namespaces of inlined comprehensions are also hoisted into + # the enclosing scope's children list. + inner, = self.check_inlined_listcomp( + outer, outer_ids, nested=nested, nchildren=1) + self.check_inlined_listcomp(inner, inner_ids, nested=True) + self.assertIs(hoisted, inner) + + def test_inlined_comprehension(self): + st = symtable.symtable("[x for x in [1]]", "?", "exec") + self.assertEqual(sorted(st.get_identifiers()), []) + children = st.get_children() + self.assertEqual(len(children), 1) + self.check_inlined_listcomp(children[0], ["x"], nested=False) + + def test_inlined_nested_comprehension(self): + st = symtable.symtable("[[y for y in x] for x in [1]]", "?", "exec") + self.assertEqual(sorted(st.get_identifiers()), []) + children = st.get_children() + self.assertEqual(len(children), 2) + self.check_nested_inlined_listcomp( + children[0], children[1], ["x"], ["y"], nested=False) + + def test_inlined_sibling_nested_comprehensions(self): + st = symtable.symtable( + "def f(): [[y for y in x] for x in [1]]; [[w for w in z] for z in [2]]", + "?", "exec") + f = find_block(st, "f") + self.assertIs(f.get_type(), symtable.SymbolTableType.FUNCTION) + self.assertEqual(sorted(f.get_identifiers()), []) + children = f.get_children() + self.assertEqual(len(children), 4) + self.check_nested_inlined_listcomp( + children[0], children[1], ["x"], ["y"], nested=True) + self.check_nested_inlined_listcomp( + children[2], children[3], ["z"], ["w"], nested=True) + def test__symtable_refleak(self): # Regression test for reference leak in PyUnicode_FSDecoder. # See https://github.com/python/cpython/issues/139748. diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 38e56ae7042098..beb00a8af6858b 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1357,13 +1357,16 @@ _testinternalcapi_assemble_code_object_impl(PyObject *module, umd.u_cellvars = PyDict_GetItemString(metadata, "cellvars"); umd.u_freevars = PyDict_GetItemString(metadata, "freevars"); umd.u_fasthidden = PyDict_GetItemString(metadata, "fasthidden"); + if (umd.u_fasthidden == Py_None) { + umd.u_fasthidden = NULL; + } assert(PyDict_Check(umd.u_consts)); assert(PyDict_Check(umd.u_names)); assert(PyDict_Check(umd.u_varnames)); assert(PyDict_Check(umd.u_cellvars)); assert(PyDict_Check(umd.u_freevars)); - assert(PyDict_Check(umd.u_fasthidden)); + assert(umd.u_fasthidden == NULL || PySet_Check(umd.u_fasthidden)); umd.u_argcount = get_nonnegative_int_from_dict(metadata, "argcount"); umd.u_posonlyargcount = get_nonnegative_int_from_dict(metadata, "posonlyargcount"); diff --git a/Modules/symtablemodule.c b/Modules/symtablemodule.c index 7e20b5c7173ae5..0ce5b73add9524 100644 --- a/Modules/symtablemodule.c +++ b/Modules/symtablemodule.c @@ -144,6 +144,8 @@ symtable_init_constants(PyObject *m) return -1; if (PyModule_AddIntConstant(m, "TYPE_TYPE_VARIABLE", TypeVariableBlock) < 0) return -1; + if (PyModule_AddIntConstant(m, "TYPE_INLINED_COMPREHENSION", InlinedComprehensionBlock) < 0) + return -1; if (PyModule_AddIntMacro(m, LOCAL) < 0) return -1; if (PyModule_AddIntMacro(m, GLOBAL_EXPLICIT) < 0) return -1; diff --git a/Python/assemble.c b/Python/assemble.c index 4bbebe30299906..486c4f83898cae 100644 --- a/Python/assemble.c +++ b/Python/assemble.c @@ -520,13 +520,15 @@ compute_localsplus_info(_PyCompile_CodeUnitMetadata *umd, int nlocalsplus, _PyLocals_Kind kind = CO_FAST_LOCAL | argvarkinds[i].kind; - int has_key = PyDict_Contains(umd->u_fasthidden, k); - RETURN_IF_ERROR(has_key); - if (has_key) { - kind |= CO_FAST_HIDDEN; + if (umd->u_fasthidden != NULL) { + int hidden = PySet_Contains(umd->u_fasthidden, k); + RETURN_IF_ERROR(hidden); + if (hidden) { + kind |= CO_FAST_HIDDEN; + } } - has_key = PyDict_Contains(umd->u_cellvars, k); + int has_key = PyDict_Contains(umd->u_cellvars, k); RETURN_IF_ERROR(has_key); if (has_key) { kind |= CO_FAST_CELL; diff --git a/Python/codegen.c b/Python/codegen.c index 79b84f13e629c7..e58bd5b97d3323 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -3342,7 +3342,7 @@ codegen_nameop(compiler *c, location loc, case COMPILE_OP_DEREF: switch (ctx) { case Load: - if (SYMTABLE_ENTRY(c)->ste_type == ClassBlock && !_PyCompile_IsInInlinedComp(c)) { + if (SYMTABLE_ENTRY(c)->ste_type == ClassBlock) { op = LOAD_FROM_DICT_OR_DEREF; // First load the locals if (codegen_addop_noarg(INSTR_SEQUENCE(c), LOAD_LOCALS, loc) < 0) { @@ -3395,8 +3395,9 @@ codegen_nameop(compiler *c, location loc, case COMPILE_OP_NAME: switch (ctx) { case Load: - op = (SYMTABLE_ENTRY(c)->ste_type == ClassBlock - && _PyCompile_IsInInlinedComp(c)) + /* LOAD_NAME in a class reads the class dict; inlined comps must not. */ + op = (SCOPE_TYPE(c) == COMPILE_SCOPE_CLASS + && SYMTABLE_ENTRY(c)->ste_type == InlinedComprehensionBlock) ? LOAD_GLOBAL : LOAD_NAME; break; @@ -4917,9 +4918,6 @@ codegen_push_inlined_comprehension_locals(compiler *c, location loc, PySTEntryObject *comp, _PyCompile_InlinedComprehensionState *state) { - int in_class_block = (SYMTABLE_ENTRY(c)->ste_type == ClassBlock) && - !_PyCompile_IsInInlinedComp(c); - PySTEntryObject *outer = SYMTABLE_ENTRY(c); // iterate over names bound in the comprehension and ensure we isolate // them from the outer scope as needed PyObject *k, *v; @@ -4930,11 +4928,7 @@ codegen_push_inlined_comprehension_locals(compiler *c, location loc, RETURN_IF_ERROR(symbol); long scope = SYMBOL_TO_SCOPE(symbol); - long outsymbol = _PyST_GetSymbol(outer, k); - RETURN_IF_ERROR(outsymbol); - long outsc = SYMBOL_TO_SCOPE(outsymbol); - - if ((symbol & DEF_LOCAL && !(symbol & DEF_NONLOCAL)) || in_class_block) { + if ((symbol & DEF_LOCAL) && !(symbol & DEF_NONLOCAL)) { // local names bound in comprehension must be isolated from // outer scope; push existing value (which may be NULL if // not defined) on stack @@ -4949,15 +4943,17 @@ codegen_push_inlined_comprehension_locals(compiler *c, location loc, // comprehension and restore the original one after ADDOP_NAME(c, loc, LOAD_FAST_AND_CLEAR, k, varnames); if (scope == CELL) { - if (outsc == FREE) { - ADDOP_NAME(c, loc, MAKE_CELL, k, freevars); - } else { - ADDOP_NAME(c, loc, MAKE_CELL, k, cellvars); - } + ADDOP_NAME(c, loc, MAKE_CELL, k, cellvars); } if (PyList_Append(state->pushed_locals, k) < 0) { return ERROR; } + if (METADATA(c)->u_fasthidden != NULL) { + /* For Module/Class scopes, assemble needs to set CO_FAST_HIDDEN on these names */ + if (PySet_Add(METADATA(c)->u_fasthidden, k) < 0) { + return ERROR; + } + } } } if (state->pushed_locals) { @@ -4986,7 +4982,7 @@ push_inlined_comprehension_state(compiler *c, location loc, _PyCompile_InlinedComprehensionState *state) { RETURN_IF_ERROR( - _PyCompile_TweakInlinedComprehensionScopes(c, loc, comp, state)); + _PyCompile_EnterInlinedComprehensionScope(c, loc, comp, state)); RETURN_IF_ERROR( codegen_push_inlined_comprehension_locals(c, loc, comp, state)); return SUCCESS; @@ -5044,7 +5040,7 @@ pop_inlined_comprehension_state(compiler *c, location loc, _PyCompile_InlinedComprehensionState *state) { RETURN_IF_ERROR(codegen_pop_inlined_comprehension_locals(c, loc, state)); - RETURN_IF_ERROR(_PyCompile_RevertInlinedComprehensionScopes(c, loc, state)); + RETURN_IF_ERROR(_PyCompile_ExitInlinedComprehensionScope(c, loc, state)); return SUCCESS; } @@ -5054,13 +5050,13 @@ codegen_comprehension(compiler *c, expr_ty e, int type, expr_ty val, bool avoid_creation) { PyCodeObject *co = NULL; - _PyCompile_InlinedComprehensionState inline_state = {NULL, NULL, NULL, NO_LABEL}; + _PyCompile_InlinedComprehensionState inline_state = {NULL, NO_LABEL, NULL}; comprehension_ty outermost; PySTEntryObject *entry = _PySymtable_Lookup(SYMTABLE(c), (void *)e); if (entry == NULL) { goto error; } - int is_inlined = entry->ste_comp_inlined; + int is_inlined = (entry->ste_type == InlinedComprehensionBlock); int is_async_comprehension = entry->ste_coroutine; location loc = LOC(e); @@ -5069,7 +5065,7 @@ codegen_comprehension(compiler *c, expr_ty e, int type, IterStackPosition iter_state; if (is_inlined) { VISIT(c, expr, outermost->iter); - if (push_inlined_comprehension_state(c, loc, entry, &inline_state)) { + if (push_inlined_comprehension_state(c, loc, entry, &inline_state) < 0) { goto error; } iter_state = ITERABLE_ON_STACK; @@ -5140,8 +5136,8 @@ codegen_comprehension(compiler *c, expr_ty e, int type, } if (is_inlined) { - if (pop_inlined_comprehension_state(c, loc, &inline_state)) { - goto error; + if (pop_inlined_comprehension_state(c, loc, &inline_state) < 0) { + goto error_in_scope; } return SUCCESS; } @@ -5181,15 +5177,18 @@ codegen_comprehension(compiler *c, expr_ty e, int type, return SUCCESS; error_in_scope: - if (!is_inlined) { + if (is_inlined) { + if (inline_state.saved_ste != NULL) { + pop_inlined_comprehension_state(c, loc, &inline_state); + } + } + else { _PyCompile_ExitScope(c); } error: Py_XDECREF(co); Py_XDECREF(entry); Py_XDECREF(inline_state.pushed_locals); - Py_XDECREF(inline_state.temp_symbols); - Py_XDECREF(inline_state.fast_hidden); return ERROR; } diff --git a/Python/compile.c b/Python/compile.c index f3852041bce69c..35ae41c6c76404 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -68,7 +68,6 @@ struct compiler_unit { instr_sequence *u_stashed_instr_sequence; /* temporarily stashed parent instruction sequence */ int u_nfblocks; - int u_in_inlined_comp; int u_in_conditional_block; _PyCompile_FBlockInfo u_fblock[CO_MAXBLOCKS]; @@ -670,14 +669,18 @@ _PyCompile_EnterScope(compiler *c, identifier name, int scope_type, return ERROR; } - u->u_metadata.u_fasthidden = PyDict_New(); - if (!u->u_metadata.u_fasthidden) { - compiler_unit_free(u); - return ERROR; + if (scope_type == COMPILE_SCOPE_MODULE || scope_type == COMPILE_SCOPE_CLASS) { + u->u_metadata.u_fasthidden = PySet_New(NULL); + if (!u->u_metadata.u_fasthidden) { + compiler_unit_free(u); + return ERROR; + } + } + else { + u->u_metadata.u_fasthidden = NULL; } u->u_nfblocks = 0; - u->u_in_inlined_comp = 0; u->u_metadata.u_firstlineno = lineno; u->u_metadata.u_consts = PyDict_New(); if (!u->u_metadata.u_consts) { @@ -1013,6 +1016,13 @@ _PyCompile_ResolveNameop(compiler *c, PyObject *mangled, int scope, PyObject *dict = c->u->u_metadata.u_names; *optype = COMPILE_OP_NAME; + PySTEntryObject *ste = c->u->u_ste; + assert(ste != NULL); + while (ste->ste_parent != NULL) { + assert(ste->ste_type == InlinedComprehensionBlock); + ste = ste->ste_parent; + } + assert(scope >= 0); switch (scope) { case FREE: @@ -1024,21 +1034,12 @@ _PyCompile_ResolveNameop(compiler *c, PyObject *mangled, int scope, *optype = COMPILE_OP_DEREF; break; case LOCAL: - if (_PyST_IsFunctionLike(c->u->u_ste)) { + if (_PyST_IsFunctionLike(ste) || c->u->u_ste->ste_type == InlinedComprehensionBlock) { *optype = COMPILE_OP_FAST; } - else { - PyObject *item; - RETURN_IF_ERROR(PyDict_GetItemRef(c->u->u_metadata.u_fasthidden, mangled, - &item)); - if (item == Py_True) { - *optype = COMPILE_OP_FAST; - } - Py_XDECREF(item); - } break; case GLOBAL_IMPLICIT: - if (_PyST_IsFunctionLike(c->u->u_ste)) { + if (_PyST_IsFunctionLike(ste)) { *optype = COMPILE_OP_GLOBAL; } break; @@ -1057,120 +1058,24 @@ _PyCompile_ResolveNameop(compiler *c, PyObject *mangled, int scope, } int -_PyCompile_TweakInlinedComprehensionScopes(compiler *c, location loc, - PySTEntryObject *entry, - _PyCompile_InlinedComprehensionState *state) +_PyCompile_EnterInlinedComprehensionScope(compiler *c, location loc, + PySTEntryObject *entry, + _PyCompile_InlinedComprehensionState *state) { - int in_class_block = (c->u->u_ste->ste_type == ClassBlock) && !c->u->u_in_inlined_comp; - c->u->u_in_inlined_comp++; - - PyObject *k, *v; - Py_ssize_t pos = 0; - while (PyDict_Next(entry->ste_symbols, &pos, &k, &v)) { - long symbol = PyLong_AsLong(v); - assert(symbol >= 0 || PyErr_Occurred()); - RETURN_IF_ERROR(symbol); - long scope = SYMBOL_TO_SCOPE(symbol); - - long outsymbol = _PyST_GetSymbol(c->u->u_ste, k); - RETURN_IF_ERROR(outsymbol); - long outsc = SYMBOL_TO_SCOPE(outsymbol); - - // If a name has different scope inside than outside the comprehension, - // we need to temporarily handle it with the right scope while - // compiling the comprehension. If it's free in the comprehension - // scope, no special handling; it should be handled the same as the - // enclosing scope. (If it's free in outer scope and cell in inner - // scope, we can't treat it as both cell and free in the same function, - // but treating it as free throughout is fine; it's *_DEREF - // either way.) - if ((scope != outsc && scope != FREE && !(scope == CELL && outsc == FREE)) - || in_class_block) { - if (state->temp_symbols == NULL) { - state->temp_symbols = PyDict_New(); - if (state->temp_symbols == NULL) { - return ERROR; - } - } - // update the symbol to the in-comprehension version and save - // the outer version; we'll restore it after running the - // comprehension - if (PyDict_SetItem(c->u->u_ste->ste_symbols, k, v) < 0) { - return ERROR; - } - PyObject *outv = PyLong_FromLong(outsymbol); - if (outv == NULL) { - return ERROR; - } - int res = PyDict_SetItem(state->temp_symbols, k, outv); - Py_DECREF(outv); - RETURN_IF_ERROR(res); - } - // locals handling for names bound in comprehension (DEF_LOCAL | - // DEF_NONLOCAL occurs in assignment expression to nonlocal) - if ((symbol & DEF_LOCAL && !(symbol & DEF_NONLOCAL)) || in_class_block) { - if (!_PyST_IsFunctionLike(c->u->u_ste)) { - // non-function scope: override this name to use fast locals - PyObject *orig; - if (PyDict_GetItemRef(c->u->u_metadata.u_fasthidden, k, &orig) < 0) { - return ERROR; - } - assert(orig == NULL || orig == Py_True || orig == Py_False); - if (orig != Py_True) { - if (PyDict_SetItem(c->u->u_metadata.u_fasthidden, k, Py_True) < 0) { - Py_XDECREF(orig); - return ERROR; - } - if (state->fast_hidden == NULL) { - state->fast_hidden = PySet_New(NULL); - if (state->fast_hidden == NULL) { - Py_XDECREF(orig); - return ERROR; - } - } - if (PySet_Add(state->fast_hidden, k) < 0) { - Py_XDECREF(orig); - return ERROR; - } - } - Py_XDECREF(orig); - } - } - } + assert(state->saved_ste == NULL); + state->saved_ste = c->u->u_ste; + c->u->u_ste = (PySTEntryObject *)Py_NewRef(entry); return SUCCESS; } int -_PyCompile_RevertInlinedComprehensionScopes(compiler *c, location loc, - _PyCompile_InlinedComprehensionState *state) +_PyCompile_ExitInlinedComprehensionScope(compiler *c, location loc, + _PyCompile_InlinedComprehensionState *state) { - c->u->u_in_inlined_comp--; - if (state->temp_symbols) { - PyObject *k, *v; - Py_ssize_t pos = 0; - while (PyDict_Next(state->temp_symbols, &pos, &k, &v)) { - if (PyDict_SetItem(c->u->u_ste->ste_symbols, k, v)) { - return ERROR; - } - } - Py_CLEAR(state->temp_symbols); - } - if (state->fast_hidden) { - while (PySet_Size(state->fast_hidden) > 0) { - PyObject *k = PySet_Pop(state->fast_hidden); - if (k == NULL) { - return ERROR; - } - // we set to False instead of clearing, so we can track which names - // were temporarily fast-locals and should use CO_FAST_HIDDEN - if (PyDict_SetItem(c->u->u_metadata.u_fasthidden, k, Py_False)) { - Py_DECREF(k); - return ERROR; - } - Py_DECREF(k); - } - Py_CLEAR(state->fast_hidden); - } + assert(state->saved_ste != NULL); + Py_DECREF(c->u->u_ste); + c->u->u_ste = state->saved_ste; + state->saved_ste = NULL; return SUCCESS; } @@ -1362,12 +1267,6 @@ _PyCompile_ScopeType(compiler *c) return c->u->u_scope_type; } -int -_PyCompile_IsInInlinedComp(compiler *c) -{ - return c->u->u_in_inlined_comp; -} - PyObject * _PyCompile_Qualname(compiler *c) { diff --git a/Python/symtable.c b/Python/symtable.c index 8da04b40e8ad14..f9576aae228726 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -5,8 +5,10 @@ #include "pycore_runtime.h" // _Py_ID() #include "pycore_symtable.h" // PySTEntryObject #include "pycore_unicodeobject.h" // _PyUnicode_EqualToASCIIString +#include "setobject.h" #include // offsetof() +#include // Set this to 1 to dump all symtables to stdout for debugging @@ -128,14 +130,15 @@ ste_new(struct symtable *st, identifier name, _Py_block_ty block, if (st->st_cur != NULL && (st->st_cur->ste_nested || - _PyST_IsFunctionLike(st->st_cur))) + _PyST_IsFunctionLike(st->st_cur) || + st->st_cur->ste_type == InlinedComprehensionBlock)) ste->ste_nested = 1; ste->ste_generator = 0; ste->ste_coroutine = 0; ste->ste_comprehension = NoComprehension; ste->ste_returns_value = 0; ste->ste_needs_class_closure = 0; - ste->ste_comp_inlined = 0; + ste->ste_parent = (block == InlinedComprehensionBlock) ? st->st_cur : NULL; ste->ste_comp_iter_target = 0; ste->ste_can_see_class_scope = 0; ste->ste_comp_iter_expr = 0; @@ -295,6 +298,7 @@ static void _dump_symtable(PySTEntryObject* ste, PyObject* prefix) case TypeVariableBlock: blocktype = "TypeVariableBlock"; break; case TypeAliasBlock: blocktype = "TypeAliasBlock"; break; case TypeParametersBlock: blocktype = "TypeParametersBlock"; break; + case InlinedComprehensionBlock: blocktype = "InlinedComprehensionBlock"; break; } const char *comptype = ""; switch (ste->ste_comprehension) { @@ -308,7 +312,7 @@ static void _dump_symtable(PySTEntryObject* ste, PyObject* prefix) ( "%U=== Symtable for %U ===\n" "%U%s%s\n" - "%U%s%s%s%s%s%s%s%s%s%s%s\n" + "%U%s%s%s%s%s%s%s%s%s%s\n" "%Ulineno: %d col_offset: %d\n" "%U--- Symbols ---\n" ), @@ -326,7 +330,6 @@ static void _dump_symtable(PySTEntryObject* ste, PyObject* prefix) ste->ste_returns_value ? " returns_value" : "", ste->ste_needs_class_closure ? " needs_class_closure" : "", ste->ste_needs_classdict ? " needs_classdict" : "", - ste->ste_comp_inlined ? " comp_inlined" : "", ste->ste_comp_iter_target ? " comp_iter_target" : "", ste->ste_can_see_class_scope ? " can_see_class_scope" : "", prefix, @@ -537,22 +540,26 @@ _PySymtable_LookupOptional(struct symtable *st, void *key, long _PyST_GetSymbol(PySTEntryObject *ste, PyObject *name) { - PyObject *v; - if (PyDict_GetItemRef(ste->ste_symbols, name, &v) < 0) { - return -1; - } - if (!v) { - return 0; - } - long symbol = PyLong_AsLong(v); - Py_DECREF(v); - if (symbol < 0) { - if (!PyErr_Occurred()) { - PyErr_SetString(PyExc_SystemError, "invalid symbol"); + while (ste != NULL) { + PyObject *v; + if (PyDict_GetItemRef(ste->ste_symbols, name, &v) < 0) { + return -1; } - return -1; + if (v != NULL) { + long symbol = PyLong_AsLong(v); + Py_DECREF(v); + if (symbol < 0) { + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_SystemError, "invalid symbol"); + } + return -1; + } + return symbol; + } + assert(ste->ste_parent == NULL || ste->ste_type == InlinedComprehensionBlock); + ste = ste->ste_parent; } - return symbol; + return 0; } int @@ -575,6 +582,13 @@ _PyST_IsFunctionLike(PySTEntryObject *ste) || ste->ste_type == TypeParametersBlock; } +/* True if this block binds locals that are visible to nested scopes */ +static int +ste_binds_locals_for_children(PySTEntryObject *ste) +{ + return _PyST_IsFunctionLike(ste) || ste->ste_type == InlinedComprehensionBlock; +} + static int error_at_directive(PySTEntryObject *ste, PyObject *name) { @@ -801,37 +815,102 @@ is_free_in_any_child(PySTEntryObject *entry, PyObject *key) return 0; } +/* True if name is FREE in the comprehension and bound in the enclosing class. + * Those names are kept in the compressed delta so lookup does not treat them + * as class locals. */ static int -inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, - PyObject *scopes, PyObject *comp_free, - PyObject *inlined_cells) +class_binds_free_name(PySTEntryObject *ste, PyObject *name, long comp_flags) { + if (SYMBOL_TO_SCOPE(comp_flags) != FREE) { + return 0; + } + if (ste->ste_type != ClassBlock) { + return 0; + } + PyObject *v = PyDict_GetItemWithError(ste->ste_symbols, name); + if (v == NULL) { + return PyErr_Occurred() ? -1 : 0; + } + long class_flags = PyLong_AsLong(v); + if (class_flags == -1 && PyErr_Occurred()) { + return -1; + } + if (class_flags & (DEF_LOCAL | DEF_GLOBAL | DEF_FREE_CLASS | DEF_TYPE_PARAM)) + { + return 1; + } + return 0; +} + +static PyObject * +get_freevar_names(PySTEntryObject *ste) +{ + PyObject *free = PySet_New(NULL); + if (free == NULL) { + return NULL; + } PyObject *k, *v; Py_ssize_t pos = 0; - int remove_dunder_class = 0; - int remove_dunder_classdict = 0; - int remove_dunder_cond_annotations = 0; + while (PyDict_Next(ste->ste_symbols, &pos, &k, &v)) { + long flags = PyLong_AsLong(v); + if (flags == -1 && PyErr_Occurred()) { + Py_DECREF(free); + return NULL; + } + if (SYMBOL_TO_SCOPE(flags) == FREE) { + if (PySet_Add(free, k) < 0) { + Py_DECREF(free); + return NULL; + } + } + } + return free; +} + +static int +finalize_inlined_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, + PyObject *comp_free, PyObject *outer_newfree, + PyObject *inlined_cells) +{ + PyObject *k, *v; + Py_ssize_t pos = 0; + PyObject *to_remove = NULL; + + assert(comp->ste_type == InlinedComprehensionBlock); + assert(comp->ste_parent != NULL); + + to_remove = PyList_New(0); + if (to_remove == NULL) { + return 0; + } while (PyDict_Next(comp->ste_symbols, &pos, &k, &v)) { - // skip comprehension parameter long comp_flags = PyLong_AsLong(v); if (comp_flags == -1 && PyErr_Occurred()) { - return 0; - } - if (comp_flags & DEF_PARAM) { - assert(_PyUnicode_EqualToASCIIString(k, ".0")); - continue; + goto error; } int scope = SYMBOL_TO_SCOPE(comp_flags); int only_flags = comp_flags & ((1 << SCOPE_OFFSET) - 1); if (scope == CELL || only_flags & DEF_COMP_CELL) { if (PySet_Add(inlined_cells, k) < 0) { - return 0; + goto error; + } + if (!(only_flags & DEF_COMP_CELL)) { + comp_flags |= DEF_COMP_CELL; + PyObject *newv = PyLong_FromLong(comp_flags); + if (newv == NULL) { + goto error; + } + if (PyDict_SetItem(comp->ste_symbols, k, newv) < 0) { + Py_DECREF(newv); + goto error; + } + Py_DECREF(newv); } } PyObject *existing = PyDict_GetItemWithError(ste->ste_symbols, k); if (existing == NULL && PyErr_Occurred()) { - return 0; + goto error; } // __class__, __classdict__ and __conditional_annotations__ are // not allowed to be free through a class scope (see @@ -840,71 +919,103 @@ inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, (_PyUnicode_EqualToASCIIString(k, "__class__") || _PyUnicode_EqualToASCIIString(k, "__classdict__") || _PyUnicode_EqualToASCIIString(k, "__conditional_annotations__"))) { - scope = GLOBAL_IMPLICIT; int child_needs_free = is_free_in_any_child(comp, k); if (child_needs_free < 0) { - return 0; + goto error; } if (!child_needs_free) { if (PySet_Discard(comp_free, k) < 0) { - return 0; + goto error; } } - if (_PyUnicode_EqualToASCIIString(k, "__class__")) { - remove_dunder_class = 1; - } - else if (_PyUnicode_EqualToASCIIString(k, "__conditional_annotations__")) { - remove_dunder_cond_annotations = 1; - } - else { - remove_dunder_classdict = 1; - } - } - if (!existing) { - // name does not exist in scope, copy from comprehension - assert(scope != FREE || PySet_Contains(comp_free, k) == 1); - PyObject *v_flags = PyLong_FromLong(only_flags); - if (v_flags == NULL) { - return 0; + long new_flags = only_flags | (GLOBAL_IMPLICIT << SCOPE_OFFSET); + PyObject *newv = PyLong_FromLong(new_flags); + if (newv == NULL) { + goto error; } - int ok = PyDict_SetItem(ste->ste_symbols, k, v_flags); - Py_DECREF(v_flags); - if (ok < 0) { - return 0; + if (PyDict_SetItem(comp->ste_symbols, k, newv) < 0) { + Py_DECREF(newv); + goto error; } - SET_SCOPE(scopes, k, scope); + Py_DECREF(newv); + continue; } - else { + if (existing) { long flags = PyLong_AsLong(existing); if (flags == -1 && PyErr_Occurred()) { - return 0; + goto error; } if ((flags & DEF_BOUND) && ste->ste_type != ClassBlock) { // free vars in comprehension that are locals in outer scope can // now simply be locals, unless they are free in comp children, - // or if the outer scope is a class block + // needed as cells by sibling nested scopes, or if the outer + // scope is a class block int ok = is_free_in_any_child(comp, k); if (ok < 0) { - return 0; + goto error; } if (!ok) { - if (PySet_Discard(comp_free, k) < 0) { - return 0; + int in_newfree = PySet_Contains(outer_newfree, k); + if (in_newfree < 0) { + goto error; + } + if (!in_newfree) { + if (PySet_Discard(comp_free, k) < 0) { + goto error; + } } } } } + else { + assert(scope != FREE || PySet_Contains(comp_free, k) == 1); + } + + /* keep bindings, globals, and class-bound frees in the delta; + drop other names (typically FREE uses) so lookup climbs to parent. */ + if ((comp_flags & DEF_LOCAL) && !(comp_flags & DEF_NONLOCAL)) { + continue; + } + if (scope == GLOBAL_IMPLICIT || scope == GLOBAL_EXPLICIT) { + continue; + } + int keep = class_binds_free_name(ste, k, comp_flags); + if (keep < 0) { + goto error; + } + if (!keep) { + if (PyList_Append(to_remove, k) < 0) { + goto error; + } + } } - if (remove_dunder_class && PyDict_DelItemString(comp->ste_symbols, "__class__") < 0) { - return 0; - } - if (remove_dunder_classdict && PyDict_DelItemString(comp->ste_symbols, "__classdict__") < 0) { - return 0; + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(to_remove); i++) { + PyObject *name = PyList_GET_ITEM(to_remove, i); + if (PyDict_DelItem(comp->ste_symbols, name) < 0) { + goto error; + } } - if (remove_dunder_cond_annotations && PyDict_DelItemString(comp->ste_symbols, "__conditional_annotations__") < 0) { - return 0; + Py_CLEAR(to_remove); + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(comp->ste_children); i++) { + PySTEntryObject *child = (PySTEntryObject *)PyList_GET_ITEM(comp->ste_children, i); + if (child->ste_type != InlinedComprehensionBlock) { + continue; + } + PyObject *child_free = get_freevar_names(child); + if (child_free == NULL) { + return 0; + } + int ok = finalize_inlined_comprehension(ste, child, child_free, + outer_newfree, inlined_cells); + Py_DECREF(child_free); + if (!ok) { + return 0; + } } return 1; +error: + Py_XDECREF(to_remove); + return 0; } #undef SET_SCOPE @@ -1210,7 +1321,7 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, /* Populate global and bound sets to be passed to children. */ if (ste->ste_type != ClassBlock) { /* Add function locals to bound set */ - if (_PyST_IsFunctionLike(ste)) { + if (ste_binds_locals_for_children(ste)) { temp = PyNumber_InPlaceOr(newbound, local); if (!temp) goto error; @@ -1262,24 +1373,18 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, } } - // we inline all non-generator-expression comprehensions, - // except those in annotation scopes that are nested in classes - int inline_comp = - entry->ste_comprehension && - !entry->ste_generator && - !ste->ste_can_see_class_scope; - + // Compress InlinedComprehensionBlocks ste_symbols to a delta (bindings + + // class FREE overrides). Nested deltas are finalized recursively. if (!analyze_child_block(entry, newbound, newfree, newglobal, type_params, new_class_entry, &child_free)) { goto error; } - if (inline_comp) { - if (!inline_comprehension(ste, entry, scopes, child_free, inlined_cells)) { + if (entry->ste_type == InlinedComprehensionBlock && ste->ste_type != InlinedComprehensionBlock) { + if (!finalize_inlined_comprehension(ste, entry, child_free, newfree, inlined_cells)) { Py_DECREF(child_free); goto error; } - entry->ste_comp_inlined = 1; } temp = PyNumber_InPlaceOr(newfree, child_free); Py_DECREF(child_free); @@ -1294,8 +1399,9 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, PySTEntryObject* entry; assert(c && PySTEntry_Check(c)); entry = (PySTEntryObject*)c; - if (entry->ste_comp_inlined && - PyList_SetSlice(ste->ste_children, i, i + 1, + if (entry->ste_type == InlinedComprehensionBlock && + PyList_GET_SIZE(entry->ste_children) > 0 && + PyList_SetSlice(ste->ste_children, i+1, i + 1, entry->ste_children) < 0) { goto error; @@ -1303,10 +1409,12 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, } /* Check if any local variables must be converted to cell variables */ - if (_PyST_IsFunctionLike(ste) && !analyze_cells(scopes, newfree, inlined_cells)) + if (ste_binds_locals_for_children(ste) && !analyze_cells(scopes, newfree, inlined_cells)) { goto error; - else if (ste->ste_type == ClassBlock && !drop_class_free(ste, newfree)) + } + else if (ste->ste_type == ClassBlock && !drop_class_free(ste, newfree)) { goto error; + } /* Records the results of the analysis in the symbol table entry */ if (!update_symbols(ste->ste_symbols, scopes, bound, newfree, inlined_cells, (ste->ste_type == ClassBlock) || ste->ste_can_see_class_scope)) @@ -2582,7 +2690,7 @@ symtable_visit_expr(struct symtable *st, expr_ty e) return 0; } if (!allows_top_level_await(st)) { - if (!_PyST_IsFunctionLike(st->st_cur)) { + if (!ste_binds_locals_for_children(st->st_cur)) { PyErr_SetString(PyExc_SyntaxError, "'await' outside function"); SET_ERROR_LOCATION(st->st_filename, LOCATION(e)); @@ -2660,7 +2768,8 @@ symtable_visit_expr(struct symtable *st, expr_ty e) } /* Special-case super: it counts as a use of __class__ */ if (e->v.Name.ctx == Load && - _PyST_IsFunctionLike(st->st_cur) && + (_PyST_IsFunctionLike(st->st_cur) || + st->st_cur->ste_type == InlinedComprehensionBlock) && _PyUnicode_EqualToASCIIString(e->v.Name.id, "super")) { if (!symtable_add_def(st, &_Py_ID(__class__), USE, LOCATION(e))) return 0; @@ -3103,9 +3212,16 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e, st->st_cur->ste_comp_iter_expr++; VISIT(st, expr, outermost->iter); st->st_cur->ste_comp_iter_expr--; + + /* Non-generator comprehensions are inlined into the enclosing compilation + * unit (including generator expressions), except in annotation scopes + * that can see a class. */ + int will_inline = !is_generator && !st->st_cur->ste_can_see_class_scope; + _Py_block_ty block = will_inline ? InlinedComprehensionBlock : FunctionBlock; + /* Create comprehension scope for the rest */ if (!scope_name || - !symtable_enter_block(st, scope_name, FunctionBlock, (void *)e, LOCATION(e))) { + !symtable_enter_block(st, scope_name, block, (void *)e, LOCATION(e))) { return 0; } switch(e->kind) { @@ -3126,8 +3242,8 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e, st->st_cur->ste_coroutine = 1; } - /* Outermost iter is received as an argument */ - if (!symtable_implicit_arg(st, 0)) { + /* Outermost iter is received as an argument for non-inlined comps */ + if (!will_inline && !symtable_implicit_arg(st, 0)) { symtable_exit_block(st); return 0; } From 8dfe96671b07aab27aefa34ef29cc47c5e0bbdbe Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Wed, 2 Sep 2026 12:12:21 +0100 Subject: [PATCH 02/15] gh-124697: Add NEWS entry for inlined comprehension symbol tables Co-authored-by: Cursor --- .../Library/2026-09-02-12-11-00.gh-issue-124697.sUbScp.rst | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2026-09-02-12-11-00.gh-issue-124697.sUbScp.rst diff --git a/Misc/NEWS.d/next/Library/2026-09-02-12-11-00.gh-issue-124697.sUbScp.rst b/Misc/NEWS.d/next/Library/2026-09-02-12-11-00.gh-issue-124697.sUbScp.rst new file mode 100644 index 00000000000000..6ba511dc063461 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-09-02-12-11-00.gh-issue-124697.sUbScp.rst @@ -0,0 +1,3 @@ +The :mod:`symtable` module now represents inlined list, set and dict +comprehensions (:pep:`709`) as their own symbol tables, of type +:attr:`~symtable.SymbolTableType.INLINED_COMPREHENSION`. From 97ace7dead9f043bc5dc5de851b67072a66059aa Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Wed, 2 Sep 2026 12:18:58 +0100 Subject: [PATCH 03/15] gh-124697: Add What's New entry for inlined comprehension symbol tables Co-authored-by: Cursor --- Doc/whatsnew/3.16.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index 3262acd87d6d49..373aa8ac9e7c41 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -498,6 +498,11 @@ symtable like the builtin :func:`compile`. (Contributed by Serhiy Storchaka in :gh:`153844`.) +* Inlined list, set and dict comprehensions (:pep:`709`) are now represented + as their own symbol tables, of type + :attr:`~symtable.SymbolTableType.INLINED_COMPREHENSION`. + (Contributed by Irit Katriel in :gh:`124697`.) + tkinter ------- From 21ab5dd48a89dda52fd8eeb48189cf77aea69b9e Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Wed, 2 Sep 2026 17:11:00 +0100 Subject: [PATCH 04/15] simpligy logic and edit docs --- Doc/whatsnew/3.16.rst | 8 ++++-- Include/internal/pycore_compile.h | 4 +-- ...-09-02-12-11-00.gh-issue-124697.sUbScp.rst | 2 +- Python/codegen.c | 4 +-- Python/compile.c | 26 +++++++++++-------- 5 files changed, 26 insertions(+), 18 deletions(-) diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index 373aa8ac9e7c41..dfe8d95fa1c841 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -499,8 +499,12 @@ symtable (Contributed by Serhiy Storchaka in :gh:`153844`.) * Inlined list, set and dict comprehensions (:pep:`709`) are now represented - as their own symbol tables, of type - :attr:`~symtable.SymbolTableType.INLINED_COMPREHENSION`. + as their own symbol table entries, of type + :attr:`~symtable.SymbolTableType.INLINED_COMPREHENSION`. This entry type + represents a sub-scope, and holds information only on the symbols whose + scopes are different in the comprehension and the enclosing scope. + Sub-scopes are a new mechanism that can be used when a symbol's scope + changes within the same compilation unit. (Contributed by Irit Katriel in :gh:`124697`.) diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index bf94fc8db5e545..05f6e7cbd99adf 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -181,10 +181,10 @@ typedef struct { PySTEntryObject *saved_ste; } _PyCompile_InlinedComprehensionState; -int _PyCompile_EnterInlinedComprehensionScope(struct _PyCompiler *c, _Py_SourceLocation loc, +int _PyCompile_EnterInlinedComprehensionScope(struct _PyCompiler *c, PySTEntryObject *entry, _PyCompile_InlinedComprehensionState *state); -int _PyCompile_ExitInlinedComprehensionScope(struct _PyCompiler *c, _Py_SourceLocation loc, +int _PyCompile_ExitInlinedComprehensionScope(struct _PyCompiler *c, _PyCompile_InlinedComprehensionState *state); int _PyCompile_AddDeferredAnnotation(struct _PyCompiler *c, stmt_ty s, PyObject **conditional_annotation_index); diff --git a/Misc/NEWS.d/next/Library/2026-09-02-12-11-00.gh-issue-124697.sUbScp.rst b/Misc/NEWS.d/next/Library/2026-09-02-12-11-00.gh-issue-124697.sUbScp.rst index 6ba511dc063461..8dfdbeaf1856fc 100644 --- a/Misc/NEWS.d/next/Library/2026-09-02-12-11-00.gh-issue-124697.sUbScp.rst +++ b/Misc/NEWS.d/next/Library/2026-09-02-12-11-00.gh-issue-124697.sUbScp.rst @@ -1,3 +1,3 @@ The :mod:`symtable` module now represents inlined list, set and dict -comprehensions (:pep:`709`) as their own symbol tables, of type +comprehensions (:pep:`709`) as their own symbol table entries of type :attr:`~symtable.SymbolTableType.INLINED_COMPREHENSION`. diff --git a/Python/codegen.c b/Python/codegen.c index e58bd5b97d3323..90a17a90539d10 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -4982,7 +4982,7 @@ push_inlined_comprehension_state(compiler *c, location loc, _PyCompile_InlinedComprehensionState *state) { RETURN_IF_ERROR( - _PyCompile_EnterInlinedComprehensionScope(c, loc, comp, state)); + _PyCompile_EnterInlinedComprehensionScope(c, comp, state)); RETURN_IF_ERROR( codegen_push_inlined_comprehension_locals(c, loc, comp, state)); return SUCCESS; @@ -5040,7 +5040,7 @@ pop_inlined_comprehension_state(compiler *c, location loc, _PyCompile_InlinedComprehensionState *state) { RETURN_IF_ERROR(codegen_pop_inlined_comprehension_locals(c, loc, state)); - RETURN_IF_ERROR(_PyCompile_ExitInlinedComprehensionScope(c, loc, state)); + RETURN_IF_ERROR(_PyCompile_ExitInlinedComprehensionScope(c, state)); return SUCCESS; } diff --git a/Python/compile.c b/Python/compile.c index 35ae41c6c76404..41def0b5d52191 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -1018,10 +1018,6 @@ _PyCompile_ResolveNameop(compiler *c, PyObject *mangled, int scope, PySTEntryObject *ste = c->u->u_ste; assert(ste != NULL); - while (ste->ste_parent != NULL) { - assert(ste->ste_type == InlinedComprehensionBlock); - ste = ste->ste_parent; - } assert(scope >= 0); switch (scope) { @@ -1034,15 +1030,24 @@ _PyCompile_ResolveNameop(compiler *c, PyObject *mangled, int scope, *optype = COMPILE_OP_DEREF; break; case LOCAL: - if (_PyST_IsFunctionLike(ste) || c->u->u_ste->ste_type == InlinedComprehensionBlock) { + /* Inlined comprehensions isolate their locals as FAST, even when + * nested in class or module scope. */ + if (_PyST_IsFunctionLike(ste) || ste->ste_type == InlinedComprehensionBlock) { *optype = COMPILE_OP_FAST; } break; - case GLOBAL_IMPLICIT: - if (_PyST_IsFunctionLike(ste)) { + case GLOBAL_IMPLICIT: { + /* Opcode depends on the enclosing non-inlined scope. */ + PySTEntryObject *enclosing = ste; + while (enclosing->ste_parent != NULL) { + assert(enclosing->ste_type == InlinedComprehensionBlock); + enclosing = enclosing->ste_parent; + } + if (_PyST_IsFunctionLike(enclosing)) { *optype = COMPILE_OP_GLOBAL; } break; + } case GLOBAL_EXPLICIT: *optype = COMPILE_OP_GLOBAL; break; @@ -1058,8 +1063,7 @@ _PyCompile_ResolveNameop(compiler *c, PyObject *mangled, int scope, } int -_PyCompile_EnterInlinedComprehensionScope(compiler *c, location loc, - PySTEntryObject *entry, +_PyCompile_EnterInlinedComprehensionScope(compiler *c, PySTEntryObject *entry, _PyCompile_InlinedComprehensionState *state) { assert(state->saved_ste == NULL); @@ -1069,8 +1073,8 @@ _PyCompile_EnterInlinedComprehensionScope(compiler *c, location loc, } int -_PyCompile_ExitInlinedComprehensionScope(compiler *c, location loc, - _PyCompile_InlinedComprehensionState *state) +_PyCompile_ExitInlinedComprehensionScope(compiler *c, + _PyCompile_InlinedComprehensionState *state) { assert(state->saved_ste != NULL); Py_DECREF(c->u->u_ste); From 045913c37b53db7635d0373ddc496f262dc28656 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Wed, 2 Sep 2026 17:14:41 +0100 Subject: [PATCH 05/15] redundant check --- Python/compile.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Python/compile.c b/Python/compile.c index 41def0b5d52191..bd8171c40e6cf4 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -1416,10 +1416,7 @@ _PyCompile_OptimizeAndAssemble(compiler *c, int addNone) PyObject *filename = c->c_filename; int code_flags = compute_code_flags(c); - if (code_flags < 0) { - return NULL; - } - + assert(code_flags >= 0); if (_PyCodegen_AddReturnAtEnd(c, addNone) < 0) { return NULL; } From 0c2ad7e1c6df3549ecb10ca30f13701ada17ec9b Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Wed, 2 Sep 2026 17:48:05 +0100 Subject: [PATCH 06/15] rename helper and use it in more places --- Python/symtable.c | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/Python/symtable.c b/Python/symtable.c index f9576aae228726..aac0c626beaff5 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -91,6 +91,12 @@ #define IS_ASYNC_DEF(st) ((st)->st_cur->ste_type == FunctionBlock && (st)->st_cur->ste_coroutine) +static int +ste_uses_fast_locals(PySTEntryObject *ste) +{ + return _PyST_IsFunctionLike(ste) || ste->ste_type == InlinedComprehensionBlock; +} + static PySTEntryObject * ste_new(struct symtable *st, identifier name, _Py_block_ty block, void *key, _Py_SourceLocation loc) @@ -130,8 +136,7 @@ ste_new(struct symtable *st, identifier name, _Py_block_ty block, if (st->st_cur != NULL && (st->st_cur->ste_nested || - _PyST_IsFunctionLike(st->st_cur) || - st->st_cur->ste_type == InlinedComprehensionBlock)) + ste_uses_fast_locals(st->st_cur))) ste->ste_nested = 1; ste->ste_generator = 0; ste->ste_coroutine = 0; @@ -582,13 +587,6 @@ _PyST_IsFunctionLike(PySTEntryObject *ste) || ste->ste_type == TypeParametersBlock; } -/* True if this block binds locals that are visible to nested scopes */ -static int -ste_binds_locals_for_children(PySTEntryObject *ste) -{ - return _PyST_IsFunctionLike(ste) || ste->ste_type == InlinedComprehensionBlock; -} - static int error_at_directive(PySTEntryObject *ste, PyObject *name) { @@ -1321,7 +1319,7 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, /* Populate global and bound sets to be passed to children. */ if (ste->ste_type != ClassBlock) { /* Add function locals to bound set */ - if (ste_binds_locals_for_children(ste)) { + if (ste_uses_fast_locals(ste)) { temp = PyNumber_InPlaceOr(newbound, local); if (!temp) goto error; @@ -1409,7 +1407,7 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, } /* Check if any local variables must be converted to cell variables */ - if (ste_binds_locals_for_children(ste) && !analyze_cells(scopes, newfree, inlined_cells)) { + if (ste_uses_fast_locals(ste) && !analyze_cells(scopes, newfree, inlined_cells)) { goto error; } else if (ste->ste_type == ClassBlock && !drop_class_free(ste, newfree)) { @@ -2690,7 +2688,7 @@ symtable_visit_expr(struct symtable *st, expr_ty e) return 0; } if (!allows_top_level_await(st)) { - if (!ste_binds_locals_for_children(st->st_cur)) { + if (!ste_uses_fast_locals(st->st_cur)) { PyErr_SetString(PyExc_SyntaxError, "'await' outside function"); SET_ERROR_LOCATION(st->st_filename, LOCATION(e)); @@ -2768,8 +2766,7 @@ symtable_visit_expr(struct symtable *st, expr_ty e) } /* Special-case super: it counts as a use of __class__ */ if (e->v.Name.ctx == Load && - (_PyST_IsFunctionLike(st->st_cur) || - st->st_cur->ste_type == InlinedComprehensionBlock) && + ste_uses_fast_locals(st->st_cur) && _PyUnicode_EqualToASCIIString(e->v.Name.id, "super")) { if (!symtable_add_def(st, &_Py_ID(__class__), USE, LOCATION(e))) return 0; From 3f7b483268b8dceca816ca690be6f6ac0a86cb5e Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Thu, 3 Sep 2026 13:14:03 +0100 Subject: [PATCH 07/15] finalize nested against enclosing listcomp --- Lib/test/test_listcomps.py | 80 ++++++++++++++++++++++++++++++++++++++ Lib/test/test_symtable.py | 19 +++++++++ Python/symtable.c | 7 +++- 3 files changed, 104 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py index fca9acbc6b1ef6..f5fe3235ceec2b 100644 --- a/Lib/test/test_listcomps.py +++ b/Lib/test/test_listcomps.py @@ -407,6 +407,86 @@ def test_nested(self): outputs = {"y": [[0, 1], [0, 1, 4]]} self._check_in_scopes(code, outputs) + def test_nested_inner_uses_outer_iter(self): + # Inner comprehension reads the outer iteration variable. In a class + # this must not be treated as a class-level name of the same name. + code = """ + x = 99 + y = [[x for _ in (0,)] for x in (42,)] + """ + outputs = {"y": [[42]]} + self._check_in_scopes(code, outputs) + + def test_nested_mixed_comprehensions_use_outer_iter(self): + cases = [ + ("y = [{x for _ in (0,)} for x in (42,)]", {"y": [{42}]}), + ("y = [{x: x for _ in (0,)} for x in (42,)]", {"y": [{42: 42}]}), + ("y = {[x for _ in (0,)][0] for x in (42,)}", {"y": {42}}), + ("y = {x: [x for _ in (0,)] for x in (42,)}", {"y": {42: [42]}}), + ] + for line, outputs in cases: + with self.subTest(line=line): + code = f"x = 99\n{line}" + self._check_in_scopes(code, outputs) + + def test_nested_triple_inner_uses_outer_iter(self): + code = """ + x = 99 + y = [[[x for _ in (0,)] for _ in (0,)] for x in (42,)] + """ + outputs = {"y": [[[42]]]} + self._check_in_scopes(code, outputs) + + def test_nested_inner_uses_outer_iter_in_iter(self): + code = """ + x = 99 + y = [[_ for _ in (x,)] for x in (42,)] + """ + outputs = {"y": [[42]]} + self._check_in_scopes(code, outputs) + + def test_nested_inner_uses_outer_iter_in_if(self): + code = """ + x = 99 + y = [[1 for _ in (0,) if x] for x in (42,)] + """ + outputs = {"y": [[1]]} + self._check_in_scopes(code, outputs) + + def test_nested_sibling_inners_use_outer_iter(self): + code = """ + x = 99 + y = [([x for _ in (0,)], [x for _ in (1,)]) for x in (42,)] + """ + outputs = {"y": [([42], [42])]} + self._check_in_scopes(code, outputs) + + def test_nested_lambda_captures_outer_iter(self): + code = """ + x = 99 + y = [[lambda: x for _ in (0,)] for x in (42,)] + z = y[0][0]() + """ + outputs = {"z": 42} + self._check_in_scopes(code, outputs) + + def test_nested_references___class__(self): + code = """ + res = [[__class__ for _ in (0,)] for _ in (1,)] + """ + self._check_in_scopes(code, raises=NameError) + + def test_nested_references___class___via_lambda(self): + class _C: + res = [[lambda: __class__ for _ in (0,)] for _ in (1,)] + self.assertIs(_C.res[0][0](), _C) + + def test_nested_references_super(self): + code = """ + res = [[super for _ in (0,)] for _ in (1,)] + """ + self._check_in_scopes(code, outputs={"res": [[super]]}) + def test_nested_2(self): code = """ l = [1, 2, 3] diff --git a/Lib/test/test_symtable.py b/Lib/test/test_symtable.py index 5be898abd2d169..6c26a7387b1ded 100644 --- a/Lib/test/test_symtable.py +++ b/Lib/test/test_symtable.py @@ -546,6 +546,25 @@ def test_inlined_nested_comprehension(self): self.check_nested_inlined_listcomp( children[0], children[1], ["x"], ["y"], nested=False) + def test_inlined_nested_comprehension_class_iter_var(self): + st = symtable.symtable( + "class C:\n" + " x = 99\n" + " [[x for _ in (0,)] for x in (42,)]", + "?", "exec") + C = find_block(st, "C") + children = C.get_children() + self.assertEqual(len(children), 2) + self.check_nested_inlined_listcomp( + children[0], children[1], ["x"], ["_"], nested=False) + self.assertFalse(C.lookup("x").is_free()) + self.assertTrue(C.lookup("x").is_local()) + self.assertFalse(children[0].lookup("x").is_free()) + self.assertTrue(children[0].lookup("x").is_cell()) + self.assertFalse(children[1].lookup("_").is_free()) + with self.assertRaises(KeyError): + children[1].lookup("x") + def test_inlined_sibling_nested_comprehensions(self): st = symtable.symtable( "def f(): [[y for y in x] for x in [1]]; [[w for w in z] for z in [2]]", diff --git a/Python/symtable.c b/Python/symtable.c index aac0c626beaff5..d1931160f4b257 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -994,6 +994,8 @@ finalize_inlined_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, } } Py_CLEAR(to_remove); + /* Finalize nested inlined comprehensions against this comprehension, + * not the original enclosing scope. */ for (Py_ssize_t i = 0; i < PyList_GET_SIZE(comp->ste_children); i++) { PySTEntryObject *child = (PySTEntryObject *)PyList_GET_ITEM(comp->ste_children, i); if (child->ste_type != InlinedComprehensionBlock) { @@ -1003,7 +1005,7 @@ finalize_inlined_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, if (child_free == NULL) { return 0; } - int ok = finalize_inlined_comprehension(ste, child, child_free, + int ok = finalize_inlined_comprehension(comp, child, child_free, outer_newfree, inlined_cells); Py_DECREF(child_free); if (!ok) { @@ -1372,7 +1374,8 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, } // Compress InlinedComprehensionBlocks ste_symbols to a delta (bindings + - // class FREE overrides). Nested deltas are finalized recursively. + // class FREE overrides). Nested deltas are finalized recursively + // against their immediate parent. if (!analyze_child_block(entry, newbound, newfree, newglobal, type_params, new_class_entry, &child_free)) { From 9d9ebfcbb9d155e9649cccd40bd5e7544464ec30 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Thu, 3 Sep 2026 14:15:13 +0100 Subject: [PATCH 08/15] do not splice --- Lib/test/test_symtable.py | 39 +++++++++++++++++++++++++-------------- Python/symtable.c | 15 --------------- 2 files changed, 25 insertions(+), 29 deletions(-) diff --git a/Lib/test/test_symtable.py b/Lib/test/test_symtable.py index 6c26a7387b1ded..28df60ff6e6ec6 100644 --- a/Lib/test/test_symtable.py +++ b/Lib/test/test_symtable.py @@ -523,13 +523,11 @@ def check_inlined_listcomp(self, st, identifiers, *, nested, nchildren=0): self.assertEqual(len(children), nchildren) return children - def check_nested_inlined_listcomp(self, outer, hoisted, outer_ids, inner_ids, *, nested): - # Nested namespaces of inlined comprehensions are also hoisted into - # the enclosing scope's children list. + def check_nested_inlined_listcomp(self, outer, outer_ids, inner_ids, *, nested): inner, = self.check_inlined_listcomp( outer, outer_ids, nested=nested, nchildren=1) self.check_inlined_listcomp(inner, inner_ids, nested=True) - self.assertIs(hoisted, inner) + return inner def test_inlined_comprehension(self): st = symtable.symtable("[x for x in [1]]", "?", "exec") @@ -542,9 +540,9 @@ def test_inlined_nested_comprehension(self): st = symtable.symtable("[[y for y in x] for x in [1]]", "?", "exec") self.assertEqual(sorted(st.get_identifiers()), []) children = st.get_children() - self.assertEqual(len(children), 2) + self.assertEqual(len(children), 1) self.check_nested_inlined_listcomp( - children[0], children[1], ["x"], ["y"], nested=False) + children[0], ["x"], ["y"], nested=False) def test_inlined_nested_comprehension_class_iter_var(self): st = symtable.symtable( @@ -554,16 +552,16 @@ def test_inlined_nested_comprehension_class_iter_var(self): "?", "exec") C = find_block(st, "C") children = C.get_children() - self.assertEqual(len(children), 2) - self.check_nested_inlined_listcomp( - children[0], children[1], ["x"], ["_"], nested=False) + self.assertEqual(len(children), 1) + inner = self.check_nested_inlined_listcomp( + children[0], ["x"], ["_"], nested=False) self.assertFalse(C.lookup("x").is_free()) self.assertTrue(C.lookup("x").is_local()) self.assertFalse(children[0].lookup("x").is_free()) self.assertTrue(children[0].lookup("x").is_cell()) - self.assertFalse(children[1].lookup("_").is_free()) + self.assertFalse(inner.lookup("_").is_free()) with self.assertRaises(KeyError): - children[1].lookup("x") + inner.lookup("x") def test_inlined_sibling_nested_comprehensions(self): st = symtable.symtable( @@ -573,11 +571,24 @@ def test_inlined_sibling_nested_comprehensions(self): self.assertIs(f.get_type(), symtable.SymbolTableType.FUNCTION) self.assertEqual(sorted(f.get_identifiers()), []) children = f.get_children() - self.assertEqual(len(children), 4) + self.assertEqual(len(children), 2) self.check_nested_inlined_listcomp( - children[0], children[1], ["x"], ["y"], nested=True) + children[0], ["x"], ["y"], nested=True) self.check_nested_inlined_listcomp( - children[2], children[3], ["z"], ["w"], nested=True) + children[1], ["z"], ["w"], nested=True) + + def test_deeply_nested_inlined_comprehensions(self): + depth = 24 + source = "[" * depth + "0" + " for x in ()]" * depth + st = symtable.symtable(source, "?", "exec") + cur = st + for _ in range(depth): + children = cur.get_children() + self.assertEqual(len(children), 1) + cur = children[0] + self.assertIs(cur.get_type(), + symtable.SymbolTableType.INLINED_COMPREHENSION) + self.assertEqual(cur.get_children(), []) def test__symtable_refleak(self): # Regression test for reference leak in PyUnicode_FSDecoder. diff --git a/Python/symtable.c b/Python/symtable.c index d1931160f4b257..9ff9ccbf46dd77 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -1394,21 +1394,6 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, Py_DECREF(temp); } - /* Splice children of inlined comprehensions into our children list */ - for (i = PyList_GET_SIZE(ste->ste_children) - 1; i >= 0; --i) { - PyObject* c = PyList_GET_ITEM(ste->ste_children, i); - PySTEntryObject* entry; - assert(c && PySTEntry_Check(c)); - entry = (PySTEntryObject*)c; - if (entry->ste_type == InlinedComprehensionBlock && - PyList_GET_SIZE(entry->ste_children) > 0 && - PyList_SetSlice(ste->ste_children, i+1, i + 1, - entry->ste_children) < 0) - { - goto error; - } - } - /* Check if any local variables must be converted to cell variables */ if (ste_uses_fast_locals(ste) && !analyze_cells(scopes, newfree, inlined_cells)) { goto error; From bee5d57cebe4d431c96528a98bce7b9fa71b291b Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Thu, 3 Sep 2026 14:54:01 +0100 Subject: [PATCH 09/15] fix error handling in push/pop_inlined_comprehension_state --- Python/codegen.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Python/codegen.c b/Python/codegen.c index 90a17a90539d10..9a2ba1ae6cce09 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -4983,8 +4983,10 @@ push_inlined_comprehension_state(compiler *c, location loc, { RETURN_IF_ERROR( _PyCompile_EnterInlinedComprehensionScope(c, comp, state)); - RETURN_IF_ERROR( - codegen_push_inlined_comprehension_locals(c, loc, comp, state)); + if (codegen_push_inlined_comprehension_locals(c, loc, comp, state) < 0){ + _PyCompile_ExitInlinedComprehensionScope(c, state); + return ERROR; + } return SUCCESS; } @@ -5039,9 +5041,9 @@ static int pop_inlined_comprehension_state(compiler *c, location loc, _PyCompile_InlinedComprehensionState *state) { - RETURN_IF_ERROR(codegen_pop_inlined_comprehension_locals(c, loc, state)); + int result = codegen_pop_inlined_comprehension_locals(c, loc, state); RETURN_IF_ERROR(_PyCompile_ExitInlinedComprehensionScope(c, state)); - return SUCCESS; + return result; } static int From 0a13f5916d1dbe1e6cb16b09a38b5f5993d98752 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Thu, 3 Sep 2026 19:14:35 +0100 Subject: [PATCH 10/15] do not remove free variables from comprehension scope --- Lib/test/test_symtable.py | 37 ++++++++++++++++++++-- Python/compile.c | 26 ++++++++++++++++ Python/symtable.c | 65 ++------------------------------------- 3 files changed, 63 insertions(+), 65 deletions(-) diff --git a/Lib/test/test_symtable.py b/Lib/test/test_symtable.py index 28df60ff6e6ec6..3536523ac0b822 100644 --- a/Lib/test/test_symtable.py +++ b/Lib/test/test_symtable.py @@ -544,6 +544,37 @@ def test_inlined_nested_comprehension(self): self.check_nested_inlined_listcomp( children[0], ["x"], ["y"], nested=False) + def test_inlined_comprehension_use_of_enclosing_free_in_function(self): + st = symtable.symtable( + "def outer(x):\n" + " def inner():\n" + " return [x for y in ()]", + "?", "exec") + inner = find_block(find_block(st, "outer"), "inner") + self.assertTrue(inner.lookup("x").is_free()) + comp, = inner.get_children() + self.assertTrue(comp.lookup("x").is_free()) + self.assertTrue(comp.lookup("x").is_referenced()) + + def test_inlined_comprehension_use_of_enclosing_free_in_class(self): + st = symtable.symtable( + "def f():\n" + " y = 1\n" + " class C:\n" + " y = 2\n" + " vals = [(x, y) for x in range(2)]", + "?", "exec") + f = find_block(st, "f") + self.assertTrue(f.lookup("y").is_cell()) + C = find_block(f, "C") + self.assertTrue(C.lookup("y").is_local()) + self.assertFalse(C.lookup("y").is_free()) + self.assertTrue(C.lookup("y").is_free_class()) + comp, = C.get_children() + self.assertTrue(comp.lookup("y").is_free()) + self.assertTrue(comp.lookup("y").is_referenced()) + self.assertTrue(comp.lookup("x").is_local()) + def test_inlined_nested_comprehension_class_iter_var(self): st = symtable.symtable( "class C:\n" @@ -554,14 +585,14 @@ def test_inlined_nested_comprehension_class_iter_var(self): children = C.get_children() self.assertEqual(len(children), 1) inner = self.check_nested_inlined_listcomp( - children[0], ["x"], ["_"], nested=False) + children[0], ["x"], ["_", "x"], nested=False) self.assertFalse(C.lookup("x").is_free()) self.assertTrue(C.lookup("x").is_local()) self.assertFalse(children[0].lookup("x").is_free()) self.assertTrue(children[0].lookup("x").is_cell()) self.assertFalse(inner.lookup("_").is_free()) - with self.assertRaises(KeyError): - inner.lookup("x") + self.assertTrue(inner.lookup("x").is_free()) + self.assertTrue(inner.lookup("x").is_referenced()) def test_inlined_sibling_nested_comprehensions(self): st = symtable.symtable( diff --git a/Python/compile.c b/Python/compile.c index bd8171c40e6cf4..acff5a5eb9e9b5 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -910,6 +910,26 @@ compiler_mod(compiler *c, mod_ty mod) return co; } +/* Inlined comprehensions are compiled in the enclosing unit. If a name is + * FREE in the comprehension, resolve it in enclosing tables until it is no + * longer FREE. Stop if the next table is a class: nested scopes (including + * inlined comprehensions) do not see class locals, so the name stays FREE. */ +static int +compiler_resolve_inlined_free(PySTEntryObject **ste, int scope, PyObject *name) +{ + while (scope == FREE && (*ste)->ste_type == InlinedComprehensionBlock) { + PySTEntryObject *parent = (*ste)->ste_parent; + assert(parent != NULL); + if (parent->ste_type == ClassBlock) { + break; + } + *ste = parent; + scope = _PyST_GetScope(*ste, name); + RETURN_IF_ERROR(scope); + } + return scope; +} + int _PyCompile_GetRefType(compiler *c, PyObject *name) { @@ -921,6 +941,9 @@ _PyCompile_GetRefType(compiler *c, PyObject *name) } PySTEntryObject *ste = c->u->u_ste; int scope = _PyST_GetScope(ste, name); + RETURN_IF_ERROR(scope); + scope = compiler_resolve_inlined_free(&ste, scope, name); + RETURN_IF_ERROR(scope); if (scope == 0) { PyErr_Format(PyExc_SystemError, "_PyST_GetScope(name=%R) failed: " @@ -1020,6 +1043,9 @@ _PyCompile_ResolveNameop(compiler *c, PyObject *mangled, int scope, assert(ste != NULL); assert(scope >= 0); + scope = compiler_resolve_inlined_free(&ste, scope, mangled); + RETURN_IF_ERROR(scope); + switch (scope) { case FREE: dict = c->u->u_metadata.u_freevars; diff --git a/Python/symtable.c b/Python/symtable.c index 9ff9ccbf46dd77..5552a3ec252998 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -813,33 +813,6 @@ is_free_in_any_child(PySTEntryObject *entry, PyObject *key) return 0; } -/* True if name is FREE in the comprehension and bound in the enclosing class. - * Those names are kept in the compressed delta so lookup does not treat them - * as class locals. */ -static int -class_binds_free_name(PySTEntryObject *ste, PyObject *name, long comp_flags) -{ - if (SYMBOL_TO_SCOPE(comp_flags) != FREE) { - return 0; - } - if (ste->ste_type != ClassBlock) { - return 0; - } - PyObject *v = PyDict_GetItemWithError(ste->ste_symbols, name); - if (v == NULL) { - return PyErr_Occurred() ? -1 : 0; - } - long class_flags = PyLong_AsLong(v); - if (class_flags == -1 && PyErr_Occurred()) { - return -1; - } - if (class_flags & (DEF_LOCAL | DEF_GLOBAL | DEF_FREE_CLASS | DEF_TYPE_PARAM)) - { - return 1; - } - return 0; -} - static PyObject * get_freevar_names(PySTEntryObject *ste) { @@ -865,6 +838,7 @@ get_freevar_names(PySTEntryObject *ste) return free; } + static int finalize_inlined_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, PyObject *comp_free, PyObject *outer_newfree, @@ -872,16 +846,10 @@ finalize_inlined_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, { PyObject *k, *v; Py_ssize_t pos = 0; - PyObject *to_remove = NULL; assert(comp->ste_type == InlinedComprehensionBlock); assert(comp->ste_parent != NULL); - to_remove = PyList_New(0); - if (to_remove == NULL) { - return 0; - } - while (PyDict_Next(comp->ste_symbols, &pos, &k, &v)) { long comp_flags = PyLong_AsLong(v); if (comp_flags == -1 && PyErr_Occurred()) { @@ -968,32 +936,7 @@ finalize_inlined_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, else { assert(scope != FREE || PySet_Contains(comp_free, k) == 1); } - - /* keep bindings, globals, and class-bound frees in the delta; - drop other names (typically FREE uses) so lookup climbs to parent. */ - if ((comp_flags & DEF_LOCAL) && !(comp_flags & DEF_NONLOCAL)) { - continue; - } - if (scope == GLOBAL_IMPLICIT || scope == GLOBAL_EXPLICIT) { - continue; - } - int keep = class_binds_free_name(ste, k, comp_flags); - if (keep < 0) { - goto error; - } - if (!keep) { - if (PyList_Append(to_remove, k) < 0) { - goto error; - } - } - } - for (Py_ssize_t i = 0; i < PyList_GET_SIZE(to_remove); i++) { - PyObject *name = PyList_GET_ITEM(to_remove, i); - if (PyDict_DelItem(comp->ste_symbols, name) < 0) { - goto error; - } } - Py_CLEAR(to_remove); /* Finalize nested inlined comprehensions against this comprehension, * not the original enclosing scope. */ for (Py_ssize_t i = 0; i < PyList_GET_SIZE(comp->ste_children); i++) { @@ -1014,7 +957,6 @@ finalize_inlined_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, } return 1; error: - Py_XDECREF(to_remove); return 0; } @@ -1287,7 +1229,6 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, inlined_cells = PySet_New(NULL); if (!inlined_cells) goto error; - /* Class namespace has no effect on names visible in nested functions, so populate the global and bound sets to be passed to child blocks before analyzing @@ -1382,7 +1323,8 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, goto error; } if (entry->ste_type == InlinedComprehensionBlock && ste->ste_type != InlinedComprehensionBlock) { - if (!finalize_inlined_comprehension(ste, entry, child_free, newfree, inlined_cells)) { + if (!finalize_inlined_comprehension(ste, entry, child_free, newfree, + inlined_cells)) { Py_DECREF(child_free); goto error; } @@ -1405,7 +1347,6 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, if (!update_symbols(ste->ste_symbols, scopes, bound, newfree, inlined_cells, (ste->ste_type == ClassBlock) || ste->ste_can_see_class_scope)) goto error; - temp = PyNumber_InPlaceOr(free, newfree); if (!temp) goto error; From f4ad426afb3ed135048fb4bf29d217a6abf5547d Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Thu, 3 Sep 2026 19:53:24 +0100 Subject: [PATCH 11/15] fix test --- Lib/test/test_listcomps.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py index f5fe3235ceec2b..65213c53131b5b 100644 --- a/Lib/test/test_listcomps.py +++ b/Lib/test/test_listcomps.py @@ -419,14 +419,25 @@ def test_nested_inner_uses_outer_iter(self): def test_nested_mixed_comprehensions_use_outer_iter(self): cases = [ - ("y = [{x for _ in (0,)} for x in (42,)]", {"y": [{42}]}), - ("y = [{x: x for _ in (0,)} for x in (42,)]", {"y": [{42: 42}]}), - ("y = {[x for _ in (0,)][0] for x in (42,)}", {"y": {42}}), - ("y = {x: [x for _ in (0,)] for x in (42,)}", {"y": {42: [42]}}), + (""" + x = 99 + y = [{x for _ in (0,)} for x in (42,)] + """, {"y": [{42}]}), + (""" + x = 99 + y = [{x: x for _ in (0,)} for x in (42,)] + """, {"y": [{42: 42}]}), + (""" + x = 99 + y = {[x for _ in (0,)][0] for x in (42,)} + """, {"y": {42}}), + (""" + x = 99 + y = {x: [x for _ in (0,)] for x in (42,)} + """, {"y": {42: [42]}}), ] - for line, outputs in cases: - with self.subTest(line=line): - code = f"x = 99\n{line}" + for code, outputs in cases: + with self.subTest(code=code): self._check_in_scopes(code, outputs) def test_nested_triple_inner_uses_outer_iter(self): From ef2a6dae0797f9514951f0073821d79514ad7077 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Thu, 3 Sep 2026 21:39:41 +0100 Subject: [PATCH 12/15] fix stale comment --- Python/symtable.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Python/symtable.c b/Python/symtable.c index 5552a3ec252998..17285eb91a349f 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -1314,9 +1314,9 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, } } - // Compress InlinedComprehensionBlocks ste_symbols to a delta (bindings + - // class FREE overrides). Nested deltas are finalized recursively - // against their immediate parent. + // Finalize inlined comprehensions against the nearest non-inlined + // enclosing scope. Nested ones are finalized recursively against + // their immediate parent. if (!analyze_child_block(entry, newbound, newfree, newglobal, type_params, new_class_entry, &child_free)) { From 8b37711902bf81d155d5497a4a107aaac830e493 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Thu, 3 Sep 2026 21:52:00 +0100 Subject: [PATCH 13/15] make the symtable honest about __class__ etc --- Include/internal/pycore_symtable.h | 1 + Lib/test/test_symtable.py | 12 ++++++++++++ Python/compile.c | 12 +++++++----- Python/symtable.c | 28 ++++++++++++---------------- 4 files changed, 32 insertions(+), 21 deletions(-) diff --git a/Include/internal/pycore_symtable.h b/Include/internal/pycore_symtable.h index db609243f2f41e..88725a2a3fd343 100644 --- a/Include/internal/pycore_symtable.h +++ b/Include/internal/pycore_symtable.h @@ -146,6 +146,7 @@ extern PyTypeObject PySTEntry_Type; extern long _PyST_GetSymbol(PySTEntryObject *, PyObject *); extern int _PyST_GetScope(PySTEntryObject *, PyObject *); extern int _PyST_IsFunctionLike(PySTEntryObject *); +extern int _PyST_IsClassClosureName(PyObject *); extern struct symtable* _PySymtable_Build( struct _mod *mod, diff --git a/Lib/test/test_symtable.py b/Lib/test/test_symtable.py index 3536523ac0b822..68e873f8addd04 100644 --- a/Lib/test/test_symtable.py +++ b/Lib/test/test_symtable.py @@ -575,6 +575,18 @@ def test_inlined_comprehension_use_of_enclosing_free_in_class(self): self.assertTrue(comp.lookup("y").is_referenced()) self.assertTrue(comp.lookup("x").is_local()) + def test_inlined_comprehension_class_closure_names_are_free(self): + st = symtable.symtable( + "class C:\n" + " [__class__ for x in [1]]", + "?", "exec") + C = find_block(st, "C") + comp, = C.get_children() + self.assertTrue(comp.lookup("__class__").is_free()) + self.assertTrue(comp.lookup("__class__").is_referenced()) + with self.assertRaises(KeyError): + C.lookup("__class__") + def test_inlined_nested_comprehension_class_iter_var(self): st = symtable.symtable( "class C:\n" diff --git a/Python/compile.c b/Python/compile.c index acff5a5eb9e9b5..74697e67e7e891 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -913,7 +913,9 @@ compiler_mod(compiler *c, mod_ty mod) /* Inlined comprehensions are compiled in the enclosing unit. If a name is * FREE in the comprehension, resolve it in enclosing tables until it is no * longer FREE. Stop if the next table is a class: nested scopes (including - * inlined comprehensions) do not see class locals, so the name stays FREE. */ + * inlined comprehensions) do not see class locals, so the name stays FREE. + * __class__ and friends are not allowed to be free through a class; treat + * those loads as implicit globals. */ static int compiler_resolve_inlined_free(PySTEntryObject **ste, int scope, PyObject *name) { @@ -921,6 +923,9 @@ compiler_resolve_inlined_free(PySTEntryObject **ste, int scope, PyObject *name) PySTEntryObject *parent = (*ste)->ste_parent; assert(parent != NULL); if (parent->ste_type == ClassBlock) { + if (_PyST_IsClassClosureName(name)) { + return GLOBAL_IMPLICIT; + } break; } *ste = parent; @@ -933,10 +938,7 @@ compiler_resolve_inlined_free(PySTEntryObject **ste, int scope, PyObject *name) int _PyCompile_GetRefType(compiler *c, PyObject *name) { - if (c->u->u_scope_type == COMPILE_SCOPE_CLASS && - (_PyUnicode_EqualToASCIIString(name, "__class__") || - _PyUnicode_EqualToASCIIString(name, "__classdict__") || - _PyUnicode_EqualToASCIIString(name, "__conditional_annotations__"))) { + if (c->u->u_scope_type == COMPILE_SCOPE_CLASS && _PyST_IsClassClosureName(name)) { return CELL; } PySTEntryObject *ste = c->u->u_ste; diff --git a/Python/symtable.c b/Python/symtable.c index 17285eb91a349f..298562f574a7d1 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -587,6 +587,14 @@ _PyST_IsFunctionLike(PySTEntryObject *ste) || ste->ste_type == TypeParametersBlock; } +int +_PyST_IsClassClosureName(PyObject *name) +{ + return _PyUnicode_EqualToASCIIString(name, "__class__") + || _PyUnicode_EqualToASCIIString(name, "__classdict__") + || _PyUnicode_EqualToASCIIString(name, "__conditional_annotations__"); +} + static int error_at_directive(PySTEntryObject *ste, PyObject *name) { @@ -878,13 +886,11 @@ finalize_inlined_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, if (existing == NULL && PyErr_Occurred()) { goto error; } - // __class__, __classdict__ and __conditional_annotations__ are - // not allowed to be free through a class scope (see - // drop_class_free) unless children scopes need it + // These names are not allowed to be free through a class (see + // drop_class_free) unless a nested child needs them. Keep FREE + // on this table; compile treats the load as a global. if (scope == FREE && ste->ste_type == ClassBlock && - (_PyUnicode_EqualToASCIIString(k, "__class__") || - _PyUnicode_EqualToASCIIString(k, "__classdict__") || - _PyUnicode_EqualToASCIIString(k, "__conditional_annotations__"))) { + _PyST_IsClassClosureName(k)) { int child_needs_free = is_free_in_any_child(comp, k); if (child_needs_free < 0) { goto error; @@ -894,16 +900,6 @@ finalize_inlined_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, goto error; } } - long new_flags = only_flags | (GLOBAL_IMPLICIT << SCOPE_OFFSET); - PyObject *newv = PyLong_FromLong(new_flags); - if (newv == NULL) { - goto error; - } - if (PyDict_SetItem(comp->ste_symbols, k, newv) < 0) { - Py_DECREF(newv); - goto error; - } - Py_DECREF(newv); continue; } if (existing) { From ce7433ee4063a327cbe5385f29dd3aca348afde0 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Thu, 3 Sep 2026 21:59:48 +0100 Subject: [PATCH 14/15] no more climbing in _PyST_GetSymbol - symbol table is honest now --- Python/symtable.c | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/Python/symtable.c b/Python/symtable.c index 298562f574a7d1..aa1e5ff6d89001 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -545,26 +545,22 @@ _PySymtable_LookupOptional(struct symtable *st, void *key, long _PyST_GetSymbol(PySTEntryObject *ste, PyObject *name) { - while (ste != NULL) { - PyObject *v; - if (PyDict_GetItemRef(ste->ste_symbols, name, &v) < 0) { - return -1; - } - if (v != NULL) { - long symbol = PyLong_AsLong(v); - Py_DECREF(v); - if (symbol < 0) { - if (!PyErr_Occurred()) { - PyErr_SetString(PyExc_SystemError, "invalid symbol"); - } - return -1; - } - return symbol; + PyObject *v; + if (PyDict_GetItemRef(ste->ste_symbols, name, &v) < 0) { + return -1; + } + if (v == NULL) { + return 0; + } + long symbol = PyLong_AsLong(v); + Py_DECREF(v); + if (symbol < 0) { + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_SystemError, "invalid symbol"); } - assert(ste->ste_parent == NULL || ste->ste_type == InlinedComprehensionBlock); - ste = ste->ste_parent; + return -1; } - return 0; + return symbol; } int From 3515cc01007166a5fdbc5e22a9dbf7240943fa3a Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Fri, 4 Sep 2026 00:21:37 +0100 Subject: [PATCH 15/15] remove DEF_COMP_CELL --- Doc/whatsnew/3.16.rst | 9 ++-- Include/internal/pycore_symtable.h | 1 - Lib/symtable.py | 12 +++-- Lib/test/test_listcomps.py | 14 ++++++ Lib/test/test_symtable.py | 18 +++++++- Modules/symtablemodule.c | 1 - Python/compile.c | 71 +++++++++++++++++++++++++++++- Python/symtable.c | 27 ++---------- 8 files changed, 116 insertions(+), 37 deletions(-) diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index dfe8d95fa1c841..e5f4459fecaf21 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -500,11 +500,10 @@ symtable * Inlined list, set and dict comprehensions (:pep:`709`) are now represented as their own symbol table entries, of type - :attr:`~symtable.SymbolTableType.INLINED_COMPREHENSION`. This entry type - represents a sub-scope, and holds information only on the symbols whose - scopes are different in the comprehension and the enclosing scope. - Sub-scopes are a new mechanism that can be used when a symbol's scope - changes within the same compilation unit. + :attr:`~symtable.SymbolTableType.INLINED_COMPREHENSION`. Each such entry is + a lexical child of the enclosing scope and records the comprehension's own + locals, cells, and free names. It does not correspond to a separate + compilation unit. (Contributed by Irit Katriel in :gh:`124697`.) diff --git a/Include/internal/pycore_symtable.h b/Include/internal/pycore_symtable.h index 88725a2a3fd343..c1aee56009e162 100644 --- a/Include/internal/pycore_symtable.h +++ b/Include/internal/pycore_symtable.h @@ -177,7 +177,6 @@ _Py_IsPrivateName(PyObject *); #define DEF_ANNOT (2<<7) /* this name is annotated */ #define DEF_COMP_ITER (2<<8) /* this name is a comprehension iteration variable */ #define DEF_TYPE_PARAM (2<<9) /* this name is a type parameter */ -#define DEF_COMP_CELL (2<<10) /* this name is a cell in an inlined comprehension */ #define DEF_BOUND (DEF_LOCAL | DEF_PARAM | DEF_IMPORT) diff --git a/Lib/symtable.py b/Lib/symtable.py index 3d9c4f6b6ea983..3e56e8ab99c1c3 100644 --- a/Lib/symtable.py +++ b/Lib/symtable.py @@ -7,7 +7,7 @@ DEF_NONLOCAL, DEF_LOCAL, DEF_PARAM, DEF_TYPE_PARAM, DEF_FREE_CLASS, DEF_IMPORT, DEF_BOUND, DEF_ANNOT, - DEF_COMP_ITER, DEF_COMP_CELL, + DEF_COMP_ITER, SCOPE_OFF, SCOPE_MASK, FREE, LOCAL, GLOBAL_IMPLICIT, GLOBAL_EXPLICIT, CELL ) @@ -154,8 +154,10 @@ def lookup(self, name): flags = self._table.symbols[name] namespaces = self.__check_children(name) module_scope = (self._table.name == "top") + inlined = (self._table.type == _symtable.TYPE_INLINED_COMPREHENSION) sym = self._symbols[name] = Symbol(name, flags, namespaces, - module_scope=module_scope) + module_scope=module_scope, + inlined_comprehension=inlined) return sym def get_symbols(self): @@ -249,12 +251,14 @@ class Class(SymbolTable): class Symbol: - def __init__(self, name, flags, namespaces=None, *, module_scope=False): + def __init__(self, name, flags, namespaces=None, *, module_scope=False, + inlined_comprehension=False): self.__name = name self.__flags = flags self.__scope = _get_scope(flags) self.__namespaces = namespaces or () self.__module_scope = module_scope + self.__inlined_comprehension = inlined_comprehension def __repr__(self): flags_str = '|'.join(self._flags_str()) @@ -348,7 +352,7 @@ def is_comp_iter(self): def is_comp_cell(self): """Return *True* if the symbol is a cell in an inlined comprehension. """ - return bool(self.__flags & DEF_COMP_CELL) + return self.is_cell() and self.__inlined_comprehension def is_namespace(self): """Returns *True* if name binding introduces new namespace. diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py index 65213c53131b5b..ab80ac6330ec2f 100644 --- a/Lib/test/test_listcomps.py +++ b/Lib/test/test_listcomps.py @@ -277,6 +277,20 @@ def f(): outputs = {"y": [1]} self._check_in_scopes(code, outputs, scopes=["module", "function"]) + def test_inlined_comp_cell_with_enclosing_free(self): + # The listcomp cell and the enclosing free must not share an index. + code = """ + def outer(y): + def inner(): + return [lambda: x for x in (1, 2)], y + return inner() + funcs, val = outer(99) + z = [f() for f in funcs] + w = val + """ + outputs = {"z": [2, 2], "w": 99} + self._check_in_scopes(code, outputs) + def test_free_inner_cell_outer(self): code = """ g = 2 diff --git a/Lib/test/test_symtable.py b/Lib/test/test_symtable.py index 68e873f8addd04..6609017141f721 100644 --- a/Lib/test/test_symtable.py +++ b/Lib/test/test_symtable.py @@ -431,7 +431,7 @@ def test_symbol_repr(self): st2 = symtable.symtable("[(lambda: x) for x in [1]]", "?", "exec") self.assertEqual(repr(st2.get_children()[0].lookup("x")), - "") + "") st3 = symtable.symtable("def f():\n" " x = 1\n" @@ -556,6 +556,20 @@ def test_inlined_comprehension_use_of_enclosing_free_in_function(self): self.assertTrue(comp.lookup("x").is_free()) self.assertTrue(comp.lookup("x").is_referenced()) + def test_inlined_comprehension_comp_cell_not_on_enclosing(self): + st = symtable.symtable( + "def f():\n" + " x = 1\n" + " return [(lambda: x) for x in [1]]", + "?", "exec") + f = find_block(st, "f") + self.assertTrue(f.lookup("x").is_cell()) + self.assertFalse(f.lookup("x").is_comp_cell()) + comp, = (c for c in f.get_children() + if c.get_type() is symtable.SymbolTableType.INLINED_COMPREHENSION) + self.assertTrue(comp.lookup("x").is_cell()) + self.assertTrue(comp.lookup("x").is_comp_cell()) + def test_inlined_comprehension_use_of_enclosing_free_in_class(self): st = symtable.symtable( "def f():\n" @@ -600,8 +614,10 @@ def test_inlined_nested_comprehension_class_iter_var(self): children[0], ["x"], ["_", "x"], nested=False) self.assertFalse(C.lookup("x").is_free()) self.assertTrue(C.lookup("x").is_local()) + self.assertFalse(C.lookup("x").is_comp_cell()) self.assertFalse(children[0].lookup("x").is_free()) self.assertTrue(children[0].lookup("x").is_cell()) + self.assertTrue(children[0].lookup("x").is_comp_cell()) self.assertFalse(inner.lookup("_").is_free()) self.assertTrue(inner.lookup("x").is_free()) self.assertTrue(inner.lookup("x").is_referenced()) diff --git a/Modules/symtablemodule.c b/Modules/symtablemodule.c index 0ce5b73add9524..3028158fc917e1 100644 --- a/Modules/symtablemodule.c +++ b/Modules/symtablemodule.c @@ -128,7 +128,6 @@ symtable_init_constants(PyObject *m) if (PyModule_AddIntMacro(m, DEF_BOUND) < 0) return -1; if (PyModule_AddIntMacro(m, DEF_ANNOT) < 0) return -1; if (PyModule_AddIntMacro(m, DEF_COMP_ITER) < 0) return -1; - if (PyModule_AddIntMacro(m, DEF_COMP_CELL) < 0) return -1; if (PyModule_AddIntConstant(m, "TYPE_FUNCTION", FunctionBlock) < 0) return -1; diff --git a/Python/compile.c b/Python/compile.c index 74697e67e7e891..bbd21281ce5a57 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -595,6 +595,75 @@ dictbytype(PyObject *src, int scope_type, int flag, Py_ssize_t offset) return dest; } +static int +add_cell_names_from_symbols(PyObject *symbols, PyObject *names) +{ + Py_ssize_t pos = 0; + PyObject *k, *v; + while (PyDict_Next(symbols, &pos, &k, &v)) { + long flags = PyLong_AsLong(v); + if (flags == -1 && PyErr_Occurred()) { + return ERROR; + } + if (SYMBOL_TO_SCOPE(flags) == CELL) { + if (PySet_Add(names, k) < 0) { + return ERROR; + } + } + } + return SUCCESS; +} + +static int +add_inlined_comprehension_cell_names(PySTEntryObject *ste, PyObject *names) +{ + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(ste->ste_children); i++) { + PySTEntryObject *child = + (PySTEntryObject *)PyList_GET_ITEM(ste->ste_children, i); + if (child->ste_type != InlinedComprehensionBlock) { + continue; + } + if (add_cell_names_from_symbols(child->ste_symbols, names) < 0) { + return ERROR; + } + if (add_inlined_comprehension_cell_names(child, names) < 0) { + return ERROR; + } + } + return SUCCESS; +} + +/* Cells of the shared unit: this table's CELL names, plus cells that live + * only on inlined comprehension children. */ +static PyObject * +compiler_cellvars(PySTEntryObject *ste) +{ + PyObject *names = PySet_New(NULL); + if (names == NULL) { + return NULL; + } + if (add_cell_names_from_symbols(ste->ste_symbols, names) < 0) { + Py_DECREF(names); + return NULL; + } + if (add_inlined_comprehension_cell_names(ste, names) < 0) { + Py_DECREF(names); + return NULL; + } + PyObject *sorted = PySequence_List(names); + Py_DECREF(names); + if (sorted == NULL) { + return NULL; + } + if (PyList_Sort(sorted) < 0) { + Py_DECREF(sorted); + return NULL; + } + PyObject *cellvars = list2dict(sorted); + Py_DECREF(sorted); + return cellvars; +} + int _PyCompile_EnterScope(compiler *c, identifier name, int scope_type, void *key, int lineno, PyObject *private, @@ -626,7 +695,7 @@ _PyCompile_EnterScope(compiler *c, identifier name, int scope_type, compiler_unit_free(u); return ERROR; } - u->u_metadata.u_cellvars = dictbytype(u->u_ste->ste_symbols, CELL, DEF_COMP_CELL, 0); + u->u_metadata.u_cellvars = compiler_cellvars(u->u_ste); if (!u->u_metadata.u_cellvars) { compiler_unit_free(u); return ERROR; diff --git a/Python/symtable.c b/Python/symtable.c index aa1e5ff6d89001..94a9785669b0e1 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -361,7 +361,6 @@ static void _dump_symtable(PySTEntryObject* ste, PyObject* prefix) if (flags & DEF_ANNOT) printf(" DEF_ANNOT"); if (flags & DEF_COMP_ITER) printf(" DEF_COMP_ITER"); if (flags & DEF_TYPE_PARAM) printf(" DEF_TYPE_PARAM"); - if (flags & DEF_COMP_CELL) printf(" DEF_COMP_CELL"); switch (scope) { case LOCAL: printf(" LOCAL"); break; case GLOBAL_EXPLICIT: printf(" GLOBAL_EXPLICIT"); break; @@ -860,23 +859,10 @@ finalize_inlined_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, goto error; } int scope = SYMBOL_TO_SCOPE(comp_flags); - int only_flags = comp_flags & ((1 << SCOPE_OFFSET) - 1); - if (scope == CELL || only_flags & DEF_COMP_CELL) { + if (scope == CELL) { if (PySet_Add(inlined_cells, k) < 0) { goto error; } - if (!(only_flags & DEF_COMP_CELL)) { - comp_flags |= DEF_COMP_CELL; - PyObject *newv = PyLong_FromLong(comp_flags); - if (newv == NULL) { - goto error; - } - if (PyDict_SetItem(comp->ste_symbols, k, newv) < 0) { - Py_DECREF(newv); - goto error; - } - Py_DECREF(newv); - } } PyObject *existing = PyDict_GetItemWithError(ste->ste_symbols, k); if (existing == NULL && PyErr_Occurred()) { @@ -1037,7 +1023,7 @@ drop_class_free(PySTEntryObject *ste, PyObject *free) static int update_symbols(PyObject *symbols, PyObject *scopes, PyObject *bound, PyObject *free, - PyObject *inlined_cells, int classflag) + int classflag) { PyObject *name = NULL, *itr = NULL; PyObject *v = NULL, *v_scope = NULL, *v_new = NULL, *v_free = NULL; @@ -1049,13 +1035,6 @@ update_symbols(PyObject *symbols, PyObject *scopes, if (flags == -1 && PyErr_Occurred()) { return 0; } - int contains = PySet_Contains(inlined_cells, name); - if (contains < 0) { - return 0; - } - if (contains) { - flags |= DEF_COMP_CELL; - } if (PyDict_GetItemRef(scopes, name, &v_scope) < 0) { return 0; } @@ -1336,7 +1315,7 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, goto error; } /* Records the results of the analysis in the symbol table entry */ - if (!update_symbols(ste->ste_symbols, scopes, bound, newfree, inlined_cells, + if (!update_symbols(ste->ste_symbols, scopes, bound, newfree, (ste->ste_type == ClassBlock) || ste->ste_can_see_class_scope)) goto error; temp = PyNumber_InPlaceOr(free, newfree);