From 814a28bfe9e0e3b4372e4d17b5ab4fd5c3a5d204 Mon Sep 17 00:00:00 2001 From: Neil Schemenauer Date: Tue, 10 Nov 2026 21:06:50 -0700 Subject: [PATCH] gh-161792: Fix crashes in the callable iterator calliter_iternext() used borrowed references to it_callable and it_sentinel, which could be freed by __setstate__() during the call or comparison (gh-158031), or by another thread in the free-threaded build. Take strong references when reading the fields, and update them with an atomic exchange plus a delayed DECREF. --- .../test_free_threading/test_iteration.py | 26 +++++ Lib/test/test_iter.py | 17 +++ ...1-11-04-58-51.gh-issue-161792.42EcBI8x.rst | 4 + Objects/iterobject.c | 102 ++++++++++++------ 4 files changed, 119 insertions(+), 30 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-11-11-04-58-51.gh-issue-161792.42EcBI8x.rst diff --git a/Lib/test/test_free_threading/test_iteration.py b/Lib/test/test_free_threading/test_iteration.py index 540e8edcc67628..bf42304b41e8b9 100644 --- a/Lib/test/test_free_threading/test_iteration.py +++ b/Lib/test/test_free_threading/test_iteration.py @@ -1,3 +1,4 @@ +import itertools import sys import threading import unittest @@ -170,6 +171,31 @@ def drain(it, barrier, results): self.assertEqual(sys.getrefcount(seq), refcount_before) +class ContendedCallIterExhaustionTest(unittest.TestCase): + """Test exhausting a shared iter(callable, sentinel) iterator. + + The values seen by each thread are unspecified, but exhausting the + iterator from several threads at once must not crash. + """ + + def test_shared_iterator_exhaustion(self): + niters = 2000 if support.check_sanitizer(thread=True) else 20000 + + def make_iter(): + counter = itertools.count().__next__ + return iter(lambda: counter(), 3) + + shared = [make_iter()] + + def worker(): + for _ in range(niters): + it = shared[0] + if next(it, None) is None: + shared[0] = make_iter() + + threading_helper.run_concurrently(worker, nthreads=NUMTHREADS) + + class ContendedRangeIterationTest(ContendedTupleIterationTest): def make_testdata(self, n): return range(n) diff --git a/Lib/test/test_iter.py b/Lib/test/test_iter.py index fe8617309da29d..6ebde4848a5a49 100644 --- a/Lib/test/test_iter.py +++ b/Lib/test/test_iter.py @@ -532,6 +532,23 @@ def spam(): with self.assertRaises(StopIteration): next(spam.iterator) + def test_iter_function_sentinel_replaced_during_compare(self): + # gh-158031: the sentinel's __eq__ replaces the sentinel, which must + # stay alive until the comparison is finished. + class Sentinel: + def __eq__(self, other): + it.__setstate__(((), StopIteration)) + return NotImplemented + + class Result: + def __eq__(self, other): + self.other_type = type(other) + return NotImplemented + + it = iter(Result, Sentinel()) + result = next(it) + self.assertIs(result.other_type, Sentinel) + # Test exception propagation through function iterator def test_exception_function(self): def spam(state=[0]): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-11-11-04-58-51.gh-issue-161792.42EcBI8x.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-11-04-58-51.gh-issue-161792.42EcBI8x.rst new file mode 100644 index 00000000000000..a4b685d96bc896 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-11-04-58-51.gh-issue-161792.42EcBI8x.rst @@ -0,0 +1,4 @@ +Fix crashes in the callable iterator returned by :func:`iter` with a +sentinel, when :meth:`!__setstate__` is called while the iterator is in use, +or when the iterator is shared between threads in the :term:`free-threaded +build`. diff --git a/Objects/iterobject.c b/Objects/iterobject.c index fcef1cc9c30143..683683ceaa6600 100644 --- a/Objects/iterobject.c +++ b/Objects/iterobject.c @@ -238,11 +238,33 @@ PyCallIter_New(PyObject *callable, PyObject *sentinel) return _PyCallIter_NewEx(callable, sentinel, NULL); } +// Sharing a callable iterator between threads, or calling __setstate__() +// on an iterator that is in use, is not supported: the values produced are +// unspecified, but the interpreter must not crash. + +static inline PyObject * +calliter_swap(PyObject **field, PyObject *value) +{ +#ifdef Py_GIL_DISABLED + return _Py_atomic_exchange_ptr(field, value); +#else + PyObject *old = *field; + *field = value; + return old; +#endif +} + +static inline PyObject * +calliter_load(PyObject **field) +{ + return Py_XNewRef(FT_ATOMIC_LOAD_PTR_ACQUIRE(*field)); +} + static void calliter_exhaust(calliterobject *it) { - Py_CLEAR(it->it_callable); - Py_CLEAR(it->it_sentinel); + _PyObject_XDecRefDelayed(calliter_swap(&it->it_callable, NULL)); + _PyObject_XDecRefDelayed(calliter_swap(&it->it_sentinel, NULL)); } static void @@ -270,19 +292,26 @@ static PyObject * calliter_iternext(PyObject *op) { calliterobject *it = (calliterobject*)op; - PyObject *result; + PyObject *callable, *sentinel, *stop_exc, *result; - if (it->it_callable == NULL) { + callable = calliter_load(&it->it_callable); + if (callable == NULL) { return NULL; } + result = _PyObject_CallNoArgs(callable); + Py_DECREF(callable); - result = _PyObject_CallNoArgs(it->it_callable); /* The call can exhaust the iterator re-entrantly. */ - if (result != NULL && it->it_callable != NULL) { - if (it->it_sentinel == NULL) { + if (result != NULL && + FT_ATOMIC_LOAD_PTR_RELAXED(it->it_callable) != NULL) + { + sentinel = calliter_load(&it->it_sentinel); + if (sentinel == NULL) { return result; /* Common case, fast path */ } - int ok = PyObject_RichCompareBool(it->it_sentinel, result, Py_EQ); + /* The comparison can run code that replaces the sentinel. */ + int ok = PyObject_RichCompareBool(sentinel, result, Py_EQ); + Py_DECREF(sentinel); if (ok == 0) { return result; /* Common case, fast path */ } @@ -291,14 +320,19 @@ calliter_iternext(PyObject *op) calliter_exhaust(it); } } - else if (PyErr_ExceptionMatches(it->it_stop_exc)) { - PyErr_Clear(); - calliter_exhaust(it); - } - else if (PyErr_ExceptionMatches(PyExc_StopIteration)) { - /* It would be mistaken for the end of the iteration (see PEP 479). */ - _PyErr_FormatFromCause(PyExc_RuntimeError, - "callable raised StopIteration"); + else if (result == NULL) { + stop_exc = calliter_load(&it->it_stop_exc); + if (PyErr_ExceptionMatches(stop_exc)) { + PyErr_Clear(); + calliter_exhaust(it); + } + else if (PyErr_ExceptionMatches(PyExc_StopIteration)) { + /* It would be mistaken for the end of the iteration (see + PEP 479). */ + _PyErr_FormatFromCause(PyExc_RuntimeError, + "callable raised StopIteration"); + } + Py_DECREF(stop_exc); } Py_XDECREF(result); return NULL; @@ -314,22 +348,29 @@ calliter_reduce(PyObject *op, PyObject *Py_UNUSED(ignored)) * call must be before access of iterator pointers. * see issue #101765 */ - if (it->it_callable == NULL) { - return Py_BuildValue("N(())", iter); + PyObject *callable = calliter_load(&it->it_callable); + PyObject *sentinel = calliter_load(&it->it_sentinel); + PyObject *stop_exc = calliter_load(&it->it_stop_exc); + PyObject *res; + if (callable == NULL) { + res = Py_BuildValue("N(())", iter); } /* Only the sentinel can be passed as an argument of iter(), so other attributes are restored from the state (see calliter_setstate()). */ - if (it->it_sentinel == NULL) { - return Py_BuildValue("N(OO)(()O)", iter, it->it_callable, Py_None, - it->it_stop_exc); + else if (sentinel == NULL) { + res = Py_BuildValue("N(OO)(()O)", iter, callable, Py_None, stop_exc); } - else if (it->it_stop_exc == PyExc_StopIteration) { - return Py_BuildValue("N(OO)", iter, it->it_callable, it->it_sentinel); + else if (stop_exc == PyExc_StopIteration) { + res = Py_BuildValue("N(OO)", iter, callable, sentinel); } else { - return Py_BuildValue("N(OO)((O)O)", iter, it->it_callable, Py_None, - it->it_sentinel, it->it_stop_exc); + res = Py_BuildValue("N(OO)((O)O)", iter, callable, Py_None, + sentinel, stop_exc); } + Py_XDECREF(callable); + Py_XDECREF(sentinel); + Py_DECREF(stop_exc); + return res; } static PyObject * @@ -349,11 +390,12 @@ calliter_setstate(PyObject *op, PyObject *state) if (_PyEval_CheckExceptTypeValid(_PyThreadState_GET(), stop_exc) < 0) { return NULL; } - if (it->it_callable != NULL) { - Py_XSETREF(it->it_sentinel, - PyTuple_GET_SIZE(sentinel) ? - Py_NewRef(PyTuple_GET_ITEM(sentinel, 0)) : NULL); - Py_SETREF(it->it_stop_exc, Py_NewRef(stop_exc)); + if (FT_ATOMIC_LOAD_PTR_RELAXED(it->it_callable) != NULL) { + PyObject *value = PyTuple_GET_SIZE(sentinel) ? + Py_NewRef(PyTuple_GET_ITEM(sentinel, 0)) : NULL; + _PyObject_XDecRefDelayed(calliter_swap(&it->it_sentinel, value)); + _PyObject_XDecRefDelayed(calliter_swap(&it->it_stop_exc, + Py_NewRef(stop_exc))); } Py_RETURN_NONE;