Skip to content

Commit 814a28b

Browse files
committed
gh-161792: Fix crashes in the callable iterator
calliter_iternext() used borrowed references to it_callable and it_sentinel, which could be freed by __setstate__() during the call or comparison (gh-158031), or by another thread in the free-threaded build. Take strong references when reading the fields, and update them with an atomic exchange plus a delayed DECREF.
1 parent acab532 commit 814a28b

4 files changed

Lines changed: 119 additions & 30 deletions

File tree

‎Lib/test/test_free_threading/test_iteration.py‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import itertools
12
import sys
23
import threading
34
import unittest
@@ -170,6 +171,31 @@ def drain(it, barrier, results):
170171
self.assertEqual(sys.getrefcount(seq), refcount_before)
171172

172173

174+
class ContendedCallIterExhaustionTest(unittest.TestCase):
175+
"""Test exhausting a shared iter(callable, sentinel) iterator.
176+
177+
The values seen by each thread are unspecified, but exhausting the
178+
iterator from several threads at once must not crash.
179+
"""
180+
181+
def test_shared_iterator_exhaustion(self):
182+
niters = 2000 if support.check_sanitizer(thread=True) else 20000
183+
184+
def make_iter():
185+
counter = itertools.count().__next__
186+
return iter(lambda: counter(), 3)
187+
188+
shared = [make_iter()]
189+
190+
def worker():
191+
for _ in range(niters):
192+
it = shared[0]
193+
if next(it, None) is None:
194+
shared[0] = make_iter()
195+
196+
threading_helper.run_concurrently(worker, nthreads=NUMTHREADS)
197+
198+
173199
class ContendedRangeIterationTest(ContendedTupleIterationTest):
174200
def make_testdata(self, n):
175201
return range(n)

‎Lib/test/test_iter.py‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -532,6 +532,23 @@ def spam():
532532
with self.assertRaises(StopIteration):
533533
next(spam.iterator)
534534

535+
def test_iter_function_sentinel_replaced_during_compare(self):
536+
# gh-158031: the sentinel's __eq__ replaces the sentinel, which must
537+
# stay alive until the comparison is finished.
538+
class Sentinel:
539+
def __eq__(self, other):
540+
it.__setstate__(((), StopIteration))
541+
return NotImplemented
542+
543+
class Result:
544+
def __eq__(self, other):
545+
self.other_type = type(other)
546+
return NotImplemented
547+
548+
it = iter(Result, Sentinel())
549+
result = next(it)
550+
self.assertIs(result.other_type, Sentinel)
551+
535552
# Test exception propagation through function iterator
536553
def test_exception_function(self):
537554
def spam(state=[0]):
Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
Fix crashes in the callable iterator returned by :func:`iter` with a
2+
sentinel, when :meth:`!__setstate__` is called while the iterator is in use,
3+
or when the iterator is shared between threads in the :term:`free-threaded
4+
build`.

‎Objects/iterobject.c‎

Lines changed: 72 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -238,11 +238,33 @@ PyCallIter_New(PyObject *callable, PyObject *sentinel)
238238
return _PyCallIter_NewEx(callable, sentinel, NULL);
239239
}
240240

241+
// Sharing a callable iterator between threads, or calling __setstate__()
242+
// on an iterator that is in use, is not supported: the values produced are
243+
// unspecified, but the interpreter must not crash.
244+
245+
static inline PyObject *
246+
calliter_swap(PyObject **field, PyObject *value)
247+
{
248+
#ifdef Py_GIL_DISABLED
249+
return _Py_atomic_exchange_ptr(field, value);
250+
#else
251+
PyObject *old = *field;
252+
*field = value;
253+
return old;
254+
#endif
255+
}
256+
257+
static inline PyObject *
258+
calliter_load(PyObject **field)
259+
{
260+
return Py_XNewRef(FT_ATOMIC_LOAD_PTR_ACQUIRE(*field));
261+
}
262+
241263
static void
242264
calliter_exhaust(calliterobject *it)
243265
{
244-
Py_CLEAR(it->it_callable);
245-
Py_CLEAR(it->it_sentinel);
266+
_PyObject_XDecRefDelayed(calliter_swap(&it->it_callable, NULL));
267+
_PyObject_XDecRefDelayed(calliter_swap(&it->it_sentinel, NULL));
246268
}
247269

248270
static void
@@ -270,19 +292,26 @@ static PyObject *
270292
calliter_iternext(PyObject *op)
271293
{
272294
calliterobject *it = (calliterobject*)op;
273-
PyObject *result;
295+
PyObject *callable, *sentinel, *stop_exc, *result;
274296

275-
if (it->it_callable == NULL) {
297+
callable = calliter_load(&it->it_callable);
298+
if (callable == NULL) {
276299
return NULL;
277300
}
301+
result = _PyObject_CallNoArgs(callable);
302+
Py_DECREF(callable);
278303

279-
result = _PyObject_CallNoArgs(it->it_callable);
280304
/* The call can exhaust the iterator re-entrantly. */
281-
if (result != NULL && it->it_callable != NULL) {
282-
if (it->it_sentinel == NULL) {
305+
if (result != NULL &&
306+
FT_ATOMIC_LOAD_PTR_RELAXED(it->it_callable) != NULL)
307+
{
308+
sentinel = calliter_load(&it->it_sentinel);
309+
if (sentinel == NULL) {
283310
return result; /* Common case, fast path */
284311
}
285-
int ok = PyObject_RichCompareBool(it->it_sentinel, result, Py_EQ);
312+
/* The comparison can run code that replaces the sentinel. */
313+
int ok = PyObject_RichCompareBool(sentinel, result, Py_EQ);
314+
Py_DECREF(sentinel);
286315
if (ok == 0) {
287316
return result; /* Common case, fast path */
288317
}
@@ -291,14 +320,19 @@ calliter_iternext(PyObject *op)
291320
calliter_exhaust(it);
292321
}
293322
}
294-
else if (PyErr_ExceptionMatches(it->it_stop_exc)) {
295-
PyErr_Clear();
296-
calliter_exhaust(it);
297-
}
298-
else if (PyErr_ExceptionMatches(PyExc_StopIteration)) {
299-
/* It would be mistaken for the end of the iteration (see PEP 479). */
300-
_PyErr_FormatFromCause(PyExc_RuntimeError,
301-
"callable raised StopIteration");
323+
else if (result == NULL) {
324+
stop_exc = calliter_load(&it->it_stop_exc);
325+
if (PyErr_ExceptionMatches(stop_exc)) {
326+
PyErr_Clear();
327+
calliter_exhaust(it);
328+
}
329+
else if (PyErr_ExceptionMatches(PyExc_StopIteration)) {
330+
/* It would be mistaken for the end of the iteration (see
331+
PEP 479). */
332+
_PyErr_FormatFromCause(PyExc_RuntimeError,
333+
"callable raised StopIteration");
334+
}
335+
Py_DECREF(stop_exc);
302336
}
303337
Py_XDECREF(result);
304338
return NULL;
@@ -314,22 +348,29 @@ calliter_reduce(PyObject *op, PyObject *Py_UNUSED(ignored))
314348
* call must be before access of iterator pointers.
315349
* see issue #101765 */
316350

317-
if (it->it_callable == NULL) {
318-
return Py_BuildValue("N(())", iter);
351+
PyObject *callable = calliter_load(&it->it_callable);
352+
PyObject *sentinel = calliter_load(&it->it_sentinel);
353+
PyObject *stop_exc = calliter_load(&it->it_stop_exc);
354+
PyObject *res;
355+
if (callable == NULL) {
356+
res = Py_BuildValue("N(())", iter);
319357
}
320358
/* Only the sentinel can be passed as an argument of iter(), so other
321359
attributes are restored from the state (see calliter_setstate()). */
322-
if (it->it_sentinel == NULL) {
323-
return Py_BuildValue("N(OO)(()O)", iter, it->it_callable, Py_None,
324-
it->it_stop_exc);
360+
else if (sentinel == NULL) {
361+
res = Py_BuildValue("N(OO)(()O)", iter, callable, Py_None, stop_exc);
325362
}
326-
else if (it->it_stop_exc == PyExc_StopIteration) {
327-
return Py_BuildValue("N(OO)", iter, it->it_callable, it->it_sentinel);
363+
else if (stop_exc == PyExc_StopIteration) {
364+
res = Py_BuildValue("N(OO)", iter, callable, sentinel);
328365
}
329366
else {
330-
return Py_BuildValue("N(OO)((O)O)", iter, it->it_callable, Py_None,
331-
it->it_sentinel, it->it_stop_exc);
367+
res = Py_BuildValue("N(OO)((O)O)", iter, callable, Py_None,
368+
sentinel, stop_exc);
332369
}
370+
Py_XDECREF(callable);
371+
Py_XDECREF(sentinel);
372+
Py_DECREF(stop_exc);
373+
return res;
333374
}
334375

335376
static PyObject *
@@ -349,11 +390,12 @@ calliter_setstate(PyObject *op, PyObject *state)
349390
if (_PyEval_CheckExceptTypeValid(_PyThreadState_GET(), stop_exc) < 0) {
350391
return NULL;
351392
}
352-
if (it->it_callable != NULL) {
353-
Py_XSETREF(it->it_sentinel,
354-
PyTuple_GET_SIZE(sentinel) ?
355-
Py_NewRef(PyTuple_GET_ITEM(sentinel, 0)) : NULL);
356-
Py_SETREF(it->it_stop_exc, Py_NewRef(stop_exc));
393+
if (FT_ATOMIC_LOAD_PTR_RELAXED(it->it_callable) != NULL) {
394+
PyObject *value = PyTuple_GET_SIZE(sentinel) ?
395+
Py_NewRef(PyTuple_GET_ITEM(sentinel, 0)) : NULL;
396+
_PyObject_XDecRefDelayed(calliter_swap(&it->it_sentinel, value));
397+
_PyObject_XDecRefDelayed(calliter_swap(&it->it_stop_exc,
398+
Py_NewRef(stop_exc)));
357399
}
358400
Py_RETURN_NONE;
359401

0 commit comments

Comments
 (0)