[mypyc] Make for loop over list memory safe on free-threaded builds

Author: JukkaL

mypyc/irbuild/for_helpers.py

@@ -28,6 +28,7 @@
     Var,
 )
 from mypy.types import LiteralType, TupleType, get_proper_type, get_proper_types
+from mypyc.common import IS_FREE_THREADED
 from mypyc.ir.ops import (
     ERR_NEVER,
     BasicBlock,
@@ -81,7 +82,12 @@
 )
 from mypyc.primitives.exc_ops import no_err_occurred_op, propagate_if_error_op
 from mypyc.primitives.generic_ops import aiter_op, anext_op, iter_op, next_op
-from mypyc.primitives.list_ops import list_append_op, list_get_item_unsafe_op, new_list_set_item_op
+from mypyc.primitives.list_ops import (
+    list_append_op,
+    list_get_item_int64_op,
+    list_get_item_unsafe_op,
+    new_list_set_item_op,
+)
 from mypyc.primitives.misc_ops import stop_async_iteration_op
 from mypyc.primitives.registry import CFunctionDescription
 from mypyc.primitives.set_ops import set_add_op
@@ -866,7 +872,10 @@ def unsafe_index(builder: IRBuilder, target: Value, index: Value, line: int) ->
     # since we want to use __getitem__ if we don't have an unsafe version,
     # so we just check manually.
     if is_list_rprimitive(target.type):
-        return builder.primitive_op(list_get_item_unsafe_op, [target, index], line)
+        if not IS_FREE_THREADED:
+            return builder.primitive_op(list_get_item_unsafe_op, [target, index], line)
+        else:
+            return builder.primitive_op(list_get_item_int64_op, [target, index], line)
     elif is_tuple_rprimitive(target.type):
         return builder.call_c(tuple_get_item_unsafe_op, [target, index], line)
     elif is_str_rprimitive(target.type):

mypyc/test-data/irbuild-basic.test

@@ -1870,7 +1870,7 @@ L0:
     r5 = a.f(12, 6, r4)
     return 1
 
-[case testListComprehension]
+[case testListComprehension_withgil]
 from typing import List
 
 def f() -> List[int]:
@@ -1931,7 +1931,7 @@ L7:
 L8:
     return r0
 
-[case testDictComprehension]
+[case testDictComprehension_withgil]
 from typing import Dict
 def f() -> Dict[int, int]:
     return {x: x*x for x in [1,2,3] if x != 2 if x != 3}
@@ -1993,7 +1993,7 @@ L7:
 L8:
     return r0
 
-[case testLoopsMultipleAssign]
+[case testLoopsMultipleAssign_withgil]
 from typing import List, Tuple
 def f(l: List[Tuple[int, int, int]]) -> List[int]:
     for x, y, z in l:

mypyc/test-data/irbuild-lists.test

@@ -379,7 +379,7 @@ L0:
     r2 = r1 >= 0 :: signed
     return 1
 
-[case testListBuiltFromGenerator]
+[case testListBuiltFromGenerator_withgil]
 from typing import List
 def f(source: List[int]) -> None:
     a = list(x + 1 for x in source)
@@ -448,7 +448,7 @@ L8:
     b = r11
     return 1
 
-[case testGeneratorNext]
+[case testGeneratorNext_withgil]
 from typing import List, Optional
 
 def test(x: List[int]) -> None:
@@ -489,7 +489,7 @@ L5:
     res = r6
     return 1
 
-[case testSimplifyListUnion]
+[case testSimplifyListUnion_withgil]
 from typing import List, Union, Optional
 
 def narrow(a: Union[List[str], List[bytes], int]) -> int:
@@ -923,7 +923,7 @@ L10:
     a = r3
     return 1
 
-[case testListBuiltFromStars]
+[case testListBuiltFromStars_withgil]
 from typing import Final
 
 abc: Final = "abc"

mypyc/test-data/irbuild-set.test

@@ -53,7 +53,7 @@ L0:
     r0 = PySet_New(l)
     return r0
 
-[case testNewSetFromIterable2]
+[case testNewSetFromIterable2_withgil]
 def f(x: int) -> int:
     return x
 
@@ -273,7 +273,7 @@ L4:
     e = r0
     return 1
 
-[case testNewSetFromIterable3]
+[case testNewSetFromIterable3_withgil]
 def f1(x: int) -> int:
     return x
 

mypyc/test-data/irbuild-statements.test

@@ -243,7 +243,7 @@ L1:
     r2 = r0 < r1 :: signed
     if r2 goto L2 else goto L4 :: bool
 L2:
-    r3 = list_get_item_unsafe ls, r0
+    r3 = CPyList_GetItemInt64(ls, r0)
     r4 = unbox(int, r3)
     x = r4
     r5 = CPyTagged_Add(y, x)
@@ -899,7 +899,7 @@ L0:
     r6 = r5 >= 0 :: signed
     return 1
 
-[case testForEnumerate]
+[case testForEnumerate_withgil]
 from typing import List, Iterable
 
 def f(a: List[int]) -> None:
@@ -967,7 +967,7 @@ L4:
 L5:
     return 1
 
-[case testForZip]
+[case testForZip_withgil]
 from typing import List, Iterable, Sequence
 
 def f(a: List[int], b: Sequence[bool]) -> None:

mypyc/test-data/irbuild-tuple.test

@@ -281,7 +281,7 @@ L5:
 L6:
     return r6
 
-[case testTupleBuiltFromList]
+[case testTupleBuiltFromList_withgil]
 def f(val: int) -> bool:
     return val % 2 == 0
 
@@ -1018,7 +1018,7 @@ L5:
     a = r42
     return 1
 
-[case testTupleBuiltFromStars]
+[case testTupleBuiltFromStars_withgil]
 from typing import Final
 
 abc: Final = "abc"

mypyc/test-data/irbuild-vec-i64.test

@@ -334,7 +334,7 @@ L3:
 L4:
     return r1
 
-[case testVecI64FastComprehensionFromList]
+[case testVecI64FastComprehensionFromList_nogil]
 from librt.vecs import vec
 from mypy_extensions import i64
 from typing import List
@@ -364,7 +364,7 @@ L1:
     r4 = r2 < r3 :: signed
     if r4 goto L2 else goto L4 :: bool
 L2:
-    r5 = list_get_item_unsafe l, r2
+    r5 = CPyList_GetItemInt64(l, r2)
     r6 = unbox(i64, r5)
     x = r6
     r7 = x + 1

mypyc/test-data/lowering-int.test

@@ -332,7 +332,7 @@ L4:
 L5:
     return 4
 
-[case testLowerIntForLoop_64bit]
+[case testLowerIntForLoop_withgil_64bit]
 from __future__ import annotations
 
 def f(l: list[int]) -> None:

mypyc/test/test_lowering.py

@@ -7,7 +7,7 @@
 from mypy.errors import CompileError
 from mypy.test.config import test_temp_dir
 from mypy.test.data import DataDrivenTestCase
-from mypyc.common import TOP_LEVEL_NAME
+from mypyc.common import IS_FREE_THREADED, TOP_LEVEL_NAME
 from mypyc.ir.pprint import format_func
 from mypyc.options import CompilerOptions
 from mypyc.test.testutil import (
@@ -36,6 +36,12 @@ def run_case(self, testcase: DataDrivenTestCase) -> None:
         if options is None:
             # Skipped test case
             return
+        if "_withgil" in testcase.name and IS_FREE_THREADED:
+            # Test case should only run on a non-free-threaded build.
+            return
+        if "_nogil" in testcase.name and not IS_FREE_THREADED:
+            # Test case should only run on a free-threaded build.
+            return
         with use_custom_builtins(os.path.join(self.data_prefix, ICODE_GEN_BUILTINS), testcase):
             expected_output = remove_comment_lines(testcase.output)
             expected_output = replace_word_size(expected_output)