gh-129724: Make marshalled code output deterministic

Include/internal/pycore_compile.h

@@ -201,6 +201,10 @@ int _PyCodegen_Module(struct _PyCompiler *c, _Py_SourceLocation loc, asdl_stmt_s
                       bool is_interactive);
 
 int _PyCompile_ConstCacheMergeOne(PyObject *const_cache, PyObject **obj);
+int _PyCompile_ConstCacheMergeOneLocal(PyObject *const_cache,
+                                       PyObject *local_const_cache,
+                                       PyObject **obj);
+int _PyCompile_ConstCacheAddLocal(PyObject *local_const_cache, PyObject *obj);
 
 PyCodeObject *_PyCompile_OptimizeAndAssemble(struct _PyCompiler *c, int addNone);
 

Lib/test/test_compile.py

@@ -816,12 +816,30 @@ def check_same_constant(const):
                       f2.__code__.co_consts[1])
 
         # {0} is converted to a constant frozenset({0}) by the peephole
-        # optimizer
-        f1, f2 = lambda x: x in {0}, lambda x: x in {0}
-        self.assertIs(f1.__code__.co_consts, f2.__code__.co_consts)
+        # optimizer. In free-threaded builds, frozenset-bearing co_consts are
+        # not shared across code units because later string interning can
+        # replace them.
+        ns = {}
+        exec("f1, f2 = lambda x: x in {0}, lambda x: x in {0}", ns)
+        f1, f2 = ns["f1"], ns["f2"]
+        if support.Py_GIL_DISABLED:
+            self.assertIsNot(f1.__code__.co_consts, f2.__code__.co_consts)
+        else:
+            self.assertIs(f1.__code__.co_consts, f2.__code__.co_consts)
         self.check_constant(f1, frozenset({0}))
+        self.check_constant(f2, frozenset({0}))
         self.assertTrue(f1(0))
 
+        def repeated_set_constants(x):
+            return x in {1, 2, 3}, x in {1, 2, 3}
+
+        set_constants = [
+            value
+            for value in repeated_set_constants.__code__.co_consts
+            if isinstance(value, frozenset)
+        ]
+        self.assertEqual(set_constants, [frozenset({1, 2, 3})])
+
     # Merging equal co_linetable is not a strict requirement
     # for the Python semantics, it's a more an implementation detail.
     @support.cpython_only

Lib/test/test_marshal.py

@@ -15,6 +15,27 @@
 except ImportError:
     _testcapi = None
 
+
+def _make_ambiguous_set_codes():
+    template = (lambda: None).__code__
+    left = complex(float("nan"), 0)
+    right = complex(float("nan"), 0)
+    code = template.replace(co_consts=(frozenset((left, right)),))
+    nested_code = template.replace(
+        co_consts=(frozenset((frozenset((left, right)),)),)
+    )
+    return code, nested_code
+
+
+if support.Py_GIL_DISABLED:
+    # Code construction immortalizes numeric constants in free-threaded
+    # builds. Create these constants once so refleak runs do not accumulate
+    # intentionally immortal objects.
+    _AMBIGUOUS_SET_CODES = _make_ambiguous_set_codes()
+else:
+    _AMBIGUOUS_SET_CODES = ()
+
+
 class HelperMixin:
     def helper(self, sample, *extra):
         new = marshal.loads(marshal.dumps(sample, *extra))
@@ -363,6 +384,144 @@ def test_shared_reference_tuple(self):
         self.assertEqual(b[0], big)
         self.assertIs(b[0], b[1])
 
+    @unittest.skipUnless(
+        support.Py_GIL_DISABLED,
+        "free-threaded builds require deterministic reference tracking",
+    )
+    def test_code_reference_tracking_does_not_depend_on_refcount(self):
+        for version in range(3, marshal.version + 1):
+            with self.subTest(version=version):
+                constant = (bytes(bytearray(b"payload")),)
+                code = (lambda: None).__code__.replace(co_consts=(constant,))
+                del constant
+
+                without_extra_reference = marshal.dumps(code, version)
+                extra_reference = code.co_consts[0]
+                with_extra_reference = marshal.dumps(code, version)
+                self.assertEqual(without_extra_reference, with_extra_reference)
+                self.assertEqual(
+                    marshal.loads(with_extra_reference).co_consts,
+                    code.co_consts,
+                )
+                self.assertIs(extra_reference, code.co_consts[0])
+
+    @unittest.skipUnless(
+        support.Py_GIL_DISABLED,
+        "free-threaded builds require compile-order-independent constants",
+    )
+    def test_code_serialization_does_not_depend_on_compile_order(self):
+        script = textwrap.dedent("""
+            import marshal
+            {prelude}
+            code = compile({source!r}, "/repro/target.py", "exec")
+            print(marshal.dumps(code).hex())
+        """)
+
+        cases = (
+            (
+                """
+                    value = {"foo", 2, 3}
+
+                    def function():
+                        nested = {"foo", 2, 3}
+                """,
+                'compile("import foo", "/repro/prelude.py", "exec")',
+            ),
+            (
+                """
+                    value = target["foo":"bar"]
+
+                    def function(target):
+                        return target["foo":"bar"]
+                """,
+                'compile(\'target["foo":"bar"]\', '
+                '"/repro/prelude.py", "exec")',
+            ),
+        )
+        for source, prelude in cases:
+            source = textwrap.dedent(source)
+            with self.subTest(source=source):
+                _, without_prelude, _ = assert_python_ok(
+                    "-c",
+                    script.format(prelude="", source=source),
+                    PYTHONHASHSEED="0",
+                )
+                _, with_prelude, _ = assert_python_ok(
+                    "-c",
+                    script.format(prelude=prelude, source=source),
+                    PYTHONHASHSEED="0",
+                )
+                self.assertEqual(without_prelude, with_prelude)
+
+    @unittest.skipUnless(
+        support.Py_GIL_DISABLED,
+        "only deterministic code serialization rejects ambiguous set order",
+    )
+    def test_code_serialization_set_sort_key_ties(self):
+        code, nested_code = _AMBIGUOUS_SET_CODES
+        for version in range(marshal.version + 1):
+            with self.subTest(version=version):
+                with self.assertRaisesRegex(
+                    ValueError,
+                    "cannot deterministically marshal set elements",
+                ):
+                    marshal.dumps(code, version)
+                with self.assertRaisesRegex(
+                    ValueError,
+                    "cannot deterministically marshal set elements",
+                ):
+                    marshal.dumps(nested_code, version)
+
+        source = """
+            def contains(value):
+                return value in {
+                    (1e300 * 1e300) - (1e300 * 1e300),
+                    (1e300 * 1e300) - (1e300 * 1e300),
+                }
+        """
+        code = compile(textwrap.dedent(source), "/repro/nan_set.py", "exec")
+        marshal.dumps(code)
+
+    def test_code_serialization_set_shared_subobjects(self):
+        shared = (0,)
+        for _ in range(21):
+            shared = (shared, shared)
+        code = (lambda: None).__code__.replace(
+            co_consts=(frozenset(((shared,),)),)
+        )
+
+        payload = marshal.dumps(code)
+        self.assertLess(len(payload), 1000)
+        self.assertEqual(marshal.loads(payload).co_consts, code.co_consts)
+
+        shared = frozenset((0,))
+        for _ in range(10):
+            shared = frozenset(((0, shared), (1, shared)))
+        code = (lambda: None).__code__.replace(co_consts=(shared,))
+
+        payload = marshal.dumps(code)
+        self.assertLess(len(payload), 2000)
+        self.assertEqual(marshal.loads(payload).co_consts, code.co_consts)
+
+    def test_code_serialization_recursive_constants(self):
+        recursive = []
+        recursive_slice = slice(recursive)
+        recursive.append(recursive_slice)
+        code = (lambda: None).__code__.replace(
+            co_consts=(recursive_slice,)
+        )
+        for version in range(marshal.version + 1):
+            with self.subTest(type="slice", version=version):
+                self.assertRaises(ValueError, marshal.dumps, code, version)
+
+        recursive = []
+        recursive_frozendict = frozendict({None: recursive})
+        recursive.append(recursive_frozendict)
+        code = code.replace(co_consts=(recursive_frozendict,))
+        for version in range(marshal.version + 1):
+            with self.subTest(type="frozendict", version=version):
+                self.assertRaises(ValueError, marshal.dumps, code, version)
+
     def test_reference_loop_code(self):
         def f():
             return 1234.5

Misc/NEWS.d/next/Core_and_Builtins/2026-08-31-15-55-00.gh-129724.rst

@@ -0,0 +1,4 @@
+Make reference tracking for marshalled code objects in free-threaded builds
+independent of live reference counts. Avoid unstable sharing of immutable
+constants in free-threaded compiler output. This makes free-threaded bytecode
+compilation deterministic across compilation order and concurrent compilation.

Python/compile.c

@@ -339,10 +339,38 @@ _PyCompile_SetQualname(compiler *c)
 /* Merge const *o* and return constant key object.
  * If recursive, insert all elements if o is a tuple or frozen set.
  */
+static int
+const_cache_contains_rebuilt_container(PyObject *o)
+{
+#ifdef Py_GIL_DISABLED
+    if (PyFrozenSet_CheckExact(o)) {
+        return 1;
+    }
+    if (PySlice_Check(o)) {
+        return 1;
+    }
+    if (PyTuple_CheckExact(o)) {
+        for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(o); i++) {
+            if (const_cache_contains_rebuilt_container(
+                    PyTuple_GET_ITEM(o, i)))
+            {
+                return 1;
+            }
+        }
+    }
+#else
+    (void)o;
+#endif
+    return 0;
+}
+
 static PyObject*
-const_cache_insert(PyObject *const_cache, PyObject *o, bool recursive)
+const_cache_insert(PyObject *const_cache, PyObject *local_const_cache,
+                   PyObject *o, bool recursive)
 {
-    assert(PyDict_CheckExact(const_cache));
+    assert(const_cache == NULL || PyDict_CheckExact(const_cache));
+    assert(local_const_cache == NULL ||
+           PyDict_CheckExact(local_const_cache));
     // None and Ellipsis are immortal objects, and key is the singleton.
     // No need to merge object and key.
     if (o == Py_None || o == Py_Ellipsis) {
@@ -354,28 +382,34 @@ const_cache_insert(PyObject *const_cache, PyObject *o, bool recursive)
         return NULL;
     }
 
-    PyObject *t;
-    int res = PyDict_SetDefaultRef(const_cache, key, key, &t);
-    if (res != 0) {
-        // o was not inserted into const_cache. t is either the existing value
-        // or NULL (on error).
-        Py_DECREF(key);
-        return t;
+    // Code construction can rebuild some containers after interning their
+    // elements. Do not let prior compilation decide whether code units share
+    // the original object.
+    PyObject *cache = const_cache_contains_rebuilt_container(o)
+        ? local_const_cache : const_cache;
+    if (cache != NULL) {
+        PyObject *t;
+        int res = PyDict_SetDefaultRef(cache, key, key, &t);
+        if (res != 0) {
+            // o was not inserted into const_cache. t is either the existing
+            // value or NULL (on error).
+            Py_DECREF(key);
+            return t;
+        }
+        Py_DECREF(t);
     }
-    Py_DECREF(t);
 
     if (!recursive) {
         return key;
     }
 
-    // We registered o in const_cache.
-    // When o is a tuple or frozenset, we want to merge its
-    // items too.
+    // When o is a tuple or frozenset, merge its items too.
     if (PyTuple_CheckExact(o)) {
         Py_ssize_t len = PyTuple_GET_SIZE(o);
         for (Py_ssize_t i = 0; i < len; i++) {
             PyObject *item = PyTuple_GET_ITEM(o, i);
-            PyObject *u = const_cache_insert(const_cache, item, recursive);
+            PyObject *u = const_cache_insert(
+                const_cache, local_const_cache, item, recursive);
             if (u == NULL) {
                 Py_DECREF(key);
                 return NULL;
@@ -417,7 +451,8 @@ const_cache_insert(PyObject *const_cache, PyObject *o, bool recursive)
         PyObject *item;
         Py_hash_t hash;
         while (_PySet_NextEntry(o, &pos, &item, &hash)) {
-            PyObject *k = const_cache_insert(const_cache, item, recursive);
+            PyObject *k = const_cache_insert(
+                const_cache, local_const_cache, item, recursive);
             if (k == NULL) {
                 Py_DECREF(tuple);
                 Py_DECREF(key);
@@ -454,7 +489,7 @@ const_cache_insert(PyObject *const_cache, PyObject *o, bool recursive)
 static PyObject*
 merge_consts_recursive(PyObject *const_cache, PyObject *o)
 {
-    return const_cache_insert(const_cache, o, true);
+    return const_cache_insert(const_cache, NULL, o, true);
 }
 
 Py_ssize_t
@@ -1385,7 +1420,28 @@ _PyCompile_Metadata(compiler *c)
 int
 _PyCompile_ConstCacheMergeOne(PyObject *const_cache, PyObject **obj)
 {
-    PyObject *key = const_cache_insert(const_cache, *obj, false);
+    PyObject *key = const_cache_insert(const_cache, NULL, *obj, false);
+    if (key == NULL) {
+        return ERROR;
+    }
+    if (PyTuple_CheckExact(key)) {
+        PyObject *item = PyTuple_GET_ITEM(key, 1);
+        Py_SETREF(*obj, Py_NewRef(item));
+        Py_DECREF(key);
+    }
+    else {
+        Py_SETREF(*obj, key);
+    }
+    return SUCCESS;
+}
+
+int
+_PyCompile_ConstCacheMergeOneLocal(PyObject *const_cache,
+                                   PyObject *local_const_cache,
+                                   PyObject **obj)
+{
+    PyObject *key = const_cache_insert(
+        const_cache, local_const_cache, *obj, false);
     if (key == NULL) {
         return ERROR;
     }
@@ -1400,6 +1456,20 @@ _PyCompile_ConstCacheMergeOne(PyObject *const_cache, PyObject **obj)
     return SUCCESS;
 }
 
+int
+_PyCompile_ConstCacheAddLocal(PyObject *local_const_cache, PyObject *obj)
+{
+    if (!const_cache_contains_rebuilt_container(obj)) {
+        return SUCCESS;
+    }
+    PyObject *key = const_cache_insert(NULL, local_const_cache, obj, false);
+    if (key == NULL) {
+        return ERROR;
+    }
+    Py_DECREF(key);
+    return SUCCESS;
+}
+
 static PyObject *
 consts_dict_keys_inorder(PyObject *dict)
 {