Fix skipped imports considered stale

When the option `follow_imports = skip` is used, dependency states that
are initially considered skipped might have their reason changed to not
found in `load_graph`.

If on the first mypy run, a given suppressed module is written to cache
with its suppression reason set to skipped, and it's overridden to not
found in a subsequent mypy run, then `suppressed_deps_opts` between the
cache and the current run won't match and the module will be considered
stale.

To fix the issue, change the unconditional overwrite to `setdefault`.

This also affects mypyc as the dependency being considered stale means
that mypyc will needlessly recompile it instead of reading from cache. I
have added a unit test to confirm that the dependency output files are
not overwritten with the fix.

Author: p-sawicki

mypy/build.py

@@ -4345,7 +4345,7 @@ def load_graph(
         for dep in st.ancestors + dependencies + st.suppressed:
             ignored = dep in st.suppressed_set and dep not in entry_points
             if ignored and dep not in added:
-                manager.missing_modules[dep] = SuppressionReason.NOT_FOUND
+                manager.missing_modules.setdefault(dep, SuppressionReason.NOT_FOUND)
                 # TODO: for now we skip this in the daemon as a performance optimization.
                 # This however creates a correctness issue, see #7777 and State.is_fresh().
                 if not manager.use_fine_grained_cache() or manager.options.warn_unused_configs:

mypy/test/testgraph.py

@@ -2,13 +2,24 @@
 
 from __future__ import annotations
 
+import os
 import sys
+import tempfile
 from collections.abc import Set as AbstractSet
 
-from mypy.build import BuildManager, BuildSourceSet, State, order_ascc, sorted_components
+import mypy.build as build_module
+from mypy.build import (
+    BuildManager,
+    BuildSourceSet,
+    State,
+    SuppressionReason,
+    order_ascc,
+    sorted_components,
+)
 from mypy.errors import Errors
 from mypy.fscache import FileSystemCache
 from mypy.graph_utils import strongly_connected_components, topsort
+from mypy.main import process_options
 from mypy.modulefinder import SearchPaths
 from mypy.options import Options
 from mypy.plugin import Plugin
@@ -108,6 +119,50 @@ def _make_manager(self) -> BuildManager:
         )
         return manager
 
+    def test_fine_grained_cache_preserves_suppression_reason(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            with open(os.path.join(tmp, "mypy.ini"), "w", encoding="utf-8") as f:
+                f.write("[mypy]\ncache_fine_grained = True\nfollow_imports = skip\n")
+            with open(os.path.join(tmp, "skipped.py"), "w", encoding="utf-8") as f:
+                f.write("def ignored() -> int:\n    return 1\n")
+            with open(os.path.join(tmp, "dep.py"), "w", encoding="utf-8") as f:
+                f.write("import skipped\n\ndef value() -> int:\n    return 1\n")
+            with open(os.path.join(tmp, "main.py"), "w", encoding="utf-8") as f:
+                f.write("import dep\n\ndef value() -> int:\n    return dep.value()\n")
+            with open(os.path.join(tmp, "seed.py"), "w", encoding="utf-8") as f:
+                f.write("import skipped\n")
+
+            old_cwd = os.getcwd()
+            os.chdir(tmp)
+            try:
+
+                def run_mypy(
+                    args: list[str], *, server_options: bool = False
+                ) -> build_module.BuildResult:
+                    sources, options = process_options(args, server_options=server_options)
+                    options.use_builtins_fixtures = True
+                    result = build_module.build(sources=sources, options=options)
+                    assert_equal(result.errors, [])
+                    return result
+
+                result = run_mypy(["dep.py", "main.py"])
+                assert_equal(result.graph["dep"].suppressed, ["skipped"])
+                assert_equal(result.manager.missing_modules["skipped"], SuppressionReason.SKIPPED)
+
+                result = run_mypy(["seed.py", "skipped.py"])
+                assert_equal(result.graph["seed"].dependencies, ["skipped", "builtins"])
+                assert_equal(result.graph["seed"].suppressed, [])
+
+                result = run_mypy(
+                    ["--use-fine-grained-cache", "seed.py", "dep.py", "main.py"],
+                    server_options=True,
+                )
+            finally:
+                os.chdir(old_cwd)
+
+            assert_equal(result.manager.missing_modules["skipped"], SuppressionReason.SKIPPED)
+            assert result.graph["dep"].is_fresh()
+
     def test_sorted_components(self) -> None:
         manager = self._make_manager()
         graph = {

mypyc/test/test_misc.py

@@ -3,7 +3,9 @@
 import os
 import tempfile
 import unittest
+from unittest.mock import patch
 
+import mypyc.build as mypyc_build_module
 from mypyc.build import get_header_deps, resolve_cfile_deps
 from mypyc.ir.ops import BasicBlock
 from mypyc.ir.pprint import format_blocks, generate_names_for_ir
@@ -25,6 +27,68 @@ def test_debug_op(self) -> None:
         assert code[:-1] == ["L0:", "    r0 = 'foo'", "    CPyDebug_PrintObject(r0)"]
 
 
+class TestIncrementalOutputFiles(unittest.TestCase):
+    def test_cached_dependency_output_file_not_overwritten_for_preserved_suppression(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            build_dir = os.path.join(tmp, "build")
+            with open(os.path.join(tmp, "mypy.ini"), "w", encoding="utf-8") as f:
+                f.write("[mypy]\nfollow_imports = skip\n")
+            os.mkdir(os.path.join(tmp, "skipped"))
+            with open(os.path.join(tmp, "skipped", "__init__.py"), "w", encoding="utf-8") as f:
+                f.write("x = 1\n")
+            with open(os.path.join(tmp, "skipped", "child.py"), "w", encoding="utf-8") as f:
+                f.write("import skipped\n\ndef child() -> int:\n    return 1\n")
+            with open(os.path.join(tmp, "dep.py"), "w", encoding="utf-8") as f:
+                f.write("import skipped\n\ndef value() -> int:\n    return 1\n")
+            with open(os.path.join(tmp, "main.py"), "w", encoding="utf-8") as f:
+                f.write("import dep\n\ndef value() -> int:\n    return dep.value()\n")
+
+            compiler_options = CompilerOptions(separate=True, target_dir=build_dir)
+
+            old_cwd = os.getcwd()
+            os.chdir(tmp)
+            try:
+                files = ["dep.py", "skipped/child.py", "main.py"]
+
+                groups, group_cfilenames, _ = mypyc_build_module.mypyc_build(
+                    files, compiler_options, separate=True
+                )
+
+                dep_group = next(
+                    i
+                    for i, (group_sources, _) in enumerate(groups)
+                    if [source.module for source in group_sources] == ["dep"]
+                )
+                dep_output_files = {
+                    os.path.abspath(path)
+                    for paths in group_cfilenames[dep_group]
+                    for path in paths
+                }
+
+                # Only main changed; dep should be loaded from the mypyc IR cache and reuse
+                # its existing C outputs.
+                with open("main.py", "w", encoding="utf-8") as f:
+                    f.write("import dep\n\ndef value() -> int:\n    return dep.value() + 1\n")
+
+                written_paths: set[str] = set()
+                original_write_file = mypyc_build_module.write_file
+
+                def recording_write_file(path: str, contents: str) -> None:
+                    written_paths.add(os.path.abspath(path))
+                    original_write_file(path, contents)
+
+                with patch.object(
+                    mypyc_build_module, "write_file", side_effect=recording_write_file
+                ):
+                    # skipped.child imports its skipped package ancestor; loading that cached
+                    # state should not make cached dep stale or rewrite dep's generated outputs.
+                    mypyc_build_module.mypyc_build(files, compiler_options, separate=True)
+            finally:
+                os.chdir(old_cwd)
+
+            assert written_paths.isdisjoint(dep_output_files)
+
+
 class TestHeaderDeps(unittest.TestCase):
     """
     Tests for the header-dependency tracking used to build `Extension.depends`, which drives