lightspeed: a failing teardown does not stop the survey

asv/contrib/lightspeed/survey.py

@@ -165,7 +165,7 @@ def survey_one(
         finally:
             cov.stop()
     except BaseException as exc:
-        bench.do_teardown()
+        # Teardown runs once, in the finally below; a failing teardown must not stop the survey of the other benchmarks.
         return False, f"runtime_error: {exc}", {}
     finally:
         try:

test/test_lightspeed_survey.py

@@ -29,11 +29,30 @@ def time_work():
     """))
     monkeypatch.syspath_prepend(str(tmp_path))
     yield tmp_path
-    for name in [m for m in sys.modules if m == "pkg" or m.startswith(("pkg.", "bench_pkg"))]:
+    for name in [m for m in sys.modules if m in ("pkg", "benchmarks") or m.startswith(("pkg.", "benchmarks.", "bench_"))]:
         del sys.modules[name]
 
 
 def test_only_executed_files_are_deps(project):
     ok, reason, deps = survey_one(str(project / "benchmarks"), BenchmarkId("bench_pkg.time_work"), str(project / "pkg"))
     assert ok, reason
     assert {p.split("/pkg/")[-1] for p in deps} == {"used.py"}
+
+
+def test_failing_teardown_after_a_failing_body_is_recorded(project):
+    (project / "benchmarks" / "bench_fail.py").write_text(textwrap.dedent("""
+        import pathlib
+        calls = pathlib.Path(__file__).with_name("teardowns.txt")
+
+        def time_boom():
+            raise RuntimeError("body")
+
+        def teardown():
+            calls.write_text(calls.read_text() + "x" if calls.exists() else "x")
+            raise TimeoutError("teardown")
+
+        time_boom.teardown = teardown
+    """))
+    ok, reason, deps = survey_one(str(project / "benchmarks"), BenchmarkId("bench_fail.time_boom"), str(project / "pkg"))
+    assert (ok, deps) == (False, {}) and reason.startswith("runtime_error: body")
+    assert (project / "benchmarks" / "teardowns.txt").read_text() == "x"