Run benchmarks on CodSpeed in CI

.github/workflows/codspeed.yml

@@ -0,0 +1,68 @@
+name: CodSpeed
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+  # workflow_dispatch lets CodSpeed trigger backtest runs to seed baseline data.
+  workflow_dispatch:
+permissions:
+  contents: read
+  id-token: write # OpenID Connect auth with CodSpeed (no token secret required)
+jobs:
+  # Instruction-count (Valgrind) measurement of the small, deterministic
+  # microbenchmarks. They are single-threaded and fast, so callgrind's overhead
+  # is bounded and the instruction counts are stable run-to-run.
+  simulation:
+    name: Microbenchmarks (simulation)
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v6
+    - uses: dtolnay/rust-toolchain@master
+      with:
+        toolchain: stable
+    - name: set up rust cache
+      uses: Swatinem/rust-cache@v2
+      with:
+        prefix-key: pyrefly-codspeed
+    - name: install cargo-codspeed
+      uses: taiki-e/install-action@v2
+      with:
+        tool: cargo-codspeed
+    - name: build benchmarks
+      run: cargo codspeed build -p pyrefly --bench micro
+    - name: run benchmarks
+      uses: CodSpeedHQ/action@v4
+      with:
+        mode: simulation
+        run: cargo codspeed run -p pyrefly --bench micro
+  # Wall-clock measurement of the heavy, real-world PyTorch benchmarks. They drive
+  # the full LSP server over all cores against the pinned PyTorch checkout, so they
+  # must not run under Valgrind (which serializes threads and inflates the ~GB cold
+  # start past any reasonable timeout). Walltime mode tolerates threads, I/O, and
+  # long cold starts. CodSpeed's dedicated `codspeed-macro` runners give the lowest
+  # variance for walltime; `ubuntu-latest` also works if they aren't provisioned.
+  walltime:
+    name: PyTorch benchmarks (walltime)
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v6
+    # No submodule checkout: in OSS the benchmark clones the pinned PyTorch itself
+    # (rev from benches/pytorch_pin.bzl) on first run. The runner has github egress.
+    - uses: dtolnay/rust-toolchain@master
+      with:
+        toolchain: stable
+    - name: set up rust cache
+      uses: Swatinem/rust-cache@v2
+      with:
+        prefix-key: pyrefly-codspeed-walltime
+    - name: install cargo-codspeed
+      uses: taiki-e/install-action@v2
+      with:
+        tool: cargo-codspeed
+    - name: build benchmarks
+      run: cargo codspeed build -m walltime -p pyrefly --bench pytorch
+    - name: run benchmarks
+      uses: CodSpeedHQ/action@v4
+      with:
+        mode: walltime
+        run: cargo codspeed run -p pyrefly --bench pytorch

Cargo.toml

@@ -10,6 +10,15 @@ strip = "debuginfo"
 [profile.dev]
 debug = 1
 
+# Benchmarks don't need release's whole-program LTO with a single codegen unit.
+# Disabling LTO and using multiple codegen units keeps the CodSpeed build within
+# the memory budget of standard CI runners (and builds much faster) without
+# affecting CodSpeed's deterministic instruction-count measurements.
+[profile.bench]
+inherits = "release"
+lto = false
+codegen-units = 16
+
 [profile.dbg]
 debug = true
 inherits = "dev"

crates/pyrefly_util/Cargo.toml

@@ -45,6 +45,7 @@ tracing = { version = "0.1.41", features = ["attributes", "valuable"] }
 tracing-subscriber = { version = "0.3.23", features = ["env-filter", "fmt", "registry"], default-features = false }
 uuid = { version = "1.17", features = ["js", "rng-getrandom", "v4"] }
 vec1 = { version = "1.12.1", features = ["serde"] }
+web-time = "1.1.0"
 yansi = { version = "1.0.0-rc.1", features = ["hyperlink"] }
 
 [dev-dependencies]

crates/pyrefly_util/src/lib.rs

@@ -57,6 +57,7 @@ pub mod task_heap;
 pub mod telemetry;
 pub mod test_path;
 pub mod thread_pool;
+pub mod timer;
 pub mod trace;
 pub mod uniques;
 pub mod unix_path;

crates/pyrefly_util/src/task_heap.rs

@@ -173,7 +173,9 @@ impl<K: Ord, V> TaskHeap<K, V> {
             fn drop(&mut self) {
                 let mut lock = self.0.inner.lock();
                 lock.active_workers -= 1;
-                if lock.active_workers == 0 && lock.heap.is_empty() {
+                // Only wake if a worker is actually parked. `notify_all` otherwise
+                // still issues a `futex` syscall, which CodSpeed cannot instrument.
+                if lock.active_workers == 0 && lock.heap.is_empty() && lock.paused_workers > 0 {
                     self.0.condition.notify_all();
                 }
             }
@@ -200,7 +202,11 @@ impl<K: Ord, V> TaskHeap<K, V> {
                 }
                 None => {
                     if lock.active_workers == 0 {
-                        self.condition.notify_all();
+                        // Only wake if a worker is actually parked; a bare
+                        // `notify_all` still issues an uninstrumentable `futex`.
+                        if lock.paused_workers > 0 {
+                            self.condition.notify_all();
+                        }
                         break;
                     }
                     lock.paused_workers += 1;

crates/pyrefly_util/src/thread_pool.rs

@@ -27,6 +27,13 @@ pub enum ThreadCount {
     #[default]
     AllThreads,
     NumThreads(NonZeroUsize),
+    /// Run all work inline on the calling thread with no rayon pool. A rayon
+    /// pool — even one sized to a single thread — dispatches work to a worker
+    /// and blocks the caller on a futex for the whole duration. That futex
+    /// shows up under CodSpeed's instrumented instrument as an uninstrumentable
+    /// system call spanning the entire check. Running inline keeps all work on
+    /// the measured thread.
+    Inline,
 }
 
 /// Thread count used by tests. Enough threads to see parallelism bugs, but not too many to debug through.
@@ -72,8 +79,10 @@ impl ThreadPool {
     }
 
     pub fn new(count: ThreadCount) -> Self {
-        if cfg!(target_arch = "wasm32") {
-            // ThreadPool doesn't work on WASM
+        if cfg!(target_arch = "wasm32") || count == ThreadCount::Inline {
+            // No pool: WASM can't spawn threads, and `Inline` deliberately runs
+            // work on the calling thread. `spawn_many`/`install` then invoke their
+            // closures directly, with no worker handoff.
             return Self(None);
         }
 
@@ -89,6 +98,7 @@ impl ThreadPool {
                     .unwrap_or(1);
                 builder = builder.num_threads(max_threads);
             }
+            ThreadCount::Inline => unreachable!("handled by the early return above"),
         }
         let pool = builder.build().expect("To be able to build a thread pool");
         // Only print the message once