Use PGO for Cargo

bootstrap.example.toml

@@ -990,6 +990,11 @@
 # Instrument rustdoc, so that when executed, it will gather profiles
 # to this path.
 #pgo.rustdoc.generate = "/tmp/profiles/foo.profraw"
+# Use the following profile to PGO optimize cargo.
+#pgo.cargo.use = "/tmp/profiles/foo.profraw"
+# Instrument cargo, so that when executed, it will gather profiles
+# to this path.
+#pgo.cargo.generate = "/tmp/profiles/foo.profraw"
 # Use the following profile to PGO optimize LLVM.
 #pgo.llvm.use = "/tmp/profiles/foo.profraw"
 # Instrument LLVM, so that when executed, it will gather profiles

src/bootstrap/src/core/build_steps/tool.rs

@@ -138,8 +138,13 @@ impl Step for ToolBuild {
             }
         }
 
-        if self.path == "src/tools/rustdoc" {
-            apply_pgo(builder, &mut cargo, self.build_compiler, &builder.config.rustdoc_pgo);
+        let pgo_config = match self.path {
+            "src/tools/rustdoc" => Some(&builder.config.rustdoc_pgo),
+            "src/tools/cargo" => Some(&builder.config.cargo_pgo),
+            _ => None,
+        };
+        if let Some(pgo_config) = pgo_config {
+            apply_pgo(builder, &mut cargo, self.build_compiler, pgo_config);
         }
 
         if !self.allow_features.is_empty() {

src/bootstrap/src/core/config/config.rs

@@ -236,6 +236,7 @@ pub struct Config {
     pub rust_rustflags: Vec<String>,
     pub rust_pgo: PgoConfig,
     pub rustdoc_pgo: PgoConfig,
+    pub cargo_pgo: PgoConfig,
 
     pub llvm_libunwind_default: Option<LlvmLibunwind>,
     pub enable_bolt_settings: bool,
@@ -657,7 +658,7 @@ impl Config {
             libgccjit_libs_dir: gcc_libgccjit_libs_dir,
         } = toml_gcc.unwrap_or_default();
 
-        let Pgo { rustc: pgo_rustc, llvm: pgo_llvm, rustdoc: pgo_rustdoc } =
+        let Pgo { rustc: pgo_rustc, llvm: pgo_llvm, rustdoc: pgo_rustdoc, cargo: pgo_cargo } =
             toml_pgo.unwrap_or_default();
 
         // Backcompat: flags have priority over config
@@ -703,10 +704,16 @@ impl Config {
             panic!("Cannot use and generate LLVM PGO profiles at the same time");
         }
 
-        let pgo_rustdoc = pgo_rustdoc.unwrap_or_default();
-        if pgo_rustdoc.use_profile.is_some() && pgo_rustdoc.generate_profile.is_some() {
-            panic!("Cannot use and generate rustdoc PGO profiles at the same time");
-        }
+        let init_pgo = |pgo: Option<PgoConfig>, name: &str| -> PgoConfig {
+            let pgo_config = pgo.unwrap_or_default();
+            if pgo_config.use_profile.is_some() && pgo_config.generate_profile.is_some() {
+                panic!("Cannot use and generate {name} PGO profiles at the same time");
+            }
+            pgo_config
+        };
+
+        let pgo_rustdoc = init_pgo(pgo_rustdoc, "rustdoc");
+        let pgo_cargo = init_pgo(pgo_cargo, "cargo");
 
         if rust_bootstrap_override_lld.is_some() && rust_bootstrap_override_lld_legacy.is_some() {
             panic!(
@@ -1400,6 +1407,7 @@ NOTE: Please add `--stage 2` to your command line, or if you're sure you want to
             bypass_bootstrap_lock: flags_bypass_bootstrap_lock,
             cargo_info,
             cargo_native_static: build_cargo_native_static.unwrap_or(false),
+            cargo_pgo: pgo_cargo,
             ccache,
             change_id: toml_change_id.inner,
             channel,

src/bootstrap/src/core/config/toml/pgo.rs

@@ -27,6 +27,7 @@ define_config! {
     struct Pgo {
         rustc: Option<PgoConfig> = "rustc",
         rustdoc: Option<PgoConfig> = "rustdoc",
+        cargo: Option<PgoConfig> = "cargo",
         llvm: Option<PgoConfig> = "llvm",
     }
 }

src/tools/opt-dist/src/environment.rs

@@ -61,6 +61,12 @@ impl Environment {
         self.stage0().join("bin").join(format!("cargo{}", executable_extension()))
     }
 
+    pub fn cargo_stage_2(&self) -> Utf8PathBuf {
+        self.build_artifacts()
+            .join("stage2-tools-bin")
+            .join(format!("cargo{}", executable_extension()))
+    }
+
     pub fn rustc_stage_0(&self) -> Utf8PathBuf {
         self.stage0().join("bin").join(format!("rustc{}", executable_extension()))
     }

src/tools/opt-dist/src/exec.rs

@@ -124,6 +124,11 @@ impl Bootstrap {
         self
     }
 
+    pub fn with_cargo(mut self) -> Self {
+        self.cmd = self.cmd.arg("cargo");
+        self
+    }
+
     pub fn dist(env: &Environment, dist_args: &[String]) -> Self {
         let metrics_path = env.build_root().join("metrics.json");
         let args = dist_args.iter().map(|arg| arg.as_str()).collect::<Vec<_>>();
@@ -155,6 +160,16 @@ impl Bootstrap {
         self
     }
 
+    pub fn without_llvm_lto(mut self) -> Self {
+        self.cmd = self
+            .cmd
+            .arg("--set")
+            .arg("llvm.thin-lto=false")
+            .arg("--set")
+            .arg("llvm.link-shared=true");
+        self
+    }
+
     pub fn rustc_pgo_instrument(mut self, profile_dir: &Utf8Path) -> Self {
         self.cmd = self
             .cmd
@@ -163,6 +178,14 @@ impl Bootstrap {
         self
     }
 
+    pub fn rustc_pgo_optimize(mut self, profile: &RustcPGOProfile) -> Self {
+        self.cmd = self
+            .cmd
+            .arg("--set")
+            .arg(format!(r#"pgo.rustc.use="{}""#, normalize_path(&profile.0).as_str()));
+        self
+    }
+
     pub fn rustdoc_pgo_instrument(mut self, profile_dir: &Utf8Path) -> Self {
         self.cmd = self
             .cmd
@@ -171,29 +194,27 @@ impl Bootstrap {
         self
     }
 
-    pub fn without_llvm_lto(mut self) -> Self {
+    pub fn rustdoc_pgo_optimize(mut self, profile: &RustdocPGOProfile) -> Self {
         self.cmd = self
             .cmd
             .arg("--set")
-            .arg("llvm.thin-lto=false")
-            .arg("--set")
-            .arg("llvm.link-shared=true");
+            .arg(format!(r#"pgo.rustdoc.use="{}""#, normalize_path(&profile.0).as_str()));
         self
     }
 
-    pub fn rustc_pgo_optimize(mut self, profile: &RustcPGOProfile) -> Self {
+    pub fn cargo_pgo_instrument(mut self, profile_dir: &Utf8Path) -> Self {
         self.cmd = self
             .cmd
             .arg("--set")
-            .arg(format!(r#"pgo.rustc.use="{}""#, normalize_path(&profile.0).as_str()));
+            .arg(format!(r#"pgo.cargo.generate="{}""#, normalize_path(profile_dir).as_str()));
         self
     }
 
-    pub fn rustdoc_pgo_optimize(mut self, profile: &RustdocPGOProfile) -> Self {
+    pub fn cargo_pgo_optimize(mut self, profile: &RustcPGOProfile) -> Self {
         self.cmd = self
             .cmd
             .arg("--set")
-            .arg(format!(r#"pgo.rustdoc.use="{}""#, normalize_path(&profile.0).as_str()));
+            .arg(format!(r#"pgo.cargo.use="{}""#, normalize_path(&profile.0).as_str()));
         self
     }
 

src/tools/opt-dist/src/main.rs

@@ -249,7 +249,9 @@ fn execute_pipeline(
                 // Their profiles are then merged together into a single PGO profile.
                 let mut builder = Bootstrap::build(env)
                     .with_rustdoc()
+                    .with_cargo()
                     .rustc_pgo_instrument(&rustc_profile_dir_root)
+                    .cargo_pgo_instrument(&rustc_profile_dir_root)
                     .rustdoc_pgo_instrument(&rustc_profile_dir_root);
 
                 if env.supports_shared_llvm() {
@@ -274,7 +276,9 @@ fn execute_pipeline(
             stage.section("Build PGO optimized rustc", |section| {
                 let mut cmd = Bootstrap::build(env)
                     .with_rustdoc()
+                    .with_cargo()
                     .rustc_pgo_optimize(&rustc_profile)
+                    .cargo_pgo_optimize(&rustc_profile)
                     .rustdoc_pgo_optimize(&rustdoc_profile);
                 if env.use_bolt() {
                     cmd = cmd.with_rustc_bolt_ldflags();
@@ -417,6 +421,7 @@ fn execute_pipeline(
     let mut dist = Bootstrap::dist(env, &dist_args)
         .llvm_pgo_optimize(llvm_pgo_profile.as_ref())
         .rustc_pgo_optimize(&rustc_pgo_profile)
+        .cargo_pgo_optimize(&rustc_pgo_profile)
         .rustdoc_pgo_optimize(&rustdoc_pgo_profile);
 
     // if LLVM is not built we'll have PGO optimized rustc

src/tools/opt-dist/src/training.rs

@@ -31,7 +31,7 @@ fn init_compiler_benchmarks(
         "--id",
         "Test",
         "--cargo",
-        env.cargo_stage_0().as_str(),
+        env.cargo_stage_2().as_str(),
         "--profiles",
         profiles.join(",").as_str(),
         "--scenarios",
@@ -109,6 +109,8 @@ pub fn llvm_benchmarks(env: &Environment) -> CmdBuilder {
     init_compiler_benchmarks(env, &["Debug", "Opt"], &["Full"], LLVM_PGO_CRATES)
 }
 
+// Here we're profiling the `rustc` frontend, so we also include `Check`.
+// The benchmark set includes various stress tests that put the frontend under pressure.
 pub fn rustc_benchmarks(env: &Environment) -> CmdBuilder {
     init_compiler_benchmarks(env, &["Check", "Debug", "Opt"], &["All"], RUSTC_PGO_CRATES)
 }
@@ -149,35 +151,7 @@ pub fn gather_rustc_profiles(
     profile_root: &Utf8Path,
 ) -> anyhow::Result<RustcPGOProfile> {
     log::info!("Running benchmarks with PGO instrumented rustc");
-
-    // The profile data is written into a single filepath that is being repeatedly merged when each
-    // rustc invocation ends. Empirically, this can result in some profiling data being lost. That's
-    // why we override the profile path to include the PID. This will produce many more profiling
-    // files, but the resulting profile will produce a slightly faster rustc binary.
-    let profile_template = profile_root.join("default_%m_%p.profraw");
-
-    // Here we're profiling the `rustc` frontend, so we also include `Check`.
-    // The benchmark set includes various stress tests that put the frontend under pressure.
-    with_log_group("Running benchmarks", || {
-        rustc_benchmarks(env)
-            .env("LLVM_PROFILE_FILE", profile_template.as_str())
-            .run()
-            .context("Cannot gather rustc PGO profiles")
-    })?;
-
-    let merged_profile = env.artifact_dir().join("rustc-pgo.profdata");
-    log::info!("Merging Rustc PGO profiles to {merged_profile}");
-
-    let llvm_profdata = if env.build_llvm() { LlvmProfdata::Target } else { LlvmProfdata::Host };
-
-    merge_llvm_profiles(env, &merged_profile, profile_root, llvm_profdata)?;
-    log_profile_stats("Rustc", &merged_profile, profile_root)?;
-
-    // We don't need the individual .profraw files now that they have been merged
-    // into a final .profdata
-    delete_directory(profile_root)?;
-
-    Ok(RustcPGOProfile(merged_profile))
+    gather_pgo_profiles(env, profile_root, "rustc", rustc_benchmarks(env)).map(RustcPGOProfile)
 }
 
 pub struct RustdocPGOProfile(pub Utf8PathBuf);
@@ -187,35 +161,42 @@ pub fn gather_rustdoc_profiles(
     profile_root: &Utf8Path,
 ) -> anyhow::Result<RustdocPGOProfile> {
     log::info!("Running benchmarks with PGO instrumented rustdoc");
+    gather_pgo_profiles(env, profile_root, "rustdoc", rustdoc_benchmarks(env))
+        .map(RustdocPGOProfile)
+}
 
+pub fn gather_pgo_profiles(
+    env: &Environment,
+    profile_root: &Utf8Path,
+    name: &str,
+    benchmark_cmd: CmdBuilder,
+) -> anyhow::Result<Utf8PathBuf> {
     // The profile data is written into a single filepath that is being repeatedly merged when each
     // rustc invocation ends. Empirically, this can result in some profiling data being lost. That's
     // why we override the profile path to include the PID. This will produce many more profiling
     // files, but the resulting profile will produce a slightly faster rustc binary.
     let profile_template = profile_root.join("default_%m_%p.profraw");
 
-    // Here we're profiling the `rustc` frontend, so we also include `Check`.
-    // The benchmark set includes various stress tests that put the frontend under pressure.
     with_log_group("Running benchmarks", || {
-        rustdoc_benchmarks(env)
+        benchmark_cmd
             .env("LLVM_PROFILE_FILE", profile_template.as_str())
             .run()
-            .context("Cannot gather rustdoc PGO profiles")
+            .with_context(|| format!("Cannot gather {name} PGO profiles"))
     })?;
 
-    let merged_profile = env.artifact_dir().join("rustdoc-pgo.profdata");
-    log::info!("Merging Rustdoc PGO profiles to {merged_profile}");
+    let merged_profile = env.artifact_dir().join(format!("{name}-pgo.profdata"));
+    log::info!("Merging {name} PGO profiles to {merged_profile}");
 
     let llvm_profdata = if env.build_llvm() { LlvmProfdata::Target } else { LlvmProfdata::Host };
 
     merge_llvm_profiles(env, &merged_profile, profile_root, llvm_profdata)?;
-    log_profile_stats("Rustdoc", &merged_profile, profile_root)?;
+    log_profile_stats(name, &merged_profile, profile_root)?;
 
     // We don't need the individual .profraw files now that they have been merged
     // into a final .profdata
     delete_directory(profile_root)?;
 
-    Ok(RustdocPGOProfile(merged_profile))
+    Ok(merged_profile)
 }
 
 pub struct BoltProfile(pub Utf8PathBuf);