fix(runtime): VendSecret over mTLS — the boot contract passes end to end

services/harness/src/bootenv.rs

@@ -47,9 +47,54 @@ pub fn bootstrap() {
     report_entropy_source();
     mount_cert_drive();
     hydrate_env_from_cmdline();
+    assemble_declared_secrets();
     pin_manager_hostname();
 }
 
+/// Rebuild `LANTERN_DECLARED_SECRETS` from the per-secret boot-args keys.
+///
+/// The manager emits one `lantern.secret.<ENV_NAME>=<uri>` per declared secret
+/// (a JSON blob cannot survive the kernel command line: args are
+/// whitespace-separated and quotes are stripped from values). The rest of the
+/// harness reads a JSON array, so assemble it here.
+///
+/// Without this the secret cache sees nothing declared and refuses every
+/// request, so a spec declaring secrets behaves exactly like one declaring
+/// none — VendSecret is never reached.
+fn assemble_declared_secrets() {
+    // An explicitly-provided value wins, as everywhere else in this module.
+    if std::env::var_os("LANTERN_DECLARED_SECRETS").is_some() {
+        return;
+    }
+    let Ok(cmdline) = std::fs::read_to_string(CMDLINE_PATH) else {
+        return;
+    };
+
+    let declared: Vec<serde_json::Value> = cmdline
+        .split_whitespace()
+        .filter_map(|token| token.split_once('='))
+        .filter_map(|(key, uri)| {
+            key.strip_prefix("lantern.secret.")
+                .filter(|name| !name.is_empty() && !uri.is_empty())
+                .map(|name| serde_json::json!({ "env_name": name, "secret_uri": uri }))
+        })
+        .collect();
+
+    if declared.is_empty() {
+        return;
+    }
+    let count = declared.len();
+    match serde_json::to_string(&declared) {
+        Ok(json) => {
+            // SAFETY: called from main before any thread is spawned.
+            unsafe { std::env::set_var("LANTERN_DECLARED_SECRETS", &json) };
+            // Names only — a secret URI can identify sensitive material.
+            tracing::info!(count, "bootenv: assembled declared secrets from boot-args");
+        }
+        Err(e) => tracing::warn!(error = %e, "bootenv: could not encode declared secrets"),
+    }
+}
+
 /// Name the manager is reached by from inside the guest.
 ///
 /// Must match a DNS SAN on the manager's server certificate.

services/harness/src/main.rs

@@ -155,6 +155,13 @@ async fn main() -> Result<()> {
                 tracing::error!(error = %e, "secrets: server exited");
             }
         });
+        // Warm the cache before the workload can ask, so the mTLS vend path
+        // runs on every boot rather than only when something requests a
+        // secret. Spawned: a slow or unreachable manager must not delay boot.
+        let s = Arc::clone(&secrets);
+        tokio::spawn(async move {
+            s.prefetch_declared().await;
+        });
         let s = Arc::clone(&secrets);
         tokio::spawn(async move {
             s.refresh_loop().await;

services/harness/src/manager_client.rs

@@ -149,14 +149,27 @@ impl ManagerClient {
             endpoint
         };
 
-        let mut client = pb::runtime_harness_client::RuntimeHarnessClient::connect(endpoint)
-            .await
-            .with_context(|| {
-                format!(
-                    "heartbeat: could not connect to manager at {}",
-                    self.manager_addr
-                )
-            })?;
+        // NOTE: connect via `endpoint.connect()`, NOT `Client::connect(endpoint)`.
+        // The generated client's `connect` routes through
+        // `tonic::transport::Endpoint::new`, which for an https:// URI does:
+        //
+        //     return me.tls_config(ClientTlsConfig::new().with_enabled_roots());
+        //
+        // — silently REPLACING the TLS config set above with a default one. And
+        // `with_enabled_roots` discards its receiver, so with neither
+        // `tls-native-roots` nor `tls-webpki-roots` enabled the result is an
+        // EMPTY root store, no client identity, and no domain name. That is
+        // exactly what the guest saw: UnknownIssuer (no roots), "client auth
+        // requested but no cert available" (no identity), and verification
+        // against the URI's address (no domain). `Endpoint::connect` does not
+        // go through `Endpoint::new`, so the configured TLS survives.
+        let channel = endpoint.connect().await.with_context(|| {
+            format!(
+                "heartbeat: could not connect to manager at {}",
+                self.manager_addr
+            )
+        })?;
+        let mut client = pb::runtime_harness_client::RuntimeHarnessClient::new(channel);
 
         // Outbound channel: heartbeat.rs pushes HeartbeatRequest here; we
         // forward each one (converted to pb) into the tonic stream.
@@ -281,14 +294,27 @@ impl ManagerClient {
             endpoint
         };
 
-        let mut client = pb::runtime_harness_client::RuntimeHarnessClient::connect(endpoint)
-            .await
-            .with_context(|| {
-                format!(
-                    "report: could not connect to manager at {}",
-                    self.manager_addr
-                )
-            })?;
+        // NOTE: connect via `endpoint.connect()`, NOT `Client::connect(endpoint)`.
+        // The generated client's `connect` routes through
+        // `tonic::transport::Endpoint::new`, which for an https:// URI does:
+        //
+        //     return me.tls_config(ClientTlsConfig::new().with_enabled_roots());
+        //
+        // — silently REPLACING the TLS config set above with a default one. And
+        // `with_enabled_roots` discards its receiver, so with neither
+        // `tls-native-roots` nor `tls-webpki-roots` enabled the result is an
+        // EMPTY root store, no client identity, and no domain name. That is
+        // exactly what the guest saw: UnknownIssuer (no roots), "client auth
+        // requested but no cert available" (no identity), and verification
+        // against the URI's address (no domain). `Endpoint::connect` does not
+        // go through `Endpoint::new`, so the configured TLS survives.
+        let channel = endpoint.connect().await.with_context(|| {
+            format!(
+                "report: could not connect to manager at {}",
+                self.manager_addr
+            )
+        })?;
+        let mut client = pb::runtime_harness_client::RuntimeHarnessClient::new(channel);
 
         // Convert internal HarnessReport → pb on the fly as the stream is
         // consumed. ReceiverStream + map keeps this allocation-light: no
@@ -373,14 +399,27 @@ impl ManagerClient {
             endpoint
         };
 
-        let mut client = pb::runtime_harness_client::RuntimeHarnessClient::connect(endpoint)
-            .await
-            .with_context(|| {
-                format!(
-                    "vend_secret: could not connect to manager at {}",
-                    self.manager_addr
-                )
-            })?;
+        // NOTE: connect via `endpoint.connect()`, NOT `Client::connect(endpoint)`.
+        // The generated client's `connect` routes through
+        // `tonic::transport::Endpoint::new`, which for an https:// URI does:
+        //
+        //     return me.tls_config(ClientTlsConfig::new().with_enabled_roots());
+        //
+        // — silently REPLACING the TLS config set above with a default one. And
+        // `with_enabled_roots` discards its receiver, so with neither
+        // `tls-native-roots` nor `tls-webpki-roots` enabled the result is an
+        // EMPTY root store, no client identity, and no domain name. That is
+        // exactly what the guest saw: UnknownIssuer (no roots), "client auth
+        // requested but no cert available" (no identity), and verification
+        // against the URI's address (no domain). `Endpoint::connect` does not
+        // go through `Endpoint::new`, so the configured TLS survives.
+        let channel = endpoint.connect().await.with_context(|| {
+            format!(
+                "vend_secret: could not connect to manager at {}",
+                self.manager_addr
+            )
+        })?;
+        let mut client = pb::runtime_harness_client::RuntimeHarnessClient::new(channel);
 
         let ttl_proto = if req.ttl_secs > 0 {
             Some(prost_types::Duration {

services/harness/src/secrets.rs

@@ -379,6 +379,44 @@ impl SecretCache {
     }
 
     /// Background refresh sweep — keeps active secrets ahead of expiry.
+    /// Vend every declared secret once, at boot.
+    ///
+    /// `refresh_loop` only renews entries ALREADY cached, and `get_or_vend` is
+    /// driven by the workload asking over the secrets socket — so nothing ever
+    /// populated the cache on its own. A VM with no workload (a boot or
+    /// verification VM) therefore never exercised the vend path at all, which
+    /// is precisely the security-critical part of the boot contract: the
+    /// per-VM client cert presented over mTLS and the manager's allowlist
+    /// check. Warming here makes that path run on every boot, and removes the
+    /// first-use stall for workloads that do ask.
+    ///
+    /// Best-effort by design: a secret that cannot be vended must not stop the
+    /// VM from booting — the workload gets the error when it asks.
+    ///
+    /// Note this does mean a declared-but-unused secret is held in guest memory
+    /// for the life of the VM. That is already true of any secret the workload
+    /// touches once, and declared secrets are by definition ones it may use.
+    pub async fn prefetch_declared(self: &Arc<Self>) {
+        let names: Vec<String> = self.declared.iter().map(|e| e.key().clone()).collect();
+        if names.is_empty() {
+            return;
+        }
+        let total = names.len();
+        let mut ok = 0usize;
+        for name in names {
+            match self.get_or_vend(&name).await {
+                Ok(_) => ok += 1,
+                // Names only — never the value or the URI.
+                Err(e) => tracing::warn!(env = %name, error = %e, "secrets: prefetch failed"),
+            }
+        }
+        tracing::info!(
+            vended = ok,
+            declared = total,
+            "secrets: prefetched declared secrets"
+        );
+    }
+
     pub async fn refresh_loop(self: Arc<Self>) {
         let mut ticker = tokio::time::interval(Duration::from_secs(15));
         loop {

services/runtime-manager/src/backends/firecracker.rs

@@ -543,6 +543,25 @@ pub fn build_boot_args(
     let ip_param = net.kernel_ip_param();
     let manager_addr = manager_addr_for_guest(net);
 
+    // Declared secrets, one key each. The harness needs these to know which
+    // secrets a workload may request — with none declared its cache refuses
+    // every request, so VendSecret is never reached and a spec that declares a
+    // secret behaves exactly like one that declares none.
+    //
+    // One key per secret rather than a JSON blob: the harness assembles the
+    // JSON itself. Kernel args are whitespace-separated and build_boot_args
+    // strips quotes from values, so serialised JSON would arrive mangled.
+    let declared_secrets: String = req
+        .secrets
+        .iter()
+        .filter(|s| !s.env_var.is_empty() && !s.vault_ref.is_empty())
+        .map(|s| {
+            let env = s.env_var.replace([' ', '"', '\'', '='], "_");
+            let uri = s.vault_ref.replace([' ', '"', '\''], "_");
+            format!(" lantern.secret.{env}={uri}")
+        })
+        .collect();
+
     format!(
         "console=ttyS0 reboot=k panic=1 pci=off \
          {ip_param} \
@@ -551,7 +570,7 @@ pub fn build_boot_args(
          lantern.tls_cert={tls_cert_path} \
          lantern.tls_key={tls_key_path} \
          lantern.manager_ca={manager_ca_path} \
-         lantern.env.LANTERN_MANAGER_ADDR={manager_addr}{extra_env}"
+         lantern.env.LANTERN_MANAGER_ADDR={manager_addr}{declared_secrets}{extra_env}"
     )
 }