Add composer image attach and vision model payloads.

Users can attach PNG/JPEG/WebP in the desktop composer; the engine stores bounded artifact refs and hydrates Ollama images / OpenAI image_url parts only on the wire so text-only models still work and SQLite stays small.

Co-authored-by: Cursor <cursoragent@cursor.com>

Author: ShadowfetchStudio

Cargo.lock

@@ -38,6 +38,7 @@ ratatui = { version = "=0.30.2", default-features = false, features = ["crosster
 crossterm = { version = "=0.29.0", default-features = false, features = ["event-stream", "bracketed-paste", "use-dev-tty"] }
 unicode-segmentation = "1"
 unicode-width = "0.2"
+base64 = "0.22"
 
 [dev-dependencies]
 toml = "=0.9.12"

native/core/Cargo.toml

@@ -32,7 +32,9 @@ async fn main() -> Result<()> {
     let job = engine.start(StartRequest {
         workspace:project.clone(), session_id:None, model:None, mode:"code".into(), queue:false,
         task:"Start the existing server as a managed project background process named probe-server, with the exact command node server.mjs. Use background_start, then background_output to inspect its retained log until SERVER_READY appears. Report the actual marker following SERVER_READY and leave the server running. Do not use exec, change files or stop the server; the test harness will independently check its HTTP response and clean it up.".into()
-    }).await?;
+    
+            images: Vec::new(),
+        }).await?;
     let mut approvals = 0;
     let finished = tokio::time::timeout(Duration::from_secs(300), async {
         let completion = engine.wait(&job.id);

native/core/examples/probe_background.rs

@@ -31,7 +31,9 @@ async fn main() -> Result<()> {
     let job = engine.start(StartRequest {
         workspace:project.clone(),session_id:None,model:None,mode:"review".into(),queue:false,
         task:"Inspect billing.db using the native SQLite table/query tools. Discover its schema, then use SQL SUM to calculate total_cents for paid invoices only. Report the actual integer as total_cents=NUMBER. Do not use a shell or change any data/files.".into()
-    }).await?;
+    
+            images: Vec::new(),
+        }).await?;
     let finished = tokio::time::timeout(Duration::from_secs(240), engine.wait(&job.id)).await;
     engine.shutdown().await?;
     let finished = finished??;

native/core/examples/probe_sqlite.rs

@@ -70,7 +70,9 @@ async fn main() -> Result<()> {
             tokio::time::sleep(Duration::from_millis(50)).await;
         }
     });
-    let request=StartRequest{workspace:project.clone(),task:"Fix the add function in src/lib.rs so it adds its two inputs. Inspect the file using tools, make the smallest change, then run exactly `cargo test --offline --lib` using exec with no cwd parameter. Preserve the existing tests and package files. Finish only after seeing the test result.".into(),session_id:None,model:None,mode:"code".into(),queue:false};
+    let request=StartRequest{workspace:project.clone(),task:"Fix the add function in src/lib.rs so it adds its two inputs. Inspect the file using tools, make the smallest change, then run exactly `cargo test --offline --lib` using exec with no cwd parameter. Preserve the existing tests and package files. Finish only after seeing the test result.".into(),session_id:None,model:None,mode:"code".into(),queue:false
+            images: Vec::new(),
+        };
     let first = engine.start(request).await?;
     let completed =
         tokio::time::timeout(Duration::from_secs(240), engine.wait(&first.id)).await??;
@@ -107,7 +109,9 @@ async fn main() -> Result<()> {
         "Independent verification failed: {}",
         verified.stderr
     );
-    let second=engine.start(StartRequest{workspace:project.clone(),task:"Read src/lib.rs again and tell me what add(-2, 1) now returns. Do not change files or run commands.".into(),session_id:Some(first.session_id.clone()),model:None,mode:"review".into(),queue:false}).await?;
+    let second=engine.start(StartRequest{workspace:project.clone(),task:"Read src/lib.rs again and tell me what add(-2, 1) now returns. Do not change files or run commands.".into(),session_id:Some(first.session_id.clone()),model:None,mode:"review".into(),queue:false
+            images: Vec::new(),
+        }).await?;
     let continued =
         tokio::time::timeout(Duration::from_secs(180), engine.wait(&second.id)).await??;
     ensure!(

native/core/examples/probe_task.rs

@@ -244,6 +244,14 @@ pub fn runaway_action(repeats: usize) -> RunawayAction {
 }
 
 pub fn capability_profile(provider: &str, context_limit: usize) -> CapabilityProfile {
+    capability_profile_for(provider, "", context_limit)
+}
+
+pub fn capability_profile_for(
+    provider: &str,
+    model: &str,
+    context_limit: usize,
+) -> CapabilityProfile {
     let local = matches!(
         provider,
         "ollama" | "local" | "llamacpp" | "lmstudio" | "vllm" | "mock"
@@ -254,7 +262,8 @@ pub fn capability_profile(provider: &str, context_limit: usize) -> CapabilityPro
         tools: provider != "mock",
         structured_output: !local,
         reasoning: matches!(provider, "openai" | "openrouter" | "anthropic"),
-        multimodal: matches!(provider, "openai" | "openrouter" | "ollama"),
+        multimodal: crate::vision::model_supports_vision(provider, model)
+            || (model.is_empty() && matches!(provider, "openai" | "openrouter")),
         streaming: true,
         parallel_tools: !matches!(provider, "ollama" | "llamacpp"),
         quirks: match provider {

native/core/src/autonomy.rs

@@ -76,6 +76,12 @@ pub fn repair_incomplete(messages: &mut Vec<Value>) {
 }
 
 pub fn estimate_tokens(value: &Value) -> usize {
+    if let Some(arr) = value.as_array() {
+        return arr.iter().map(estimate_tokens).sum();
+    }
+    if value.get("role").is_some() {
+        return crate::vision::estimate_message_tokens(value);
+    }
     value.to_string().len().div_ceil(3)
 }
 

native/core/src/context.rs

@@ -59,6 +59,9 @@ pub struct StartRequest {
     pub mode: String,
     #[serde(default)]
     pub queue: bool,
+    /// Workspace-relative image attachment paths (png/jpeg/webp).
+    #[serde(default)]
+    pub images: Vec<String>,
 }
 fn default_mode() -> String {
     "code".into()
@@ -71,6 +74,8 @@ pub struct Job {
     pub session_id: String,
     pub task_id: String,
     pub task: String,
+    #[serde(default)]
+    pub images: Vec<String>,
     pub status: String,
     pub mode: String,
     pub model: String,
@@ -328,8 +333,14 @@ impl Engine {
             "Application is shutting down"
         );
         ensure!(
-            !request.task.trim().is_empty() && request.task.len() <= 128_000,
-            "Task must contain between 1 and 128000 bytes"
+            (!request.task.trim().is_empty() || !request.images.is_empty())
+                && request.task.len() <= 128_000,
+            "Task must contain between 1 and 128000 bytes, or include an image attachment"
+        );
+        ensure!(
+            request.images.len() <= crate::vision::MAX_IMAGES_PER_TURN,
+            "At most {} images may be attached per task",
+            crate::vision::MAX_IMAGES_PER_TURN
         );
         ensure!(
             matches!(request.mode.as_str(), "code" | "plan" | "review")
@@ -417,6 +428,7 @@ impl Engine {
             session_id: sid,
             task_id: crate::id(),
             task: request.task,
+            images: request.images,
             status: "queued".into(),
             mode: request.mode,
             model: if context.command.is_some() {
@@ -881,9 +893,15 @@ impl Engine {
             system.push_str(extra);
         }
         messages.insert(0, json!({"role":"system","content":system}));
-        messages.push(json!({"role":"user","content":job.task}));
+        let image_refs = crate::vision::refs_from_paths(&running.workspace, &job.images)?;
+        crate::vision::ensure_vision_or_bail(
+            &running.config.model.provider,
+            &running.config.model.name,
+            image_refs.len(),
+        )?;
+        messages.push(crate::vision::user_message(&job.task, &image_refs));
         self.0.store.save_messages(&job.id, &messages)?;
-        events.emit("agent.started",json!({"job_id":job.id,"task":job.task,"mode":job.mode,"model":job.model,"native":true}))?;
+        events.emit("agent.started",json!({"job_id":job.id,"task":job.task,"mode":job.mode,"model":job.model,"native":true,"images":job.images}))?;
         if let Some(decision) = &job.routing {
             events.emit(
                 if decision.fallback_reason.is_some() {
@@ -965,7 +983,15 @@ impl Engine {
                 let mut flushed = Instant::now();
                 let mut event_error = None;
                 let response = model
-                    .chat(&messages, &schemas, running.cancel.clone(), |delta| {
+                    .chat(
+                        &crate::vision::hydrate_for_provider(
+                            &messages,
+                            &running.workspace,
+                            &running.config.model.provider,
+                        )?,
+                        &schemas,
+                        running.cancel.clone(),
+                        |delta| {
                         partial.push_str(delta);
                         pending.push_str(delta);
                         if pending.len() >= 4000 || flushed.elapsed() >= Duration::from_millis(80) {

native/core/src/engine.rs

@@ -174,6 +174,7 @@ impl Engine {
                         model: None,
                         mode: milestone["mode"].as_str().unwrap_or("code").into(),
                         queue: true,
+                        images: Vec::new(),
                     },
                     LaunchContext {
                         system_context: Some(context),

native/core/src/engine/goals.rs

@@ -32,6 +32,7 @@ pub mod service;
 pub mod sqlite;
 pub mod store;
 pub mod tools;
+pub mod vision;
 pub mod workflows;
 pub mod workspace;
 pub mod worktrees;