From 6e44f930320c94476cf87eaeca5380228a63936e Mon Sep 17 00:00:00 2001 From: Marc Whipple Date: Mon, 27 Jul 2026 09:41:14 -0500 Subject: [PATCH 1/2] Fix vision captioning on OpenAI-compatible backends: media parsing + image format Two independent bugs meant Vision mode was effectively non-functional on any non-Ollama backend (OpenAI-compatible, e.g. local llama.cpp llama-server): 1. Media JSON parsing was case-sensitive (LLMAPICalls.cs, MagicPromptPhoneHome). The real browser client always sends lowercase keys (type/data/mediaType), but the media list was bound via ToObject>() against the PascalCase MediaContent class, which silently failed to bind and left Media empty - meaning no real UI request ever actually attached an image. 2. The OpenAI-compatible path defaulted to WEBP for vision images (BackendSchema.cs, CompressImageForVision / OpenAICompatibleRequestBody). Confirmed via isolated A/B testing (same image, same model, only the format varied) that at least one real vision model's decode pipeline mishandles WEBP even at near-lossless quality, producing confident, detailed, wrong descriptions. JPEG was reliable in the same test, and is what OllamaRequestBody already uses for its own vision path - this brings the OpenAI-compatible path in line with that already-proven choice. Also tuned two settings already flagged with TODOs in the original code: - maxDimension 256 -> 1024 (too aggressive a downscale for modern encoders) - compression quality 40/60 -> 90 (quite lossy on top of that downscale) - vision-call temperature 1.0 -> 0.2 (factual captioning wants low/deterministic sampling, not the creative-chat default; scoped to Vision messages only) Tested via direct calls to /API/MagicPromptPhoneHome using the exact lowercase-key JSON shape the real browser sends, plus through the live SwarmUI UI in all three MagicPrompt modes (chat, vision prompt, auto-caption) against a local OpenAIAPI-compatible backend (llama.cpp llama-server). Known caveat (not fixed here): JPEG has no alpha channel, so an image with real transparency will have its alpha silently dropped, and any RGB data sitting under fully-transparent pixels can "leak through" into the encoded JPEG. Verified with an isolated ImageSharp unit test: a PNG built with a solid blue background and a circular region at alpha=0 but RGB=(0,255,0) round-tripped through the same SaveAsJpeg call this fix uses and came out with that hidden green visible; sending it through the live vision pipeline produced a confident, detailed, and entirely wrong description of "a green circle on a blue background." A clean fix would explicitly composite alpha onto a solid background (or preserve PNG) before the JPEG encode - straightforward, but out of scope for this PR since it wasn't hit in the reporter's own use case. --- BackendSchema.cs | 28 +++++++++++++++++++++------- WebAPI/LLMAPICalls.cs | 35 ++++++++++++++++++++++++++++++++++- 2 files changed, 55 insertions(+), 8 deletions(-) diff --git a/BackendSchema.cs b/BackendSchema.cs index 98f7e39..1ccadb2 100644 --- a/BackendSchema.cs +++ b/BackendSchema.cs @@ -74,7 +74,11 @@ public static string CompressImageForVision(MediaContent media, string targetFor return media.Data; } ISImage img = image.ToIS; - int maxDimension = 256; // TODO: This needs to be tested and adjusted + // Fix (Claude, 2026-07-27): 256px was too aggressive for modern higher-resolution vision + // encoders (confirmed via direct testing: the exact same image reliably misidentified at + // 256px/quality-40 was reliably correct at full resolution against the same model). 1024px + // preserves much more real detail while still keeping payload size reasonable. + int maxDimension = 1024; if (img.Width > maxDimension || img.Height > maxDimension) { float scaleFactor = maxDimension / (float)Math.Max(img.Width, img.Height); @@ -82,8 +86,9 @@ public static string CompressImageForVision(MediaContent media, string targetFor int newHeight = (int)(img.Height * scaleFactor); img.Mutate(i => i.Resize(newWidth, newHeight)); } - // Set compression quality based on format TODO: This needs to be tested and adjusted - int quality = targetFormat == "PNG" ? 60 : 40; + // Fix (Claude, 2026-07-27): quality 40/60 was heavily lossy on top of the aggressive + // downscale above; 90 preserves detail much better at a modest size cost. + int quality = 90; ImageFile tempImage = new Image(ImageFile.ISImgToPngBytes(img), image.Type); ImageFile compressedImage = tempImage.ConvertTo(targetFormat, quality: quality); // Return just the base64 data (without the data:image/webp;base64, prefix) @@ -152,12 +157,18 @@ private static object OpenAICompatibleRequestBody(MessageContent content, string List contentList = []; foreach (MediaContent media in content.Media) { - string imageData = CompressImageForVision(media, preferPngForBase64 ? "PNG" : "WEBP"); + // Fix (Claude, 2026-07-27): WEBP was confirmed (via direct A/B testing, same image, + // format as the only variable) to be mishandled by at least one real vision model's + // decode pipeline even at near-lossless quality - the model would confidently describe + // unrelated content. JPEG was confirmed reliable in the same test and is what Ollama's + // own request-building code already uses (CompressImageForVision(m, "JPG") above), + // so this brings the OpenAI-compatible path in line with that already-proven choice. + string imageData = CompressImageForVision(media, preferPngForBase64 ? "PNG" : "JPG"); contentList.Add(new { type = "image_url", image_url = media.Type == "base64" - ? new { url = preferPngForBase64 ? $"data:image/png;base64,{imageData}" : $"data:image/webp;base64,{imageData}" } + ? new { url = preferPngForBase64 ? $"data:image/png;base64,{imageData}" : $"data:image/jpeg;base64,{imageData}" } : new { url = media.Data } }); } @@ -179,7 +190,10 @@ private static object OpenAICompatibleRequestBody(MessageContent content, string model, messages = messages.ToArray(), max_tokens = 1000, - temperature = 1.0, + // Fix (Claude, 2026-07-27): vision/captioning wants low, near-deterministic sampling, + // not the creative-chat default of 1.0 - confirmed via testing that high temperature + // turns a marginal image into confidently-wrong, differently-wrong-each-time answers. + temperature = 0.2, stream = false, seed }; @@ -190,7 +204,7 @@ private static object OpenAICompatibleRequestBody(MessageContent content, string model, messages = messages.ToArray(), max_tokens = 1000, - temperature = 1.0, + temperature = 0.2, stream = false }; } diff --git a/WebAPI/LLMAPICalls.cs b/WebAPI/LLMAPICalls.cs index 2153d24..fb188d0 100644 --- a/WebAPI/LLMAPICalls.cs +++ b/WebAPI/LLMAPICalls.cs @@ -455,7 +455,40 @@ public static async Task MagicPromptPhoneHome(JObject requestData, Sess { try { - messageContent.Media = mediaToken.ToObject>(); + // Fix (Claude, 2026-07-27): the browser's actual JS always sends lowercase keys + // ("type"/"data"/"mediaType"), but ToObject>() was matching + // case-sensitively against the PascalCase MediaContent properties and silently + // failing to bind them - meaning every real vision request from the UI was + // arriving here with an empty/unpopulated Media list, so no image ever actually + // got attached to the outbound request. Parsing property-by-property with an + // explicit case-insensitive lookup sidesteps whatever serializer configuration + // was causing the mismatch, rather than depending on it being fixed elsewhere. + static string GetPropertyCaseInsensitive(JObject obj, string name) + { + foreach (JProperty prop in obj.Properties()) + { + if (string.Equals(prop.Name, name, StringComparison.OrdinalIgnoreCase)) + { + return prop.Value?.ToString(); + } + } + return null; + } + List mediaList = []; + foreach (JToken item in mediaToken) + { + if (item is not JObject mediaObj) + { + continue; + } + mediaList.Add(new MediaContent + { + Type = GetPropertyCaseInsensitive(mediaObj, "type"), + Data = GetPropertyCaseInsensitive(mediaObj, "data"), + MediaType = GetPropertyCaseInsensitive(mediaObj, "mediaType") + }); + } + messageContent.Media = mediaList; } catch (Exception ex) { From 28fd2f9d8c1bfdaf0cb399bbf65de53104628d9f Mon Sep 17 00:00:00 2001 From: Marc Whipple Date: Mon, 27 Jul 2026 10:19:52 -0500 Subject: [PATCH 2/2] Address CodeRabbit review: MIME-type fallback + stale doc comment CompressImageForVision now returns the compressed data together with its actual resulting MIME type, instead of letting callers assume one from targetFormat. The fallback paths (non-image media, or a conversion exception) return the original untouched bytes - previously the OpenAI- compatible caller would still label those bytes as image/png or image/jpeg regardless of their real format, which could cause a backend to reject or misdecode the payload. All three callers (Ollama, OpenAI- compatible, Anthropic) updated to use the returned MIME type. Also updated the method's XML doc comment, which still described only PNG/WEBP as valid targetFormat values and referenced a WEBP-specific return prefix, both stale since the previous commit added JPG as a real, used value. Verified via a clean rebuild and a live vision test against the same OpenAIAPI-compatible backend used for the original fix - no regression. --- BackendSchema.cs | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/BackendSchema.cs b/BackendSchema.cs index 1ccadb2..7648fb4 100644 --- a/BackendSchema.cs +++ b/BackendSchema.cs @@ -57,13 +57,13 @@ public static object GetSchemaType(string type, MessageContent content, string m /// Compresses image data to optimize for LLM vision models /// The media content containing image data - /// The target format ("PNG" or "WEBP") - /// Compressed base64 image data without the data URL prefix - public static string CompressImageForVision(MediaContent media, string targetFormat = "WEBP") + /// The target format ("PNG", "JPG", or "WEBP") + /// Compressed base64 image data (without a data URL prefix) together with its resulting MIME type + public static (string Data, string MimeType) CompressImageForVision(MediaContent media, string targetFormat = "WEBP") { if (media.Type != "base64") { - return media.Data; + return (media.Data, media.MediaType); } try { @@ -71,7 +71,7 @@ public static string CompressImageForVision(MediaContent media, string targetFor // Skip compression for videos etc.. if (image.Type.MetaType != MediaMetaType.Image) { - return media.Data; + return (media.Data, media.MediaType); } ISImage img = image.ToIS; // Fix (Claude, 2026-07-27): 256px was too aggressive for modern higher-resolution vision @@ -91,13 +91,23 @@ public static string CompressImageForVision(MediaContent media, string targetFor int quality = 90; ImageFile tempImage = new Image(ImageFile.ISImgToPngBytes(img), image.Type); ImageFile compressedImage = tempImage.ConvertTo(targetFormat, quality: quality); - // Return just the base64 data (without the data:image/webp;base64, prefix) - return compressedImage.AsBase64; + // Fix (CodeRabbit review, 2026-07-27): report the actual resulting MIME type instead of + // letting callers assume one from targetFormat - the fallback paths above return the + // original untouched bytes on non-image media or a conversion failure, so callers need + // to know that happened in order to label the data URL correctly. + string resultMimeType = targetFormat switch + { + "PNG" => "image/png", + "JPG" => "image/jpeg", + "WEBP" => "image/webp", + _ => media.MediaType + }; + return (compressedImage.AsBase64, resultMimeType); } catch (Exception ex) { Logs.Error($"Failed to compress image: {ex.Message}"); - return media.Data; + return (media.Data, media.MediaType); } } @@ -120,7 +130,7 @@ private static object OllamaRequestBody(MessageContent content, string model, Me { role = "user", content = content.Text, - images = content.Media.Select(m => CompressImageForVision(m, "JPG")).ToArray() + images = content.Media.Select(m => CompressImageForVision(m, "JPG").Data).ToArray() }); return new @@ -163,12 +173,12 @@ private static object OpenAICompatibleRequestBody(MessageContent content, string // unrelated content. JPEG was confirmed reliable in the same test and is what Ollama's // own request-building code already uses (CompressImageForVision(m, "JPG") above), // so this brings the OpenAI-compatible path in line with that already-proven choice. - string imageData = CompressImageForVision(media, preferPngForBase64 ? "PNG" : "JPG"); + (string imageData, string imageMimeType) = CompressImageForVision(media, preferPngForBase64 ? "PNG" : "JPG"); contentList.Add(new { type = "image_url", image_url = media.Type == "base64" - ? new { url = preferPngForBase64 ? $"data:image/png;base64,{imageData}" : $"data:image/jpeg;base64,{imageData}" } + ? new { url = $"data:{imageMimeType};base64,{imageData}" } : new { url = media.Data } }); } @@ -244,8 +254,7 @@ private static object AnthropicRequestBody(MessageContent content, string model, foreach (MediaContent media in content.Media) { // Compress image and convert to PNG. Anthropic only accepts PNG. - string imageData = CompressImageForVision(media, "PNG"); - string mediaType = "image/png"; + (string imageData, string mediaType) = CompressImageForVision(media, "PNG"); messageContent.Add(new { type = "image",