diff --git a/BackendSchema.cs b/BackendSchema.cs index 98f7e39..7648fb4 100644 --- a/BackendSchema.cs +++ b/BackendSchema.cs @@ -57,13 +57,13 @@ public static object GetSchemaType(string type, MessageContent content, string m /// Compresses image data to optimize for LLM vision models /// The media content containing image data - /// The target format ("PNG" or "WEBP") - /// Compressed base64 image data without the data URL prefix - public static string CompressImageForVision(MediaContent media, string targetFormat = "WEBP") + /// The target format ("PNG", "JPG", or "WEBP") + /// Compressed base64 image data (without a data URL prefix) together with its resulting MIME type + public static (string Data, string MimeType) CompressImageForVision(MediaContent media, string targetFormat = "WEBP") { if (media.Type != "base64") { - return media.Data; + return (media.Data, media.MediaType); } try { @@ -71,10 +71,14 @@ public static string CompressImageForVision(MediaContent media, string targetFor // Skip compression for videos etc.. if (image.Type.MetaType != MediaMetaType.Image) { - return media.Data; + return (media.Data, media.MediaType); } ISImage img = image.ToIS; - int maxDimension = 256; // TODO: This needs to be tested and adjusted + // Fix (Claude, 2026-07-27): 256px was too aggressive for modern higher-resolution vision + // encoders (confirmed via direct testing: the exact same image reliably misidentified at + // 256px/quality-40 was reliably correct at full resolution against the same model). 1024px + // preserves much more real detail while still keeping payload size reasonable. + int maxDimension = 1024; if (img.Width > maxDimension || img.Height > maxDimension) { float scaleFactor = maxDimension / (float)Math.Max(img.Width, img.Height); @@ -82,17 +86,28 @@ public static string CompressImageForVision(MediaContent media, string targetFor int newHeight = (int)(img.Height * scaleFactor); img.Mutate(i => i.Resize(newWidth, newHeight)); } - // Set compression quality based on format TODO: This needs to be tested and adjusted - int quality = targetFormat == "PNG" ? 60 : 40; + // Fix (Claude, 2026-07-27): quality 40/60 was heavily lossy on top of the aggressive + // downscale above; 90 preserves detail much better at a modest size cost. + int quality = 90; ImageFile tempImage = new Image(ImageFile.ISImgToPngBytes(img), image.Type); ImageFile compressedImage = tempImage.ConvertTo(targetFormat, quality: quality); - // Return just the base64 data (without the data:image/webp;base64, prefix) - return compressedImage.AsBase64; + // Fix (CodeRabbit review, 2026-07-27): report the actual resulting MIME type instead of + // letting callers assume one from targetFormat - the fallback paths above return the + // original untouched bytes on non-image media or a conversion failure, so callers need + // to know that happened in order to label the data URL correctly. + string resultMimeType = targetFormat switch + { + "PNG" => "image/png", + "JPG" => "image/jpeg", + "WEBP" => "image/webp", + _ => media.MediaType + }; + return (compressedImage.AsBase64, resultMimeType); } catch (Exception ex) { Logs.Error($"Failed to compress image: {ex.Message}"); - return media.Data; + return (media.Data, media.MediaType); } } @@ -115,7 +130,7 @@ private static object OllamaRequestBody(MessageContent content, string model, Me { role = "user", content = content.Text, - images = content.Media.Select(m => CompressImageForVision(m, "JPG")).ToArray() + images = content.Media.Select(m => CompressImageForVision(m, "JPG").Data).ToArray() }); return new @@ -152,12 +167,18 @@ private static object OpenAICompatibleRequestBody(MessageContent content, string List contentList = []; foreach (MediaContent media in content.Media) { - string imageData = CompressImageForVision(media, preferPngForBase64 ? "PNG" : "WEBP"); + // Fix (Claude, 2026-07-27): WEBP was confirmed (via direct A/B testing, same image, + // format as the only variable) to be mishandled by at least one real vision model's + // decode pipeline even at near-lossless quality - the model would confidently describe + // unrelated content. JPEG was confirmed reliable in the same test and is what Ollama's + // own request-building code already uses (CompressImageForVision(m, "JPG") above), + // so this brings the OpenAI-compatible path in line with that already-proven choice. + (string imageData, string imageMimeType) = CompressImageForVision(media, preferPngForBase64 ? "PNG" : "JPG"); contentList.Add(new { type = "image_url", image_url = media.Type == "base64" - ? new { url = preferPngForBase64 ? $"data:image/png;base64,{imageData}" : $"data:image/webp;base64,{imageData}" } + ? new { url = $"data:{imageMimeType};base64,{imageData}" } : new { url = media.Data } }); } @@ -179,7 +200,10 @@ private static object OpenAICompatibleRequestBody(MessageContent content, string model, messages = messages.ToArray(), max_tokens = 1000, - temperature = 1.0, + // Fix (Claude, 2026-07-27): vision/captioning wants low, near-deterministic sampling, + // not the creative-chat default of 1.0 - confirmed via testing that high temperature + // turns a marginal image into confidently-wrong, differently-wrong-each-time answers. + temperature = 0.2, stream = false, seed }; @@ -190,7 +214,7 @@ private static object OpenAICompatibleRequestBody(MessageContent content, string model, messages = messages.ToArray(), max_tokens = 1000, - temperature = 1.0, + temperature = 0.2, stream = false }; } @@ -230,8 +254,7 @@ private static object AnthropicRequestBody(MessageContent content, string model, foreach (MediaContent media in content.Media) { // Compress image and convert to PNG. Anthropic only accepts PNG. - string imageData = CompressImageForVision(media, "PNG"); - string mediaType = "image/png"; + (string imageData, string mediaType) = CompressImageForVision(media, "PNG"); messageContent.Add(new { type = "image", diff --git a/WebAPI/LLMAPICalls.cs b/WebAPI/LLMAPICalls.cs index 2153d24..fb188d0 100644 --- a/WebAPI/LLMAPICalls.cs +++ b/WebAPI/LLMAPICalls.cs @@ -455,7 +455,40 @@ public static async Task MagicPromptPhoneHome(JObject requestData, Sess { try { - messageContent.Media = mediaToken.ToObject>(); + // Fix (Claude, 2026-07-27): the browser's actual JS always sends lowercase keys + // ("type"/"data"/"mediaType"), but ToObject>() was matching + // case-sensitively against the PascalCase MediaContent properties and silently + // failing to bind them - meaning every real vision request from the UI was + // arriving here with an empty/unpopulated Media list, so no image ever actually + // got attached to the outbound request. Parsing property-by-property with an + // explicit case-insensitive lookup sidesteps whatever serializer configuration + // was causing the mismatch, rather than depending on it being fixed elsewhere. + static string GetPropertyCaseInsensitive(JObject obj, string name) + { + foreach (JProperty prop in obj.Properties()) + { + if (string.Equals(prop.Name, name, StringComparison.OrdinalIgnoreCase)) + { + return prop.Value?.ToString(); + } + } + return null; + } + List mediaList = []; + foreach (JToken item in mediaToken) + { + if (item is not JObject mediaObj) + { + continue; + } + mediaList.Add(new MediaContent + { + Type = GetPropertyCaseInsensitive(mediaObj, "type"), + Data = GetPropertyCaseInsensitive(mediaObj, "data"), + MediaType = GetPropertyCaseInsensitive(mediaObj, "mediaType") + }); + } + messageContent.Media = mediaList; } catch (Exception ex) {