Skip to content

Fix vision captioning on OpenAI-compatible backends: media parsing + image format #67

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
LegalInspiration wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Fix vision captioning on OpenAI-compatible backends: media parsing + image format #67

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6e44f93
Fix vision captioning on OpenAI-compatible backends: media parsing + …
Jul 27, 2026
28fd2f9
Address CodeRabbit review: MIME-type fallback + stale doc comment
Jul 27, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 41 additions & 18 deletions BackendSchema.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,42 +57,57 @@ public static object GetSchemaType(string type, MessageContent content, string m

/// <summary>Compresses image data to optimize for LLM vision models</summary>
/// <param name="media">The media content containing image data</param>
/// <param name="targetFormat">The target format ("PNG" or "WEBP")</param>
/// <returns>Compressed base64 image data without the data URL prefix</returns>
public static string CompressImageForVision(MediaContent media, string targetFormat = "WEBP")
/// <param name="targetFormat">The target format ("PNG", "JPG", or "WEBP")</param>
/// <returns>Compressed base64 image data (without a data URL prefix) together with its resulting MIME type</returns>
public static (string Data, string MimeType) CompressImageForVision(MediaContent media, string targetFormat = "WEBP")
{
if (media.Type != "base64")
{
return media.Data;
return (media.Data, media.MediaType);
}
try
{
ImageFile image = ImageFile.FromDataString($"data:{media.MediaType};base64,{media.Data}");
// Skip compression for videos etc..
if (image.Type.MetaType != MediaMetaType.Image)
{
return media.Data;
return (media.Data, media.MediaType);
}
ISImage img = image.ToIS;
int maxDimension = 256; // TODO: This needs to be tested and adjusted
// Fix (Claude, 2026-07-27): 256px was too aggressive for modern higher-resolution vision
// encoders (confirmed via direct testing: the exact same image reliably misidentified at
// 256px/quality-40 was reliably correct at full resolution against the same model). 1024px
// preserves much more real detail while still keeping payload size reasonable.
int maxDimension = 1024;
if (img.Width > maxDimension || img.Height > maxDimension)
{
float scaleFactor = maxDimension / (float)Math.Max(img.Width, img.Height);
int newWidth = (int)(img.Width * scaleFactor);
int newHeight = (int)(img.Height * scaleFactor);
img.Mutate(i => i.Resize(newWidth, newHeight));
}
// Set compression quality based on format TODO: This needs to be tested and adjusted
int quality = targetFormat == "PNG" ? 60 : 40;
// Fix (Claude, 2026-07-27): quality 40/60 was heavily lossy on top of the aggressive
// downscale above; 90 preserves detail much better at a modest size cost.
int quality = 90;
Comment thread
coderabbitai[bot] marked this conversation as resolved.
ImageFile tempImage = new Image(ImageFile.ISImgToPngBytes(img), image.Type);
ImageFile compressedImage = tempImage.ConvertTo(targetFormat, quality: quality);
// Return just the base64 data (without the data:image/webp;base64, prefix)
return compressedImage.AsBase64;
// Fix (CodeRabbit review, 2026-07-27): report the actual resulting MIME type instead of
// letting callers assume one from targetFormat - the fallback paths above return the
// original untouched bytes on non-image media or a conversion failure, so callers need
// to know that happened in order to label the data URL correctly.
string resultMimeType = targetFormat switch
{
"PNG" => "image/png",
"JPG" => "image/jpeg",
"WEBP" => "image/webp",
_ => media.MediaType
};
return (compressedImage.AsBase64, resultMimeType);
}
catch (Exception ex)
{
Logs.Error($"Failed to compress image: {ex.Message}");
return media.Data;
return (media.Data, media.MediaType);
}
}

Expand All @@ -115,7 +130,7 @@ private static object OllamaRequestBody(MessageContent content, string model, Me
{
role = "user",
content = content.Text,
images = content.Media.Select(m => CompressImageForVision(m, "JPG")).ToArray()
images = content.Media.Select(m => CompressImageForVision(m, "JPG").Data).ToArray()
});

return new
Expand Down Expand Up @@ -152,12 +167,18 @@ private static object OpenAICompatibleRequestBody(MessageContent content, string
List<object> contentList = [];
foreach (MediaContent media in content.Media)
{
string imageData = CompressImageForVision(media, preferPngForBase64 ? "PNG" : "WEBP");
// Fix (Claude, 2026-07-27): WEBP was confirmed (via direct A/B testing, same image,
// format as the only variable) to be mishandled by at least one real vision model's
// decode pipeline even at near-lossless quality - the model would confidently describe
// unrelated content. JPEG was confirmed reliable in the same test and is what Ollama's
// own request-building code already uses (CompressImageForVision(m, "JPG") above),
// so this brings the OpenAI-compatible path in line with that already-proven choice.
(string imageData, string imageMimeType) = CompressImageForVision(media, preferPngForBase64 ? "PNG" : "JPG");
contentList.Add(new
{
type = "image_url",
image_url = media.Type == "base64"
? new { url = preferPngForBase64 ? $"data:image/png;base64,{imageData}" : $"data:image/webp;base64,{imageData}" }
? new { url = $"data:{imageMimeType};base64,{imageData}" }
: new { url = media.Data }
});
}
Expand All @@ -179,7 +200,10 @@ private static object OpenAICompatibleRequestBody(MessageContent content, string
model,
messages = messages.ToArray(),
max_tokens = 1000,
temperature = 1.0,
// Fix (Claude, 2026-07-27): vision/captioning wants low, near-deterministic sampling,
// not the creative-chat default of 1.0 - confirmed via testing that high temperature
// turns a marginal image into confidently-wrong, differently-wrong-each-time answers.
temperature = 0.2,
stream = false,
seed
};
Expand All @@ -190,7 +214,7 @@ private static object OpenAICompatibleRequestBody(MessageContent content, string
model,
messages = messages.ToArray(),
max_tokens = 1000,
temperature = 1.0,
temperature = 0.2,
stream = false
};
}
Expand Down Expand Up @@ -230,8 +254,7 @@ private static object AnthropicRequestBody(MessageContent content, string model,
foreach (MediaContent media in content.Media)
{
// Compress image and convert to PNG. Anthropic only accepts PNG.
string imageData = CompressImageForVision(media, "PNG");
string mediaType = "image/png";
(string imageData, string mediaType) = CompressImageForVision(media, "PNG");
messageContent.Add(new
{
type = "image",
Expand Down
35 changes: 34 additions & 1 deletion WebAPI/LLMAPICalls.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -455,7 +455,40 @@ public static async Task<JObject> MagicPromptPhoneHome(JObject requestData, Sess
{
try
{
messageContent.Media = mediaToken.ToObject<List<MediaContent>>();
// Fix (Claude, 2026-07-27): the browser's actual JS always sends lowercase keys
// ("type"/"data"/"mediaType"), but ToObject<List<MediaContent>>() was matching
// case-sensitively against the PascalCase MediaContent properties and silently
// failing to bind them - meaning every real vision request from the UI was
// arriving here with an empty/unpopulated Media list, so no image ever actually
// got attached to the outbound request. Parsing property-by-property with an
// explicit case-insensitive lookup sidesteps whatever serializer configuration
// was causing the mismatch, rather than depending on it being fixed elsewhere.
static string GetPropertyCaseInsensitive(JObject obj, string name)
{
foreach (JProperty prop in obj.Properties())
{
if (string.Equals(prop.Name, name, StringComparison.OrdinalIgnoreCase))
{
return prop.Value?.ToString();
}
}
return null;
}
List<MediaContent> mediaList = [];
foreach (JToken item in mediaToken)
{
if (item is not JObject mediaObj)
{
continue;
}
mediaList.Add(new MediaContent
{
Type = GetPropertyCaseInsensitive(mediaObj, "type"),
Data = GetPropertyCaseInsensitive(mediaObj, "data"),
MediaType = GetPropertyCaseInsensitive(mediaObj, "mediaType")
});
}
messageContent.Media = mediaList;
}
catch (Exception ex)
{
Expand Down