From 988ee535d54b4d1014f56db60f99113c01cef91a Mon Sep 17 00:00:00 2001 From: "open-swe[bot]" Date: Mon, 11 May 2026 18:25:34 +0000 Subject: [PATCH] feat(model-profiles): add input/output MIME type fields to ModelProfile Adds informational input_mime_types and output_mime_types dict fields to ModelProfile, keyed by models.dev modality names ('image', 'audio', 'pdf', 'video'). Augments the CLI to dispatch provider-level vs model-level overrides by ModelProfile field name so non-scalar provider-level fields (like the new MIME maps) are routed correctly. Backfills anthropic, openai, and perplexity profile_augmentations.toml with documented MIME types and regenerates their _profiles.py. Co-authored-by: Mason Daugherty <61371264+mdrxy@users.noreply.github.com> --- .../language_models/model_profile.py | 45 +- .../langchain_model_profiles/cli.py | 39 +- .../tests/unit_tests/test_cli.py | 130 ++++ .../langchain_anthropic/data/_profiles.py | 253 ++++++++ .../data/profile_augmentations.toml | 4 + .../openai/langchain_openai/data/_profiles.py | 593 ++++++++++++++++++ .../data/profile_augmentations.toml | 13 + .../langchain_perplexity/data/_profiles.py | 8 + .../data/profile_augmentations.toml | 3 + 9 files changed, 1083 insertions(+), 5 deletions(-) diff --git a/libs/core/langchain_core/language_models/model_profile.py b/libs/core/langchain_core/language_models/model_profile.py index b556c0a6467f0..6ecadc96607c1 100644 --- a/libs/core/langchain_core/language_models/model_profile.py +++ b/libs/core/langchain_core/language_models/model_profile.py @@ -50,7 +50,6 @@ class ModelProfile(TypedDict, total=False): image_inputs: bool """Whether image inputs are supported.""" - # TODO: add more detail about formats? image_url_inputs: bool """Whether [image URL inputs](https://docs.langchain.com/oss/python/langchain/models#multimodal) @@ -59,17 +58,36 @@ class ModelProfile(TypedDict, total=False): pdf_inputs: bool """Whether [PDF inputs](https://docs.langchain.com/oss/python/langchain/models#multimodal) are supported.""" - # TODO: add more detail about formats? e.g. bytes or base64 audio_inputs: bool """Whether [audio inputs](https://docs.langchain.com/oss/python/langchain/models#multimodal) are supported.""" - # TODO: add more detail about formats? e.g. bytes or base64 video_inputs: bool """Whether [video inputs](https://docs.langchain.com/oss/python/langchain/models#multimodal) are supported.""" - # TODO: add more detail about formats? e.g. bytes or base64 + + input_mime_types: dict[str, list[str]] + """MIME types accepted as input, grouped by modality. + + Keys mirror the modality names used by [models.dev](https://models.dev) — for + example, `'image'`, `'audio'`, `'pdf'`, `'video'`. Values are lists of + [IANA media types](https://www.iana.org/assignments/media-types/) such as + `'image/png'` or `'audio/mpeg'`. + + This field is *informational*: an entry means the provider is known to accept + that MIME type, but an empty or missing list does not necessarily mean a + type is rejected. Consumers that need hard validation should consult the + upstream provider's documentation. + + Example: + ```python + { + "image": ["image/png", "image/jpeg", "image/gif", "image/webp"], + "pdf": ["application/pdf"], + } + ``` + """ image_tool_message: bool """Whether images can be included in tool messages.""" @@ -100,6 +118,25 @@ class ModelProfile(TypedDict, total=False): """Whether [video outputs](https://docs.langchain.com/oss/python/langchain/models#multimodal) are supported.""" + output_mime_types: dict[str, list[str]] + """MIME types produced as output, grouped by modality. + + Keys mirror the modality names used by [models.dev](https://models.dev) — for + example, `'image'`, `'audio'`, `'video'`. Values are lists of + [IANA media types](https://www.iana.org/assignments/media-types/) such as + `'image/png'` or `'audio/mpeg'`. + + This field is *informational*: an entry means the provider is known to + return that MIME type, but absence does not imply a guarantee that no other + type can be returned. Consumers that need hard validation should consult the + upstream provider's documentation. + + Example: + ```python + {"image": ["image/png"], "audio": ["audio/mpeg"]} + ``` + """ + # --- Tool calling --- tool_calling: bool """Whether the model supports [tool calling](https://docs.langchain.com/oss/python/langchain/models#tool-calling)""" diff --git a/libs/model-profiles/langchain_model_profiles/cli.py b/libs/model-profiles/langchain_model_profiles/cli.py index 06246fd23336a..6bc70f4b97c60 100644 --- a/libs/model-profiles/langchain_model_profiles/cli.py +++ b/libs/model-profiles/langchain_model_profiles/cli.py @@ -55,11 +55,30 @@ def _validate_data_dir(data_dir: Path) -> Path: return resolved +def _profile_field_names() -> frozenset[str]: + """Return the set of keys declared on `ModelProfile`, or an empty set.""" + try: + from langchain_core.language_models.model_profile import ModelProfile + except ImportError: + return frozenset() + + try: + return frozenset(get_type_hints(ModelProfile).keys()) + except (TypeError, NameError): + return frozenset() + + def _load_augmentations( data_dir: Path, ) -> tuple[dict[str, Any], dict[str, dict[str, Any]]]: """Load augmentations from `profile_augmentations.toml`. + Provider-level overrides are top-level keys under `[overrides]`. Model-level + overrides are keyed by model id under `[overrides."model-id"]`. Because TOML + subtables produce `dict` values, we distinguish the two by checking the key + against the declared `ModelProfile` field names. If `ModelProfile` cannot be + imported, we fall back to the legacy heuristic of "dict value ⇒ model id." + Args: data_dir: Directory containing `profile_augmentations.toml`. @@ -90,8 +109,26 @@ def _load_augmentations( provider_aug: dict[str, Any] = {} model_augs: dict[str, dict[str, Any]] = {} + profile_fields = _profile_field_names() + for key, value in overrides.items(): - if isinstance(value, dict): + if profile_fields: + # Schema-driven: known profile field names are provider-level; all + # other keys are treated as model identifiers (whose values must be + # dict overrides). + if key in profile_fields: + provider_aug[key] = value + elif isinstance(value, dict): + model_augs[key] = value + else: + msg = ( + f"Augmentation key '{key}' is not a declared ModelProfile " + f"field and its value is not a table of overrides." + ) + print(f"❌ {msg}", file=sys.stderr) + sys.exit(1) + # Legacy fallback when ModelProfile is unavailable. + elif isinstance(value, dict): model_augs[key] = value else: provider_aug[key] = value diff --git a/libs/model-profiles/tests/unit_tests/test_cli.py b/libs/model-profiles/tests/unit_tests/test_cli.py index aae60481717eb..5bf3a7f4f8aeb 100644 --- a/libs/model-profiles/tests/unit_tests/test_cli.py +++ b/libs/model-profiles/tests/unit_tests/test_cli.py @@ -472,3 +472,133 @@ def test_survives_get_type_hints_failure(self) -> None: side_effect=TypeError("broken"), ): _warn_undeclared_profile_keys(profiles) + + +def test_refresh_merges_provider_level_mime_types( + tmp_path: Path, mock_models_dev_response: dict +) -> None: + """Provider-level `input_mime_types` cascades to every model.""" + data_dir = tmp_path / "data" + data_dir.mkdir() + + aug_file = data_dir / "profile_augmentations.toml" + aug_file.write_text( + """ +provider = "anthropic" + +[overrides] +image_url_inputs = true + +[overrides.input_mime_types] +image = ["image/png", "image/jpeg"] +pdf = ["application/pdf"] +""" + ) + + mock_response = Mock() + mock_response.json.return_value = mock_models_dev_response + mock_response.raise_for_status = Mock() + + with ( + patch("langchain_model_profiles.cli.httpx.get", return_value=mock_response), + patch("builtins.input", return_value="y"), + ): + refresh("anthropic", data_dir) + + profiles_file = data_dir / "_profiles.py" + spec = importlib.util.spec_from_file_location("generated_mime", profiles_file) + assert spec + assert spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) # type: ignore[union-attr] + + for model_id in ("claude-3-opus", "claude-3-sonnet"): + profile = module._PROFILES[model_id] # type: ignore[attr-defined] + assert profile["input_mime_types"] == { + "image": ["image/png", "image/jpeg"], + "pdf": ["application/pdf"], + } + + +def test_refresh_model_level_mime_types_override_provider( + tmp_path: Path, mock_models_dev_response: dict +) -> None: + """Model-level MIME-type overrides win over provider-level defaults.""" + data_dir = tmp_path / "data" + data_dir.mkdir() + + aug_file = data_dir / "profile_augmentations.toml" + aug_file.write_text( + """ +provider = "anthropic" + +[overrides.input_mime_types] +image = ["image/png"] + +[overrides."claude-3-opus".input_mime_types] +image = ["image/png", "image/jpeg", "image/webp"] + +[overrides."claude-3-opus".output_mime_types] +image = ["image/png"] +""" + ) + + mock_response = Mock() + mock_response.json.return_value = mock_models_dev_response + mock_response.raise_for_status = Mock() + + with ( + patch("langchain_model_profiles.cli.httpx.get", return_value=mock_response), + patch("builtins.input", return_value="y"), + ): + refresh("anthropic", data_dir) + + profiles_file = data_dir / "_profiles.py" + spec = importlib.util.spec_from_file_location( + "generated_mime_override", profiles_file + ) + assert spec + assert spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) # type: ignore[union-attr] + + opus = module._PROFILES["claude-3-opus"] # type: ignore[attr-defined] + sonnet = module._PROFILES["claude-3-sonnet"] # type: ignore[attr-defined] + + assert opus["input_mime_types"] == { + "image": ["image/png", "image/jpeg", "image/webp"] + } + assert opus["output_mime_types"] == {"image": ["image/png"]} + assert sonnet["input_mime_types"] == {"image": ["image/png"]} + assert "output_mime_types" not in sonnet + + +def test_refresh_rejects_unknown_scalar_top_level_key( + tmp_path: Path, mock_models_dev_response: dict +) -> None: + """Unknown scalar keys at the top level are no longer silently accepted.""" + data_dir = tmp_path / "data" + data_dir.mkdir() + + aug_file = data_dir / "profile_augmentations.toml" + aug_file.write_text( + """ +provider = "anthropic" + +[overrides] +not_a_real_field = "oops" +""" + ) + + mock_response = Mock() + mock_response.json.return_value = mock_models_dev_response + mock_response.raise_for_status = Mock() + + with ( + patch("langchain_model_profiles.cli.httpx.get", return_value=mock_response), + patch("builtins.input", return_value="y"), + pytest.raises(SystemExit) as exc_info, + ): + refresh("anthropic", data_dir) + + assert exc_info.value.code == 1 diff --git a/libs/partners/anthropic/langchain_anthropic/data/_profiles.py b/libs/partners/anthropic/langchain_anthropic/data/_profiles.py index b7b7857dbfcea..a9853d4f320ad 100644 --- a/libs/partners/anthropic/langchain_anthropic/data/_profiles.py +++ b/libs/partners/anthropic/langchain_anthropic/data/_profiles.py @@ -40,6 +40,17 @@ "pdf_tool_message": True, "image_tool_message": True, "structured_output": False, + "input_mime_types": { + "image": [ + "image/jpeg", + "image/png", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "claude-3-5-haiku-latest": { "name": "Claude Haiku 3.5 (latest)", @@ -65,6 +76,17 @@ "pdf_tool_message": True, "image_tool_message": True, "structured_output": False, + "input_mime_types": { + "image": [ + "image/jpeg", + "image/png", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "claude-3-5-sonnet-20240620": { "name": "Claude Sonnet 3.5", @@ -90,6 +112,17 @@ "pdf_tool_message": True, "image_tool_message": True, "structured_output": False, + "input_mime_types": { + "image": [ + "image/jpeg", + "image/png", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "claude-3-5-sonnet-20241022": { "name": "Claude Sonnet 3.5 v2", @@ -115,6 +148,17 @@ "pdf_tool_message": True, "image_tool_message": True, "structured_output": False, + "input_mime_types": { + "image": [ + "image/jpeg", + "image/png", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "claude-3-7-sonnet-20250219": { "name": "Claude Sonnet 3.7", @@ -140,6 +184,17 @@ "pdf_tool_message": True, "image_tool_message": True, "structured_output": False, + "input_mime_types": { + "image": [ + "image/jpeg", + "image/png", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "claude-3-haiku-20240307": { "name": "Claude Haiku 3", @@ -165,6 +220,17 @@ "pdf_tool_message": True, "image_tool_message": True, "structured_output": False, + "input_mime_types": { + "image": [ + "image/jpeg", + "image/png", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "claude-3-opus-20240229": { "name": "Claude Opus 3", @@ -190,6 +256,17 @@ "pdf_tool_message": True, "image_tool_message": True, "structured_output": False, + "input_mime_types": { + "image": [ + "image/jpeg", + "image/png", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "claude-3-sonnet-20240229": { "name": "Claude Sonnet 3", @@ -215,6 +292,17 @@ "pdf_tool_message": True, "image_tool_message": True, "structured_output": False, + "input_mime_types": { + "image": [ + "image/jpeg", + "image/png", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "claude-haiku-4-5": { "name": "Claude Haiku 4.5 (latest)", @@ -240,6 +328,17 @@ "pdf_tool_message": True, "image_tool_message": True, "structured_output": True, + "input_mime_types": { + "image": [ + "image/jpeg", + "image/png", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "claude-haiku-4-5-20251001": { "name": "Claude Haiku 4.5", @@ -265,6 +364,17 @@ "pdf_tool_message": True, "image_tool_message": True, "structured_output": False, + "input_mime_types": { + "image": [ + "image/jpeg", + "image/png", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "claude-opus-4-0": { "name": "Claude Opus 4 (latest)", @@ -290,6 +400,17 @@ "pdf_tool_message": True, "image_tool_message": True, "structured_output": False, + "input_mime_types": { + "image": [ + "image/jpeg", + "image/png", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "claude-opus-4-1": { "name": "Claude Opus 4.1 (latest)", @@ -315,6 +436,17 @@ "pdf_tool_message": True, "image_tool_message": True, "structured_output": True, + "input_mime_types": { + "image": [ + "image/jpeg", + "image/png", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "claude-opus-4-1-20250805": { "name": "Claude Opus 4.1", @@ -340,6 +472,17 @@ "pdf_tool_message": True, "image_tool_message": True, "structured_output": False, + "input_mime_types": { + "image": [ + "image/jpeg", + "image/png", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "claude-opus-4-20250514": { "name": "Claude Opus 4", @@ -365,6 +508,17 @@ "pdf_tool_message": True, "image_tool_message": True, "structured_output": False, + "input_mime_types": { + "image": [ + "image/jpeg", + "image/png", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "claude-opus-4-5": { "name": "Claude Opus 4.5 (latest)", @@ -390,6 +544,17 @@ "pdf_tool_message": True, "image_tool_message": True, "structured_output": True, + "input_mime_types": { + "image": [ + "image/jpeg", + "image/png", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "claude-opus-4-5-20251101": { "name": "Claude Opus 4.5", @@ -415,6 +580,17 @@ "pdf_tool_message": True, "image_tool_message": True, "structured_output": False, + "input_mime_types": { + "image": [ + "image/jpeg", + "image/png", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "claude-opus-4-6": { "name": "Claude Opus 4.6", @@ -440,6 +616,17 @@ "pdf_tool_message": True, "image_tool_message": True, "structured_output": True, + "input_mime_types": { + "image": [ + "image/jpeg", + "image/png", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "claude-opus-4-7": { "name": "Claude Opus 4.7", @@ -465,6 +652,17 @@ "pdf_tool_message": True, "image_tool_message": True, "structured_output": True, + "input_mime_types": { + "image": [ + "image/jpeg", + "image/png", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "claude-sonnet-4-0": { "name": "Claude Sonnet 4 (latest)", @@ -490,6 +688,17 @@ "pdf_tool_message": True, "image_tool_message": True, "structured_output": False, + "input_mime_types": { + "image": [ + "image/jpeg", + "image/png", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "claude-sonnet-4-20250514": { "name": "Claude Sonnet 4", @@ -515,6 +724,17 @@ "pdf_tool_message": True, "image_tool_message": True, "structured_output": False, + "input_mime_types": { + "image": [ + "image/jpeg", + "image/png", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "claude-sonnet-4-5": { "name": "Claude Sonnet 4.5 (latest)", @@ -540,6 +760,17 @@ "pdf_tool_message": True, "image_tool_message": True, "structured_output": True, + "input_mime_types": { + "image": [ + "image/jpeg", + "image/png", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "claude-sonnet-4-5-20250929": { "name": "Claude Sonnet 4.5", @@ -565,6 +796,17 @@ "pdf_tool_message": True, "image_tool_message": True, "structured_output": False, + "input_mime_types": { + "image": [ + "image/jpeg", + "image/png", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "claude-sonnet-4-6": { "name": "Claude Sonnet 4.6", @@ -590,5 +832,16 @@ "pdf_tool_message": True, "image_tool_message": True, "structured_output": True, + "input_mime_types": { + "image": [ + "image/jpeg", + "image/png", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, } diff --git a/libs/partners/anthropic/langchain_anthropic/data/profile_augmentations.toml b/libs/partners/anthropic/langchain_anthropic/data/profile_augmentations.toml index 38480f92b69eb..9633a10ede3c5 100644 --- a/libs/partners/anthropic/langchain_anthropic/data/profile_augmentations.toml +++ b/libs/partners/anthropic/langchain_anthropic/data/profile_augmentations.toml @@ -7,6 +7,10 @@ pdf_tool_message = true image_tool_message = true structured_output = false +[overrides.input_mime_types] +image = ["image/jpeg", "image/png", "image/gif", "image/webp"] +pdf = ["application/pdf"] + [overrides."claude-haiku-4-5"] structured_output = true diff --git a/libs/partners/openai/langchain_openai/data/_profiles.py b/libs/partners/openai/langchain_openai/data/_profiles.py index 397a362d6746a..c9cca5dad45b5 100644 --- a/libs/partners/openai/langchain_openai/data/_profiles.py +++ b/libs/partners/openai/langchain_openai/data/_profiles.py @@ -40,6 +40,17 @@ "pdf_tool_message": True, "image_tool_message": True, "tool_choice": True, + "input_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "gpt-3.5-turbo": { "name": "GPT-3.5-turbo", @@ -66,6 +77,17 @@ "pdf_tool_message": False, "image_tool_message": False, "tool_choice": True, + "input_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "gpt-4": { "name": "GPT-4", @@ -92,6 +114,17 @@ "pdf_tool_message": True, "image_tool_message": True, "tool_choice": True, + "input_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "gpt-4-turbo": { "name": "GPT-4 Turbo", @@ -118,6 +151,17 @@ "pdf_tool_message": True, "image_tool_message": True, "tool_choice": True, + "input_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "gpt-4.1": { "name": "GPT-4.1", @@ -144,6 +188,17 @@ "pdf_tool_message": True, "image_tool_message": True, "tool_choice": True, + "input_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "gpt-4.1-mini": { "name": "GPT-4.1 mini", @@ -170,6 +225,17 @@ "pdf_tool_message": True, "image_tool_message": True, "tool_choice": True, + "input_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "gpt-4.1-nano": { "name": "GPT-4.1 nano", @@ -196,6 +262,17 @@ "pdf_tool_message": True, "image_tool_message": True, "tool_choice": True, + "input_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "gpt-4o": { "name": "GPT-4o", @@ -222,6 +299,17 @@ "pdf_tool_message": True, "image_tool_message": True, "tool_choice": True, + "input_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "gpt-4o-2024-05-13": { "name": "GPT-4o (2024-05-13)", @@ -248,6 +336,17 @@ "pdf_tool_message": True, "image_tool_message": True, "tool_choice": True, + "input_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "gpt-4o-2024-08-06": { "name": "GPT-4o (2024-08-06)", @@ -274,6 +373,17 @@ "pdf_tool_message": True, "image_tool_message": True, "tool_choice": True, + "input_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "gpt-4o-2024-11-20": { "name": "GPT-4o (2024-11-20)", @@ -300,6 +410,17 @@ "pdf_tool_message": True, "image_tool_message": True, "tool_choice": True, + "input_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "gpt-4o-mini": { "name": "GPT-4o mini", @@ -326,6 +447,17 @@ "pdf_tool_message": True, "image_tool_message": True, "tool_choice": True, + "input_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "gpt-5": { "name": "GPT-5", @@ -352,6 +484,17 @@ "pdf_tool_message": True, "image_tool_message": True, "tool_choice": True, + "input_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "gpt-5-chat-latest": { "name": "GPT-5 Chat (latest)", @@ -378,6 +521,17 @@ "pdf_tool_message": True, "image_tool_message": True, "tool_choice": True, + "input_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "gpt-5-codex": { "name": "GPT-5-Codex", @@ -404,6 +558,17 @@ "pdf_tool_message": True, "image_tool_message": True, "tool_choice": True, + "input_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "gpt-5-mini": { "name": "GPT-5 Mini", @@ -430,6 +595,17 @@ "pdf_tool_message": True, "image_tool_message": True, "tool_choice": True, + "input_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "gpt-5-nano": { "name": "GPT-5 Nano", @@ -456,6 +632,17 @@ "pdf_tool_message": True, "image_tool_message": True, "tool_choice": True, + "input_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "gpt-5-pro": { "name": "GPT-5 Pro", @@ -482,6 +669,17 @@ "pdf_tool_message": True, "image_tool_message": True, "tool_choice": True, + "input_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "gpt-5.1": { "name": "GPT-5.1", @@ -508,6 +706,17 @@ "pdf_tool_message": True, "image_tool_message": True, "tool_choice": True, + "input_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "gpt-5.1-chat-latest": { "name": "GPT-5.1 Chat", @@ -534,6 +743,17 @@ "pdf_tool_message": True, "image_tool_message": True, "tool_choice": True, + "input_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "gpt-5.1-codex": { "name": "GPT-5.1 Codex", @@ -560,6 +780,17 @@ "pdf_tool_message": True, "image_tool_message": True, "tool_choice": True, + "input_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "gpt-5.1-codex-max": { "name": "GPT-5.1 Codex Max", @@ -586,6 +817,17 @@ "pdf_tool_message": True, "image_tool_message": True, "tool_choice": True, + "input_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "gpt-5.1-codex-mini": { "name": "GPT-5.1 Codex mini", @@ -612,6 +854,17 @@ "pdf_tool_message": True, "image_tool_message": True, "tool_choice": True, + "input_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "gpt-5.2": { "name": "GPT-5.2", @@ -638,6 +891,17 @@ "pdf_tool_message": True, "image_tool_message": True, "tool_choice": True, + "input_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "gpt-5.2-chat-latest": { "name": "GPT-5.2 Chat", @@ -664,6 +928,17 @@ "pdf_tool_message": True, "image_tool_message": True, "tool_choice": True, + "input_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "gpt-5.2-codex": { "name": "GPT-5.2 Codex", @@ -690,6 +965,17 @@ "pdf_tool_message": True, "image_tool_message": True, "tool_choice": True, + "input_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "gpt-5.2-pro": { "name": "GPT-5.2 Pro", @@ -716,6 +1002,17 @@ "pdf_tool_message": True, "image_tool_message": True, "tool_choice": True, + "input_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "gpt-5.3-chat-latest": { "name": "GPT-5.3 Chat (latest)", @@ -742,6 +1039,17 @@ "pdf_tool_message": True, "image_tool_message": True, "tool_choice": True, + "input_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "gpt-5.3-codex": { "name": "GPT-5.3 Codex", @@ -768,6 +1076,17 @@ "pdf_tool_message": True, "image_tool_message": True, "tool_choice": True, + "input_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "gpt-5.3-codex-spark": { "name": "GPT-5.3 Codex Spark", @@ -794,6 +1113,17 @@ "pdf_tool_message": True, "image_tool_message": True, "tool_choice": True, + "input_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "gpt-5.4": { "name": "GPT-5.4", @@ -820,6 +1150,17 @@ "pdf_tool_message": True, "image_tool_message": True, "tool_choice": True, + "input_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "gpt-5.4-mini": { "name": "GPT-5.4 mini", @@ -846,6 +1187,17 @@ "pdf_tool_message": True, "image_tool_message": True, "tool_choice": True, + "input_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "gpt-5.4-nano": { "name": "GPT-5.4 nano", @@ -872,6 +1224,17 @@ "pdf_tool_message": True, "image_tool_message": True, "tool_choice": True, + "input_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "gpt-5.4-pro": { "name": "GPT-5.4 Pro", @@ -898,6 +1261,17 @@ "pdf_tool_message": True, "image_tool_message": True, "tool_choice": True, + "input_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "gpt-5.5": { "name": "GPT-5.5", @@ -924,6 +1298,17 @@ "pdf_tool_message": True, "image_tool_message": True, "tool_choice": True, + "input_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "gpt-5.5-pro": { "name": "GPT-5.5 Pro", @@ -950,6 +1335,17 @@ "pdf_tool_message": True, "image_tool_message": True, "tool_choice": True, + "input_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "gpt-image-1": { "name": "gpt-image-1", @@ -975,6 +1371,24 @@ "pdf_tool_message": True, "image_tool_message": True, "tool_choice": True, + "input_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, + "output_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/webp", + ], + }, }, "gpt-image-1-mini": { "name": "gpt-image-1-mini", @@ -1000,6 +1414,24 @@ "pdf_tool_message": True, "image_tool_message": True, "tool_choice": True, + "input_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, + "output_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/webp", + ], + }, }, "gpt-image-1.5": { "name": "gpt-image-1.5", @@ -1025,6 +1457,24 @@ "pdf_tool_message": True, "image_tool_message": True, "tool_choice": True, + "input_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, + "output_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/webp", + ], + }, }, "o1": { "name": "o1", @@ -1051,6 +1501,17 @@ "pdf_tool_message": True, "image_tool_message": True, "tool_choice": True, + "input_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "o1-mini": { "name": "o1-mini", @@ -1077,6 +1538,17 @@ "pdf_tool_message": True, "image_tool_message": True, "tool_choice": True, + "input_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "o1-preview": { "name": "o1-preview", @@ -1102,6 +1574,17 @@ "pdf_tool_message": True, "image_tool_message": True, "tool_choice": True, + "input_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "o1-pro": { "name": "o1-pro", @@ -1128,6 +1611,17 @@ "pdf_tool_message": True, "image_tool_message": True, "tool_choice": True, + "input_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "o3": { "name": "o3", @@ -1154,6 +1648,17 @@ "pdf_tool_message": True, "image_tool_message": True, "tool_choice": True, + "input_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "o3-deep-research": { "name": "o3-deep-research", @@ -1179,6 +1684,17 @@ "pdf_tool_message": True, "image_tool_message": True, "tool_choice": True, + "input_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "o3-mini": { "name": "o3-mini", @@ -1205,6 +1721,17 @@ "pdf_tool_message": True, "image_tool_message": True, "tool_choice": True, + "input_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "o3-pro": { "name": "o3-pro", @@ -1231,6 +1758,17 @@ "pdf_tool_message": True, "image_tool_message": True, "tool_choice": True, + "input_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "o4-mini": { "name": "o4-mini", @@ -1257,6 +1795,17 @@ "pdf_tool_message": True, "image_tool_message": True, "tool_choice": True, + "input_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "o4-mini-deep-research": { "name": "o4-mini-deep-research", @@ -1282,6 +1831,17 @@ "pdf_tool_message": True, "image_tool_message": True, "tool_choice": True, + "input_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "text-embedding-3-large": { "name": "text-embedding-3-large", @@ -1307,6 +1867,17 @@ "pdf_tool_message": True, "image_tool_message": True, "tool_choice": True, + "input_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "text-embedding-3-small": { "name": "text-embedding-3-small", @@ -1332,6 +1903,17 @@ "pdf_tool_message": True, "image_tool_message": True, "tool_choice": True, + "input_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, "text-embedding-ada-002": { "name": "text-embedding-ada-002", @@ -1357,5 +1939,16 @@ "pdf_tool_message": True, "image_tool_message": True, "tool_choice": True, + "input_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ], + "pdf": [ + "application/pdf", + ], + }, }, } diff --git a/libs/partners/openai/langchain_openai/data/profile_augmentations.toml b/libs/partners/openai/langchain_openai/data/profile_augmentations.toml index 6fb3ee4975fc4..a5c2ba1f900d9 100644 --- a/libs/partners/openai/langchain_openai/data/profile_augmentations.toml +++ b/libs/partners/openai/langchain_openai/data/profile_augmentations.toml @@ -7,12 +7,25 @@ pdf_tool_message = true image_tool_message = true tool_choice = true +[overrides.input_mime_types] +image = ["image/png", "image/jpeg", "image/gif", "image/webp"] +pdf = ["application/pdf"] + [overrides."gpt-3.5-turbo"] image_url_inputs = false pdf_inputs = false pdf_tool_message = false image_tool_message = false +[overrides."gpt-image-1".output_mime_types] +image = ["image/png", "image/jpeg", "image/webp"] + +[overrides."gpt-image-1-mini".output_mime_types] +image = ["image/png", "image/jpeg", "image/webp"] + +[overrides."gpt-image-1.5".output_mime_types] +image = ["image/png", "image/jpeg", "image/webp"] + [overrides."gpt-5.1-codex"] max_input_tokens = 272000 diff --git a/libs/partners/perplexity/langchain_perplexity/data/_profiles.py b/libs/partners/perplexity/langchain_perplexity/data/_profiles.py index 35c4ce828258d..1d14f8f5aec56 100644 --- a/libs/partners/perplexity/langchain_perplexity/data/_profiles.py +++ b/libs/partners/perplexity/langchain_perplexity/data/_profiles.py @@ -55,6 +55,14 @@ "tool_calling": False, "attachment": False, "temperature": False, + "input_mime_types": { + "image": [ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + ], + }, }, "sonar-pro": { "name": "Sonar Pro", diff --git a/libs/partners/perplexity/langchain_perplexity/data/profile_augmentations.toml b/libs/partners/perplexity/langchain_perplexity/data/profile_augmentations.toml index 5af031fb13719..9c35d58c1374e 100644 --- a/libs/partners/perplexity/langchain_perplexity/data/profile_augmentations.toml +++ b/libs/partners/perplexity/langchain_perplexity/data/profile_augmentations.toml @@ -11,3 +11,6 @@ audio_outputs = false video_outputs = false reasoning_output = true tool_calling = false + +[overrides."sonar-deep-research".input_mime_types] +image = ["image/png", "image/jpeg", "image/gif", "image/webp"]