feat(model-profiles): add input/output MIME type fields to ModelProfile

libs/core/langchain_core/language_models/model_profile.py

@@ -50,7 +50,6 @@ class ModelProfile(TypedDict, total=False):
 
     image_inputs: bool
     """Whether image inputs are supported."""
-    # TODO: add more detail about formats?
 
     image_url_inputs: bool
     """Whether [image URL inputs](https://docs.langchain.com/oss/python/langchain/models#multimodal)
@@ -59,17 +58,36 @@ class ModelProfile(TypedDict, total=False):
     pdf_inputs: bool
     """Whether [PDF inputs](https://docs.langchain.com/oss/python/langchain/models#multimodal)
     are supported."""
-    # TODO: add more detail about formats? e.g. bytes or base64
 
     audio_inputs: bool
     """Whether [audio inputs](https://docs.langchain.com/oss/python/langchain/models#multimodal)
     are supported."""
-    # TODO: add more detail about formats? e.g. bytes or base64
 
     video_inputs: bool
     """Whether [video inputs](https://docs.langchain.com/oss/python/langchain/models#multimodal)
     are supported."""
-    # TODO: add more detail about formats? e.g. bytes or base64
+
+    input_mime_types: dict[str, list[str]]
+    """MIME types accepted as input, grouped by modality.
+
+    Keys mirror the modality names used by [models.dev](https://models.dev) — for
+    example, `'image'`, `'audio'`, `'pdf'`, `'video'`. Values are lists of
+    [IANA media types](https://www.iana.org/assignments/media-types/) such as
+    `'image/png'` or `'audio/mpeg'`.
+
+    This field is *informational*: an entry means the provider is known to accept
+    that MIME type, but an empty or missing list does not necessarily mean a
+    type is rejected. Consumers that need hard validation should consult the
+    upstream provider's documentation.
+
+    Example:
+        ```python
+        {
+            "image": ["image/png", "image/jpeg", "image/gif", "image/webp"],
+            "pdf": ["application/pdf"],
+        }
+        ```
+    """
 
     image_tool_message: bool
     """Whether images can be included in tool messages."""
@@ -100,6 +118,25 @@ class ModelProfile(TypedDict, total=False):
     """Whether [video outputs](https://docs.langchain.com/oss/python/langchain/models#multimodal)
     are supported."""
 
+    output_mime_types: dict[str, list[str]]
+    """MIME types produced as output, grouped by modality.
+
+    Keys mirror the modality names used by [models.dev](https://models.dev) — for
+    example, `'image'`, `'audio'`, `'video'`. Values are lists of
+    [IANA media types](https://www.iana.org/assignments/media-types/) such as
+    `'image/png'` or `'audio/mpeg'`.
+
+    This field is *informational*: an entry means the provider is known to
+    return that MIME type, but absence does not imply a guarantee that no other
+    type can be returned. Consumers that need hard validation should consult the
+    upstream provider's documentation.
+
+    Example:
+        ```python
+        {"image": ["image/png"], "audio": ["audio/mpeg"]}
+        ```
+    """
+
     # --- Tool calling ---
     tool_calling: bool
     """Whether the model supports [tool calling](https://docs.langchain.com/oss/python/langchain/models#tool-calling)"""

libs/model-profiles/langchain_model_profiles/cli.py

@@ -55,11 +55,30 @@ def _validate_data_dir(data_dir: Path) -> Path:
     return resolved
 
 
+def _profile_field_names() -> frozenset[str]:
+    """Return the set of keys declared on `ModelProfile`, or an empty set."""
+    try:
+        from langchain_core.language_models.model_profile import ModelProfile
+    except ImportError:
+        return frozenset()
+
+    try:
+        return frozenset(get_type_hints(ModelProfile).keys())
+    except (TypeError, NameError):
+        return frozenset()
+
+
 def _load_augmentations(
     data_dir: Path,
 ) -> tuple[dict[str, Any], dict[str, dict[str, Any]]]:
     """Load augmentations from `profile_augmentations.toml`.
 
+    Provider-level overrides are top-level keys under `[overrides]`. Model-level
+    overrides are keyed by model id under `[overrides."model-id"]`. Because TOML
+    subtables produce `dict` values, we distinguish the two by checking the key
+    against the declared `ModelProfile` field names. If `ModelProfile` cannot be
+    imported, we fall back to the legacy heuristic of "dict value ⇒ model id."
+
     Args:
         data_dir: Directory containing `profile_augmentations.toml`.
 
@@ -90,8 +109,26 @@ def _load_augmentations(
     provider_aug: dict[str, Any] = {}
     model_augs: dict[str, dict[str, Any]] = {}
 
+    profile_fields = _profile_field_names()
+
     for key, value in overrides.items():
-        if isinstance(value, dict):
+        if profile_fields:
+            # Schema-driven: known profile field names are provider-level; all
+            # other keys are treated as model identifiers (whose values must be
+            # dict overrides).
+            if key in profile_fields:
+                provider_aug[key] = value
+            elif isinstance(value, dict):
+                model_augs[key] = value
+            else:
+                msg = (
+                    f"Augmentation key '{key}' is not a declared ModelProfile "
+                    f"field and its value is not a table of overrides."
+                )
+                print(f"❌ {msg}", file=sys.stderr)
+                sys.exit(1)
+        # Legacy fallback when ModelProfile is unavailable.
+        elif isinstance(value, dict):
             model_augs[key] = value
         else:
             provider_aug[key] = value

libs/model-profiles/tests/unit_tests/test_cli.py

@@ -472,3 +472,133 @@ def test_survives_get_type_hints_failure(self) -> None:
             side_effect=TypeError("broken"),
         ):
             _warn_undeclared_profile_keys(profiles)
+
+
+def test_refresh_merges_provider_level_mime_types(
+    tmp_path: Path, mock_models_dev_response: dict
+) -> None:
+    """Provider-level `input_mime_types` cascades to every model."""
+    data_dir = tmp_path / "data"
+    data_dir.mkdir()
+
+    aug_file = data_dir / "profile_augmentations.toml"
+    aug_file.write_text(
+        """
+provider = "anthropic"
+
+[overrides]
+image_url_inputs = true
+
+[overrides.input_mime_types]
+image = ["image/png", "image/jpeg"]
+pdf = ["application/pdf"]
+"""
+    )
+
+    mock_response = Mock()
+    mock_response.json.return_value = mock_models_dev_response
+    mock_response.raise_for_status = Mock()
+
+    with (
+        patch("langchain_model_profiles.cli.httpx.get", return_value=mock_response),
+        patch("builtins.input", return_value="y"),
+    ):
+        refresh("anthropic", data_dir)
+
+    profiles_file = data_dir / "_profiles.py"
+    spec = importlib.util.spec_from_file_location("generated_mime", profiles_file)
+    assert spec
+    assert spec.loader
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)  # type: ignore[union-attr]
+
+    for model_id in ("claude-3-opus", "claude-3-sonnet"):
+        profile = module._PROFILES[model_id]  # type: ignore[attr-defined]
+        assert profile["input_mime_types"] == {
+            "image": ["image/png", "image/jpeg"],
+            "pdf": ["application/pdf"],
+        }
+
+
+def test_refresh_model_level_mime_types_override_provider(
+    tmp_path: Path, mock_models_dev_response: dict
+) -> None:
+    """Model-level MIME-type overrides win over provider-level defaults."""
+    data_dir = tmp_path / "data"
+    data_dir.mkdir()
+
+    aug_file = data_dir / "profile_augmentations.toml"
+    aug_file.write_text(
+        """
+provider = "anthropic"
+
+[overrides.input_mime_types]
+image = ["image/png"]
+
+[overrides."claude-3-opus".input_mime_types]
+image = ["image/png", "image/jpeg", "image/webp"]
+
+[overrides."claude-3-opus".output_mime_types]
+image = ["image/png"]
+"""
+    )
+
+    mock_response = Mock()
+    mock_response.json.return_value = mock_models_dev_response
+    mock_response.raise_for_status = Mock()
+
+    with (
+        patch("langchain_model_profiles.cli.httpx.get", return_value=mock_response),
+        patch("builtins.input", return_value="y"),
+    ):
+        refresh("anthropic", data_dir)
+
+    profiles_file = data_dir / "_profiles.py"
+    spec = importlib.util.spec_from_file_location(
+        "generated_mime_override", profiles_file
+    )
+    assert spec
+    assert spec.loader
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)  # type: ignore[union-attr]
+
+    opus = module._PROFILES["claude-3-opus"]  # type: ignore[attr-defined]
+    sonnet = module._PROFILES["claude-3-sonnet"]  # type: ignore[attr-defined]
+
+    assert opus["input_mime_types"] == {
+        "image": ["image/png", "image/jpeg", "image/webp"]
+    }
+    assert opus["output_mime_types"] == {"image": ["image/png"]}
+    assert sonnet["input_mime_types"] == {"image": ["image/png"]}
+    assert "output_mime_types" not in sonnet
+
+
+def test_refresh_rejects_unknown_scalar_top_level_key(
+    tmp_path: Path, mock_models_dev_response: dict
+) -> None:
+    """Unknown scalar keys at the top level are no longer silently accepted."""
+    data_dir = tmp_path / "data"
+    data_dir.mkdir()
+
+    aug_file = data_dir / "profile_augmentations.toml"
+    aug_file.write_text(
+        """
+provider = "anthropic"
+
+[overrides]
+not_a_real_field = "oops"
+"""
+    )
+
+    mock_response = Mock()
+    mock_response.json.return_value = mock_models_dev_response
+    mock_response.raise_for_status = Mock()
+
+    with (
+        patch("langchain_model_profiles.cli.httpx.get", return_value=mock_response),
+        patch("builtins.input", return_value="y"),
+        pytest.raises(SystemExit) as exc_info,
+    ):
+        refresh("anthropic", data_dir)
+
+    assert exc_info.value.code == 1