From c6f0c12ca733419abebee5d6ab04925ac680aa1e Mon Sep 17 00:00:00 2001 From: Richard Lundeen Date: Tue, 1 Sep 2026 16:43:02 -0700 Subject: [PATCH] FEAT: Support converter configurations in REST API Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: dad4cb22-3dd3-41d5-b874-2df319502396 --- pyrit/backend/models/__init__.py | 2 + pyrit/backend/models/attacks.py | 75 ++++++- pyrit/backend/routes/attacks.py | 8 +- pyrit/backend/services/attack_service.py | 187 +++++++++++++--- tests/unit/backend/test_api_routes.py | 93 +++++++- tests/unit/backend/test_attack_service.py | 246 +++++++++++++++++++--- 6 files changed, 548 insertions(+), 63 deletions(-) diff --git a/pyrit/backend/models/__init__.py b/pyrit/backend/models/__init__.py index e13647915d..6307bb1733 100644 --- a/pyrit/backend/models/__init__.py +++ b/pyrit/backend/models/__init__.py @@ -23,6 +23,7 @@ AttackSummary, ConversationMessagesResponse, ConversationSummary, + ConverterConfigurationRequest, ConverterOptionsResponse, CreateAttackRequest, CreateAttackResponse, @@ -74,6 +75,7 @@ "UpdateMainConversationResponse": "pyrit.backend.models.attacks", "ConversationMessagesResponse": "pyrit.backend.models.attacks", "ConversationSummary": "pyrit.backend.models.attacks", + "ConverterConfigurationRequest": "pyrit.backend.models.attacks", "ConverterOptionsResponse": "pyrit.backend.models.attacks", "CreateAttackRequest": "pyrit.backend.models.attacks", "CreateAttackResponse": "pyrit.backend.models.attacks", diff --git a/pyrit/backend/models/attacks.py b/pyrit/backend/models/attacks.py index ab0ded3579..d3a1463ef7 100644 --- a/pyrit/backend/models/attacks.py +++ b/pyrit/backend/models/attacks.py @@ -10,9 +10,9 @@ import uuid from datetime import datetime, timezone -from typing import Any, Literal, cast +from typing import Annotated, Any, Literal, cast -from pydantic import BaseModel, Field, computed_field, field_serializer +from pydantic import BaseModel, Field, computed_field, field_serializer, model_validator from pyrit.backend.models._media import build_filename, infer_mime_type from pyrit.backend.models.common import PaginationInfo @@ -22,6 +22,7 @@ ConversationReference, Message, MessagePiece, + PromptDataType, Score, ) @@ -472,6 +473,26 @@ class UpdateMainConversationResponse(BaseModel): # ============================================================================ +class ConverterConfigurationRequest(BaseModel): + """Registry-backed converter configuration for one ordered pipeline.""" + + converter_ids: list[str] = Field( + ..., + min_length=1, + description="Converter instance IDs to apply in order.", + ) + indexes_to_apply: list[Annotated[int, Field(ge=0)]] | None = Field( + None, + min_length=1, + description="Zero-based message piece indexes to which this pipeline applies. Defaults to all indexes.", + ) + prompt_data_types_to_apply: list[PromptDataType] | None = Field( + None, + min_length=1, + description="Prompt data types to which this pipeline applies. Defaults to all data types.", + ) + + class AddMessageRequest(BaseModel): """ Request to add a message to an attack. @@ -492,7 +513,18 @@ class AddMessageRequest(BaseModel): description="Target registry name. Required when send=True so the backend knows which target to use.", ) converter_ids: list[str] | None = Field( - None, description="Converter instance IDs to apply (overrides attack-level)" + None, + description="Deprecated global request converter pipeline. Use request_converter_configurations instead.", + ) + request_converter_configurations: list[ConverterConfigurationRequest] | None = Field( + None, + min_length=1, + description="Ordered registry-backed converter pipelines to apply to the request.", + ) + response_converter_configurations: list[ConverterConfigurationRequest] | None = Field( + None, + min_length=1, + description="Ordered registry-backed converter pipelines to apply to the response.", ) target_conversation_id: str = Field( ..., @@ -505,6 +537,43 @@ class AddMessageRequest(BaseModel): "When present, the operator must match the attack result's operator.", ) + @model_validator(mode="after") + def _validate_converter_configurations(self) -> "AddMessageRequest": + """ + Validate converter configuration combinations and request indexes. + + Returns: + AddMessageRequest: The validated request. + + Raises: + ValueError: If converter fields conflict, cannot run, or contain an out-of-range request index. + """ + if self.converter_ids is not None and self.request_converter_configurations is not None: + raise ValueError("converter_ids and request_converter_configurations cannot both be provided") + + has_converter_configurations = any( + configurations is not None + for configurations in ( + self.converter_ids, + self.request_converter_configurations, + self.response_converter_configurations, + ) + ) + if not self.send and has_converter_configurations: + raise ValueError("Converter configurations require send=True") + + piece_count = len(self.pieces) + for configuration in self.request_converter_configurations or []: + if configuration.indexes_to_apply is None: + continue + invalid_indexes = [index for index in configuration.indexes_to_apply if index >= piece_count] + if invalid_indexes: + raise ValueError( + f"Request converter indexes {invalid_indexes} are out of range for {piece_count} message pieces" + ) + + return self + class AddMessageResponse(BaseModel): """ diff --git a/pyrit/backend/routes/attacks.py b/pyrit/backend/routes/attacks.py index c8c9e2dc09..7a45833d4e 100644 --- a/pyrit/backend/routes/attacks.py +++ b/pyrit/backend/routes/attacks.py @@ -449,10 +449,10 @@ async def add_message( # pyrit-async-suffix-exempt If send=False, just stores the message in memory without sending (useful for system messages, context injection, or replaying assistant responses). - Converters can be specified at three levels (in priority order): - 1. request.converter_ids - per-message converter instances - 2. request.converters - inline converter definitions - 3. attack.converter_ids - attack-level defaults + Request and response converters can be supplied as ordered, registry-backed + configurations. Each configuration can target message piece indexes, prompt + data types, or both. The global ``converter_ids`` request pipeline remains + available for compatibility but is deprecated. Returns: AddMessageResponse: Updated attack with new message(s). diff --git a/pyrit/backend/services/attack_service.py b/pyrit/backend/services/attack_service.py index 09d07d211d..ac94fb3c40 100644 --- a/pyrit/backend/services/attack_service.py +++ b/pyrit/backend/services/attack_service.py @@ -45,6 +45,7 @@ AttackSummary, ConversationMessagesResponse, ConversationSummary, + ConverterConfigurationRequest, CreateAttackRequest, CreateAttackResponse, CreateConversationRequest, @@ -58,6 +59,7 @@ from pyrit.backend.models.common import PaginationInfo from pyrit.backend.services.converter_service import get_converter_service from pyrit.backend.services.target_service import get_target_service +from pyrit.common.deprecation import print_deprecation_message from pyrit.memory import AttackResultKeysetCursor, CentralMemory, data_serializer_factory from pyrit.models import ( AtomicAttackIdentifier, @@ -639,6 +641,14 @@ async def add_message_async(self, *, attack_result_id: str, request: AddMessageR if request.send and not target_registry_name: raise ValueError("target_registry_name is required when send=True") + request_converter_configs = self._resolve_request_converter_configs(request=request) + response_converter_configs = self._resolve_converter_configs( + configurations=request.response_converter_configurations + ) + preconverted_indexes = { + index for index, piece in enumerate(request.pieces) if piece.converted_value is not None + } + # Get existing messages to determine sequence. # NOTE: This read-then-write is not atomic (TOCTOU). Fine for the # current single-user UI, but would need a DB-level sequence @@ -654,6 +664,9 @@ async def add_message_async(self, *, attack_result_id: str, request: AddMessageR target_registry_name=target_registry_name, request=request, sequence=sequence, + request_converter_configurations=request_converter_configs, + response_converter_configurations=response_converter_configs, + preconverted_indexes=preconverted_indexes, ) except Exception: # PromptNormalizer persists a full error piece (response_error + @@ -681,7 +694,12 @@ async def add_message_async(self, *, attack_result_id: str, request: AddMessageR target_identifier=existing_metadata.target_identifier if existing_metadata else None, ) - await self._update_attack_after_message_async(attack_result_id=attack_result_id, ar=ar, request=request) + await self._update_attack_after_message_async( + attack_result_id=attack_result_id, + ar=ar, + request_converter_configurations=request_converter_configs, + response_converter_configurations=response_converter_configs, + ) attack_detail = await self.get_attack_async(attack_result_id=attack_result_id) if attack_detail is None: @@ -748,7 +766,12 @@ def _validate_operator_match(self, *, attack_result: AttackResult, request: AddM ) async def _update_attack_after_message_async( - self, *, attack_result_id: str, ar: AttackResult, request: AddMessageRequest + self, + *, + attack_result_id: str, + ar: AttackResult, + request_converter_configurations: list[ConverterConfiguration], + response_converter_configurations: list[ConverterConfiguration], ) -> None: """ Update attack recency and converter tracking after a message is added. @@ -758,19 +781,28 @@ async def _update_attack_after_message_async( """ update_fields: dict[str, Any] = {"timestamp": datetime.now(timezone.utc)} - if request.converter_ids: - converter_objs = get_converter_service().get_converter_objects_for_ids(converter_ids=request.converter_ids) - new_converter_ids = [ - ConverterIdentifier.from_component_identifier(c.get_identifier()) for c in converter_objs - ] + request_converter_ids = self._get_converter_identifiers(configurations=request_converter_configurations) + response_converter_ids = self._get_converter_identifiers(configurations=response_converter_configurations) + if request_converter_ids or response_converter_ids: aid = ar.get_attack_strategy_identifier() if aid and ar.atomic_attack_identifier: attack_id = AttackIdentifier.from_component_identifier(aid) - existing_hashes = {c.hash for c in attack_id.request_converters} - additions = [c for c in new_converter_ids if c.hash not in existing_hashes] - if additions: - new_attack_id = self._replace_request_converters( - attack_id, request_converters=[*attack_id.request_converters, *additions] + merged_request_converters = self._merge_converter_identifiers( + existing=attack_id.request_converters, + additions=request_converter_ids, + ) + merged_response_converters = self._merge_converter_identifiers( + existing=attack_id.response_converters, + additions=response_converter_ids, + ) + if ( + merged_request_converters != attack_id.request_converters + or merged_response_converters != attack_id.response_converters + ): + new_attack_id = self._replace_converter_pipelines( + attack_id, + request_converters=merged_request_converters, + response_converters=merged_response_converters, ) new_atomic = self._replace_attack_in_atomic( AtomicAttackIdentifier.from_component_identifier(ar.atomic_attack_identifier), @@ -784,11 +816,14 @@ async def _update_attack_after_message_async( ) @staticmethod - def _replace_request_converters( - attack_id: AttackIdentifier, *, request_converters: list[ConverterIdentifier] + def _replace_converter_pipelines( + attack_id: AttackIdentifier, + *, + request_converters: list[ConverterIdentifier], + response_converters: list[ConverterIdentifier], ) -> AttackIdentifier: """ - Return a copy of ``attack_id`` with its request-converter pipeline replaced. + Return a copy of ``attack_id`` with its converter pipelines replaced. Reconstructed through the constructor (not ``model_copy``) so the after-validator re-mirrors the typed converters into ``children`` and @@ -796,7 +831,7 @@ def _replace_request_converters( preserved, so the identifier hashes identically apart from the converters. Returns: - AttackIdentifier: A new identifier with the given request converters. + AttackIdentifier: A new identifier with the given converter pipelines. """ return AttackIdentifier( class_name=attack_id.class_name, @@ -805,8 +840,29 @@ def _replace_request_converters( children=dict(attack_id.children), attributes=dict(attack_id.attributes), request_converters=request_converters, + response_converters=response_converters, ) + @staticmethod + def _merge_converter_identifiers( + *, + existing: list[ConverterIdentifier], + additions: list[ConverterIdentifier], + ) -> list[ConverterIdentifier]: + """ + Append converter identifiers once while preserving their order. + + Returns: + list[ConverterIdentifier]: The merged converter identifiers. + """ + merged = list(existing) + existing_hashes = {converter.hash for converter in existing} + for converter in additions: + if converter.hash not in existing_hashes: + merged.append(converter) + existing_hashes.add(converter.hash) + return merged + @staticmethod def _replace_attack_in_atomic( atomic: AtomicAttackIdentifier, *, attack: AttackIdentifier @@ -1149,6 +1205,9 @@ async def _send_and_store_message_async( target_registry_name: str, request: AddMessageRequest, sequence: int, + request_converter_configurations: list[ConverterConfiguration], + response_converter_configurations: list[ConverterConfiguration], + preconverted_indexes: set[int], ) -> None: """Send message to target via normalizer and store response.""" target_obj = get_target_service().get_target_object(target_registry_name=target_registry_name) @@ -1165,14 +1224,19 @@ async def _send_and_store_message_async( sequence=sequence, ) - converter_configs = self._get_converter_configs(request) + request_converter_configurations = self._exclude_preconverted_piece_indexes( + configurations=request_converter_configurations, + preconverted_indexes=preconverted_indexes, + piece_count=len(request.pieces), + ) normalizer = PromptNormalizer() await normalizer.send_prompt_async( message=pyrit_message, target=target_obj, conversation_id=conversation_id, - request_converter_configurations=converter_configs, + request_converter_configurations=request_converter_configurations, + response_converter_configurations=response_converter_configurations, ) # PromptNormalizer stores both request and response in memory automatically @@ -1237,19 +1301,90 @@ def _resolve_video_remix_metadata(self, request: AddMessageRequest) -> None: vp.prompt_metadata["video_id"] = video_id return - def _get_converter_configs(self, request: AddMessageRequest) -> list[ConverterConfiguration]: + def _resolve_request_converter_configs(self, *, request: AddMessageRequest) -> list[ConverterConfiguration]: + """ + Resolve legacy or structured request converter configurations. + + Returns: + list[ConverterConfiguration]: Resolved request configurations. + """ + if request.converter_ids is not None: + print_deprecation_message( + old_item="AddMessageRequest.converter_ids", + new_item="AddMessageRequest.request_converter_configurations", + removed_in="1.3.0", + ) + converters = get_converter_service().get_converter_objects_for_ids(converter_ids=request.converter_ids) + return ConverterConfiguration.from_converters(converters=converters) + + return self._resolve_converter_configs(configurations=request.request_converter_configurations) + + def _resolve_converter_configs( + self, + *, + configurations: list[ConverterConfigurationRequest] | None, + ) -> list[ConverterConfiguration]: """ - Get converter configurations if needed. + Resolve registry-backed converter configurations. Returns: - List of ConverterConfiguration for the converters. + list[ConverterConfiguration]: Resolved configurations in request order. + """ + converter_service = get_converter_service() + return [ + ConverterConfiguration( + converters=converter_service.get_converter_objects_for_ids(converter_ids=configuration.converter_ids), + indexes_to_apply=configuration.indexes_to_apply, + prompt_data_types_to_apply=configuration.prompt_data_types_to_apply, + ) + for configuration in configurations or [] + ] + + @staticmethod + def _exclude_preconverted_piece_indexes( + *, + configurations: list[ConverterConfiguration], + preconverted_indexes: set[int], + piece_count: int, + ) -> list[ConverterConfiguration]: """ - has_preconverted = any(p.converted_value is not None for p in request.pieces) - if has_preconverted or not request.converter_ids: - return [] + Exclude client-preconverted pieces from request converter configurations. - converters = get_converter_service().get_converter_objects_for_ids(converter_ids=request.converter_ids) - return ConverterConfiguration.from_converters(converters=converters) + Returns: + list[ConverterConfiguration]: Configurations that still apply to at least one piece. + """ + if not preconverted_indexes: + return configurations + + filtered_configurations: list[ConverterConfiguration] = [] + for configuration in configurations: + configured_indexes = configuration.indexes_to_apply + candidate_indexes = range(piece_count) if configured_indexes is None else configured_indexes + eligible_indexes = [index for index in candidate_indexes if index not in preconverted_indexes] + if not eligible_indexes: + continue + filtered_configurations.append( + ConverterConfiguration( + converters=configuration.converters, + indexes_to_apply=eligible_indexes, + prompt_data_types_to_apply=configuration.prompt_data_types_to_apply, + ) + ) + return filtered_configurations + + @staticmethod + def _get_converter_identifiers(*, configurations: list[ConverterConfiguration]) -> list[ConverterIdentifier]: + """ + Flatten resolved converter identifiers in configuration order. + + Returns: + list[ConverterIdentifier]: The converter identifiers. + """ + return [ + ConverterIdentifier.from_component_identifier(converter.get_identifier()) + for configuration in configurations + for converter in configuration.converters + ] # ============================================================================ diff --git a/tests/unit/backend/test_api_routes.py b/tests/unit/backend/test_api_routes.py index c9f8ffba5f..90f3a512d2 100644 --- a/tests/unit/backend/test_api_routes.py +++ b/tests/unit/backend/test_api_routes.py @@ -316,12 +316,89 @@ def test_add_message_success(self, client: TestClient) -> None: response = client.post( "/api/attacks/attack-1/messages", - json={"pieces": [{"original_value": "Hello"}], "target_conversation_id": "attack-1"}, + json={ + "pieces": [{"original_value": "Hello"}], + "target_conversation_id": "attack-1", + "request_converter_configurations": [ + { + "converter_ids": ["request-1", "request-2"], + "indexes_to_apply": [0], + "prompt_data_types_to_apply": ["text"], + } + ], + "response_converter_configurations": [ + { + "converter_ids": ["response-1"], + "prompt_data_types_to_apply": ["text"], + } + ], + }, ) assert response.status_code == status.HTTP_200_OK data = response.json() assert len(data["messages"]["messages"]) == 2 + request = mock_service.add_message_async.await_args.kwargs["request"] + assert request.request_converter_configurations[0].converter_ids == [ + "request-1", + "request-2", + ] + assert request.request_converter_configurations[0].indexes_to_apply == [0] + assert request.request_converter_configurations[0].prompt_data_types_to_apply == ["text"] + assert request.response_converter_configurations[0].converter_ids == ["response-1"] + + @pytest.mark.parametrize( + "converter_fields", + [ + { + "converter_ids": ["legacy-converter"], + "request_converter_configurations": [{"converter_ids": ["request-converter"]}], + }, + {"request_converter_configurations": []}, + {"request_converter_configurations": [{"converter_ids": []}]}, + {"response_converter_configurations": []}, + {"request_converter_configurations": [{"converter_ids": ["request-converter"], "indexes_to_apply": []}]}, + {"request_converter_configurations": [{"converter_ids": ["request-converter"], "indexes_to_apply": [-1]}]}, + {"request_converter_configurations": [{"converter_ids": ["request-converter"], "indexes_to_apply": [1]}]}, + { + "request_converter_configurations": [ + {"converter_ids": ["request-converter"], "prompt_data_types_to_apply": []} + ] + }, + ], + ) + def test_add_message_rejects_invalid_converter_configurations( + self, client: TestClient, converter_fields: dict[str, object] + ) -> None: + """Test invalid structured converter configurations.""" + with patch("pyrit.backend.routes.attacks.get_attack_service") as mock_get_service: + response = client.post( + "/api/attacks/attack-1/messages", + json={ + "pieces": [{"original_value": "Hello"}], + "target_conversation_id": "attack-1", + **converter_fields, + }, + ) + + assert response.status_code == status.HTTP_422_UNPROCESSABLE_CONTENT + mock_get_service.return_value.add_message_async.assert_not_called() + + def test_add_message_rejects_converter_configuration_when_send_is_false(self, client: TestClient) -> None: + """Test that converter configurations cannot be supplied without sending.""" + with patch("pyrit.backend.routes.attacks.get_attack_service") as mock_get_service: + response = client.post( + "/api/attacks/attack-1/messages", + json={ + "pieces": [{"original_value": "Hello"}], + "target_conversation_id": "attack-1", + "send": False, + "converter_ids": ["legacy-converter"], + }, + ) + + assert response.status_code == status.HTTP_422_UNPROCESSABLE_CONTENT + mock_get_service.return_value.add_message_async.assert_not_called() def test_update_attack_not_found(self, client: TestClient) -> None: """Test updating a non-existent attack returns 404.""" @@ -365,6 +442,20 @@ def test_add_message_target_not_found(self, client: TestClient) -> None: assert response.status_code == status.HTTP_404_NOT_FOUND + def test_add_message_converter_not_found(self, client: TestClient) -> None: + """Test adding a message with an unknown converter returns 404.""" + with patch("pyrit.backend.routes.attacks.get_attack_service") as mock_get_service: + mock_service = MagicMock() + mock_service.add_message_async = AsyncMock(side_effect=ValueError("Converter instance 'missing' not found")) + mock_get_service.return_value = mock_service + + response = client.post( + "/api/attacks/attack-1/messages", + json={"pieces": [{"original_value": "Hello"}], "target_conversation_id": "attack-1"}, + ) + + assert response.status_code == status.HTTP_404_NOT_FOUND + def test_add_message_bad_request(self, client: TestClient) -> None: """Test adding message with invalid request returns 400.""" with patch("pyrit.backend.routes.attacks.get_attack_service") as mock_get_service: diff --git a/tests/unit/backend/test_attack_service.py b/tests/unit/backend/test_attack_service.py index 78697d70e9..99458bfdd7 100644 --- a/tests/unit/backend/test_attack_service.py +++ b/tests/unit/backend/test_attack_service.py @@ -19,6 +19,7 @@ AddMessageRequest, AttackSummary, ConversationMessagesResponse, + ConverterConfigurationRequest, CreateAttackRequest, MessagePieceRequest, PrependedMessageRequest, @@ -39,6 +40,7 @@ MessagePiece, ) from pyrit.models.conversation_stats import ConversationStats +from pyrit.prompt_normalizer import ConverterConfiguration @pytest.fixture @@ -1321,8 +1323,10 @@ async def test_add_message_reraises_when_send_fails_without_stored_error_piece( with pytest.raises(RuntimeError, match="boom"): await attack_service.add_message_async(attack_result_id="test-id", request=request) - async def test_add_message_with_converter_ids_gets_converters(self, attack_service, mock_memory) -> None: - """Test that add_message with converter_ids gets converters from service.""" + async def test_add_message_with_legacy_converter_ids_warns_and_preserves_behavior( + self, attack_service, mock_memory + ) -> None: + """Test that legacy converter IDs warn and remain an unrestricted pipeline.""" ar = make_attack_result(conversation_id="test-id") mock_memory.get_attack_results.return_value = [ar] mock_memory.get_message_pieces.return_value = [] @@ -1332,24 +1336,27 @@ async def test_add_message_with_converter_ids_gets_converters(self, attack_servi patch("pyrit.backend.services.attack_service.get_target_service") as mock_get_target_svc, patch("pyrit.backend.services.attack_service.get_converter_service") as mock_get_conv_svc, patch("pyrit.backend.services.attack_service.PromptNormalizer") as mock_normalizer_cls, - patch("pyrit.backend.services.attack_service.ConverterConfiguration") as mock_config, ): mock_target_svc = MagicMock() mock_target_svc.get_target_object.return_value = _make_matching_target_mock() mock_get_target_svc.return_value = mock_target_svc - mock_conv_svc = MagicMock() - mock_converter = MagicMock() - mock_converter.get_identifier.return_value = ComponentIdentifier( - class_name="TestConverter", + first_converter = MagicMock() + first_converter.get_identifier.return_value = ComponentIdentifier( + class_name="FirstConverter", class_module="test_module", params={"supported_input_types": ("text",), "supported_output_types": ("text",)}, ) - mock_conv_svc.get_converter_objects_for_ids.return_value = [mock_converter] + second_converter = MagicMock() + second_converter.get_identifier.return_value = ComponentIdentifier( + class_name="SecondConverter", + class_module="test_module", + params={"supported_input_types": ("text",), "supported_output_types": ("text",)}, + ) + mock_conv_svc = MagicMock() + mock_conv_svc.get_converter_objects_for_ids.return_value = [first_converter, second_converter] mock_get_conv_svc.return_value = mock_conv_svc - mock_config.from_converters.return_value = [MagicMock()] - mock_normalizer = MagicMock() mock_normalizer.send_prompt_async = AsyncMock() mock_normalizer_cls.return_value = mock_normalizer @@ -1358,13 +1365,157 @@ async def test_add_message_with_converter_ids_gets_converters(self, attack_servi pieces=[MessagePieceRequest(original_value="Hello")], target_conversation_id="test-id", send=True, - converter_ids=["conv-1"], + converter_ids=["first", "second"], + target_registry_name="test-target", + ) + + with pytest.warns(DeprecationWarning, match="AddMessageRequest.converter_ids is deprecated"): + await attack_service.add_message_async(attack_result_id="test-id", request=request) + + configurations = mock_normalizer.send_prompt_async.call_args.kwargs["request_converter_configurations"] + assert [configuration.converters for configuration in configurations] == [ + [first_converter], + [second_converter], + ] + assert all(configuration.indexes_to_apply is None for configuration in configurations) + mock_conv_svc.get_converter_objects_for_ids.assert_called_once_with(converter_ids=["first", "second"]) + + async def test_add_message_preserves_converter_configuration_targeting(self, attack_service, mock_memory) -> None: + """Test that request and response converter targeting reaches the normalizer.""" + ar = make_attack_result(conversation_id="test-id") + mock_memory.get_attack_results.return_value = [ar] + mock_memory.get_message_pieces.return_value = [] + mock_memory.get_conversation_messages.return_value = [] + + with ( + patch("pyrit.backend.services.attack_service.get_target_service") as mock_get_target_svc, + patch("pyrit.backend.services.attack_service.get_converter_service") as mock_get_conv_svc, + patch("pyrit.backend.services.attack_service.PromptNormalizer") as mock_normalizer_cls, + ): + mock_target_svc = MagicMock() + mock_target_svc.get_target_object.return_value = _make_matching_target_mock() + mock_get_target_svc.return_value = mock_target_svc + + mock_conv_svc = MagicMock() + first_request_converter = MagicMock() + first_request_converter.get_identifier.return_value = ComponentIdentifier( + class_name="FirstRequestConverter", + class_module="test_module", + params={"supported_input_types": ("text",), "supported_output_types": ("text",)}, + ) + second_request_converter = MagicMock() + second_request_converter.get_identifier.return_value = ComponentIdentifier( + class_name="SecondRequestConverter", + class_module="test_module", + params={"supported_input_types": ("text",), "supported_output_types": ("text",)}, + ) + third_request_converter = MagicMock() + third_request_converter.get_identifier.return_value = ComponentIdentifier( + class_name="ThirdRequestConverter", + class_module="test_module", + params={"supported_input_types": ("image_path",), "supported_output_types": ("image_path",)}, + ) + response_converter = MagicMock() + response_converter.get_identifier.return_value = ComponentIdentifier( + class_name="ResponseConverter", + class_module="test_module", + params={"supported_input_types": ("text",), "supported_output_types": ("text",)}, + ) + converters_by_ids = { + ("request-1", "request-2"): [first_request_converter, second_request_converter], + ("request-3",): [third_request_converter], + ("response-1",): [response_converter], + } + mock_conv_svc.get_converter_objects_for_ids.side_effect = lambda *, converter_ids: converters_by_ids[ + tuple(converter_ids) + ] + mock_get_conv_svc.return_value = mock_conv_svc + + mock_normalizer = MagicMock() + mock_normalizer.send_prompt_async = AsyncMock() + mock_normalizer_cls.return_value = mock_normalizer + + request = AddMessageRequest( + pieces=[ + MessagePieceRequest(original_value="Hello"), + MessagePieceRequest( + data_type="image_path", + original_value="https://example.com/image.png", + ), + ], + target_conversation_id="test-id", + send=True, + request_converter_configurations=[ + ConverterConfigurationRequest( + converter_ids=["request-1", "request-2"], + indexes_to_apply=[0], + prompt_data_types_to_apply=["text"], + ), + ConverterConfigurationRequest( + converter_ids=["request-3"], + indexes_to_apply=[1], + prompt_data_types_to_apply=["image_path"], + ), + ], + response_converter_configurations=[ + ConverterConfigurationRequest( + converter_ids=["response-1"], + indexes_to_apply=[1], + prompt_data_types_to_apply=["text"], + ) + ], target_registry_name="test-target", ) await attack_service.add_message_async(attack_result_id="test-id", request=request) - mock_conv_svc.get_converter_objects_for_ids.assert_any_call(converter_ids=["conv-1"]) + call_kwargs = mock_normalizer.send_prompt_async.call_args.kwargs + request_configs = call_kwargs["request_converter_configurations"] + assert request_configs[0].converters == [first_request_converter, second_request_converter] + assert request_configs[0].indexes_to_apply == [0] + assert request_configs[0].prompt_data_types_to_apply == ["text"] + assert request_configs[1].converters == [third_request_converter] + assert request_configs[1].indexes_to_apply == [1] + assert request_configs[1].prompt_data_types_to_apply == ["image_path"] + response_config = call_kwargs["response_converter_configurations"][0] + assert response_config.converters == [response_converter] + assert response_config.indexes_to_apply == [1] + assert response_config.prompt_data_types_to_apply == ["text"] + + update_fields = mock_memory.update_attack_result_by_id.call_args.kwargs["update_fields"] + updated_atomic = AtomicAttackIdentifier.model_validate(update_fields["atomic_attack_identifier"]) + updated_attack = updated_atomic.attack_technique.attack + assert [converter.class_name for converter in updated_attack.request_converters] == [ + "FirstRequestConverter", + "SecondRequestConverter", + "ThirdRequestConverter", + ] + assert [converter.class_name for converter in updated_attack.response_converters] == ["ResponseConverter"] + assert mock_conv_svc.get_converter_objects_for_ids.call_count == 3 + + async def test_add_message_resolves_converters_before_writing(self, attack_service, mock_memory) -> None: + """Test that an unknown converter fails before message or attack writes.""" + ar = make_attack_result(conversation_id="test-id", has_target=False) + mock_memory.get_attack_results.return_value = [ar] + request = AddMessageRequest( + pieces=[MessagePieceRequest(original_value="Hello")], + target_conversation_id="test-id", + send=True, + target_registry_name="test-target", + request_converter_configurations=[ConverterConfigurationRequest(converter_ids=["missing"])], + ) + + with patch("pyrit.backend.services.attack_service.get_converter_service") as mock_get_service: + mock_get_service.return_value.get_converter_objects_for_ids.side_effect = ValueError( + "Converter instance 'missing' not found" + ) + + with pytest.raises(ValueError, match="Converter instance 'missing' not found"): + await attack_service.add_message_async(attack_result_id="test-id", request=request) + + mock_memory.add_conversation_to_memory.assert_not_called() + mock_memory.add_message_pieces_to_memory.assert_not_called() + mock_memory.update_attack_result_by_id.assert_not_called() async def test_add_message_raises_when_attack_not_found_after_update(self, attack_service, mock_memory) -> None: """Test that add_message raises ValueError when attack disappears after update.""" @@ -1429,8 +1580,10 @@ async def test_add_message_bumps_timestamp(self, attack_service, mock_memory) -> assert isinstance(update_fields["timestamp"], datetime) assert "attack_metadata" not in update_fields - async def test_converter_ids_propagate_even_when_preconverted(self, attack_service, mock_memory) -> None: - """Test that converter identifiers propagate to attack_identifier even when pieces are preconverted.""" + async def test_preconverted_piece_does_not_disable_other_piece_converters( + self, attack_service, mock_memory + ) -> None: + """Test that only the client-preconverted piece is excluded from conversion.""" ar = make_attack_result(conversation_id="test-id") mock_memory.get_attack_results.return_value = [ar] mock_memory.get_message_pieces.return_value = [] @@ -1461,24 +1614,39 @@ async def test_converter_ids_propagate_even_when_preconverted(self, attack_servi mock_normalizer_cls.return_value = mock_normalizer request = AddMessageRequest( - pieces=[MessagePieceRequest(original_value="Hello", converted_value="SGVsbG8=")], + pieces=[ + MessagePieceRequest(original_value="Hello", converted_value="SGVsbG8="), + MessagePieceRequest(original_value="World"), + ], send=True, target_conversation_id="test-id", - converter_ids=["conv-1"], + request_converter_configurations=[ConverterConfigurationRequest(converter_ids=["conv-1"])], + response_converter_configurations=[ConverterConfigurationRequest(converter_ids=["conv-1"])], target_registry_name="test-target", ) await attack_service.add_message_async(attack_result_id="test-id", request=request) - # Converter service IS called to resolve identifiers for the attack_identifier - mock_get_conv_svc.assert_called() - # Normalizer should still get empty converter configs since pieces are preconverted call_kwargs = mock_normalizer.send_prompt_async.call_args[1] - assert call_kwargs["request_converter_configurations"] == [] - # atomic_attack_identifier should be updated with converter identifiers + request_configurations = call_kwargs["request_converter_configurations"] + assert len(request_configurations) == 1 + assert request_configurations[0].indexes_to_apply == [1] + assert len(call_kwargs["response_converter_configurations"]) == 1 update_call = mock_memory.update_attack_result_by_id.call_args[1] assert "atomic_attack_identifier" in update_call["update_fields"] + def test_preconverted_piece_omits_configuration_with_no_eligible_indexes(self, attack_service) -> None: + """Test that an empty filtered selector is omitted instead of becoming unrestricted.""" + configuration = ConverterConfiguration(converters=[MagicMock()], indexes_to_apply=[0]) + + result = attack_service._exclude_preconverted_piece_indexes( + configurations=[configuration], + preconverted_indexes={0}, + piece_count=2, + ) + + assert result == [] + # ============================================================================ # Pagination Tests @@ -2561,12 +2729,15 @@ async def test_add_message_merges_converter_identifiers_without_duplicates(self, role="user", pieces=[MessagePieceRequest(original_value="Hello")], target_conversation_id="attack-1", - send=False, - converter_ids=["c-1", "c-2"], + send=True, + target_registry_name="test-target", + request_converter_configurations=[ConverterConfigurationRequest(converter_ids=["c-1", "c-2"])], ) with ( patch("pyrit.backend.services.attack_service.get_converter_service") as mock_get_converter_service, + patch("pyrit.backend.services.attack_service.get_target_service") as mock_get_target_service, + patch("pyrit.backend.services.attack_service.PromptNormalizer") as mock_normalizer_cls, patch.object( attack_service, "get_attack_async", @@ -2594,6 +2765,8 @@ async def test_add_message_merges_converter_identifiers_without_duplicates(self, MagicMock(get_identifier=MagicMock(return_value=new_converter)), ] mock_get_converter_service.return_value = mock_converter_service + mock_get_target_service.return_value.get_target_object.return_value = _make_matching_target_mock() + mock_normalizer_cls.return_value.send_prompt_async = AsyncMock() await attack_service.add_message_async(attack_result_id="attack-1", request=request) @@ -2639,12 +2812,15 @@ async def test_converter_merge_with_flat_atomic_identifier(self, attack_service, role="user", pieces=[MessagePieceRequest(original_value="Hello")], target_conversation_id="flat-1", - send=False, - converter_ids=["c-1"], + send=True, + target_registry_name="test-target", + request_converter_configurations=[ConverterConfigurationRequest(converter_ids=["c-1"])], ) with ( patch("pyrit.backend.services.attack_service.get_converter_service") as mock_get_converter_service, + patch("pyrit.backend.services.attack_service.get_target_service") as mock_get_target_service, + patch("pyrit.backend.services.attack_service.PromptNormalizer") as mock_normalizer_cls, patch.object( attack_service, "get_attack_async", @@ -2671,6 +2847,8 @@ async def test_converter_merge_with_flat_atomic_identifier(self, attack_service, MagicMock(get_identifier=MagicMock(return_value=new_converter)), ] mock_get_converter_service.return_value = mock_converter_service + mock_get_target_service.return_value.get_target_object.return_value = _make_matching_target_mock() + mock_normalizer_cls.return_value.send_prompt_async = AsyncMock() await attack_service.add_message_async(attack_result_id="flat-1", request=request) @@ -2719,12 +2897,15 @@ async def test_converter_merge_all_duplicates_does_not_rewrite_identifier(self, role="user", pieces=[MessagePieceRequest(original_value="Hello")], target_conversation_id="attack-1", - send=False, - converter_ids=["c-1"], + send=True, + target_registry_name="test-target", + request_converter_configurations=[ConverterConfigurationRequest(converter_ids=["c-1"])], ) with ( patch("pyrit.backend.services.attack_service.get_converter_service") as mock_get_converter_service, + patch("pyrit.backend.services.attack_service.get_target_service") as mock_get_target_service, + patch("pyrit.backend.services.attack_service.PromptNormalizer") as mock_normalizer_cls, patch.object( attack_service, "get_attack_async", @@ -2751,6 +2932,8 @@ async def test_converter_merge_all_duplicates_does_not_rewrite_identifier(self, MagicMock(get_identifier=MagicMock(return_value=duplicate_converter)), ] mock_get_converter_service.return_value = mock_converter_service + mock_get_target_service.return_value.get_target_object.return_value = _make_matching_target_mock() + mock_normalizer_cls.return_value.send_prompt_async = AsyncMock() await attack_service.add_message_async(attack_result_id="attack-1", request=request) @@ -2780,12 +2963,15 @@ async def test_converter_merge_preserves_sibling_children_hash(self, attack_serv role="user", pieces=[MessagePieceRequest(original_value="Hello")], target_conversation_id="attack-1", - send=False, - converter_ids=["c-1"], + send=True, + target_registry_name="test-target", + request_converter_configurations=[ConverterConfigurationRequest(converter_ids=["c-1"])], ) with ( patch("pyrit.backend.services.attack_service.get_converter_service") as mock_get_converter_service, + patch("pyrit.backend.services.attack_service.get_target_service") as mock_get_target_service, + patch("pyrit.backend.services.attack_service.PromptNormalizer") as mock_normalizer_cls, patch.object( attack_service, "get_attack_async", @@ -2812,6 +2998,8 @@ async def test_converter_merge_preserves_sibling_children_hash(self, attack_serv MagicMock(get_identifier=MagicMock(return_value=new_converter)), ] mock_get_converter_service.return_value = mock_converter_service + mock_get_target_service.return_value.get_target_object.return_value = _make_matching_target_mock() + mock_normalizer_cls.return_value.send_prompt_async = AsyncMock() await attack_service.add_message_async(attack_result_id="attack-1", request=request)