From 406034dd88eb1ac58215011a916f61b4369de174 Mon Sep 17 00:00:00 2001 From: Marton Almasy Date: Mon, 25 Nov 2024 12:12:19 +0100 Subject: [PATCH 1/4] Python rt client added, gitignore edited --- .gitignore | 3 + src/python/pyproject.toml | 44 + src/python/rtclient/__init__.py | 837 ++++++++++++++++++ src/python/rtclient/client_test.py | 424 +++++++++ src/python/rtclient/defaults.py | 18 + src/python/rtclient/low_level_client.py | 140 +++ src/python/rtclient/models.py | 706 +++++++++++++++ src/python/rtclient/util/id_generator.py | 9 + src/python/rtclient/util/id_generator_test.py | 11 + src/python/rtclient/util/message_queue.py | 104 +++ .../rtclient/util/message_queue_test.py | 200 +++++ src/python/rtclient/util/model_helpers.py | 14 + .../rtclient/util/model_helpers_test.py | 23 + src/python/rtclient/util/user_agent.py | 11 + src/python/rtclient/util/user_agent_test.py | 12 + 15 files changed, 2556 insertions(+) create mode 100644 src/python/pyproject.toml create mode 100644 src/python/rtclient/__init__.py create mode 100644 src/python/rtclient/client_test.py create mode 100644 src/python/rtclient/defaults.py create mode 100644 src/python/rtclient/low_level_client.py create mode 100644 src/python/rtclient/models.py create mode 100644 src/python/rtclient/util/id_generator.py create mode 100644 src/python/rtclient/util/id_generator_test.py create mode 100644 src/python/rtclient/util/message_queue.py create mode 100644 src/python/rtclient/util/message_queue_test.py create mode 100644 src/python/rtclient/util/model_helpers.py create mode 100644 src/python/rtclient/util/model_helpers_test.py create mode 100644 src/python/rtclient/util/user_agent.py create mode 100644 src/python/rtclient/util/user_agent_test.py diff --git a/.gitignore b/.gitignore index 8345a20..863f647 100644 --- a/.gitignore +++ b/.gitignore @@ -398,3 +398,6 @@ FodyWeavers.xsd *.sln.iml bak/RealtimeConversationClientRX.cs bak/RealtimeConversationClientRX.Observables.cs + +# .env file for python +.env \ No newline at end of file diff --git a/src/python/pyproject.toml b/src/python/pyproject.toml new file mode 100644 index 0000000..7fdf79d --- /dev/null +++ b/src/python/pyproject.toml @@ -0,0 +1,44 @@ +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +name = "rtclient" +version = "0.5.2" +description = "A client for the RT API" +authors = ["Microsoft Corporation"] + +[tool.poetry.dependencies] +python = ">=3.10" +aiohttp = "*" +azure-identity = "*" +pydantic = "*" + +[tool.poetry.dev-dependencies] +ruff = "*" +black = "*" +python-dotenv = "*" +soundfile = "*" +numpy = "*" +scipy = "*" +pytest = "*" +pytest-asyncio = "*" + +[tool.poetry.scripts] + + +[tool.ruff] +line-length = 120 +target-version = "py312" + +[tool.ruff.lint] +select = ["E", "F", "I", "UP"] +extend-ignore = ["UP007"] + +[tool.black] +line-length = 120 +target-version = ["py312"] + +[tool.pytest.ini_options] +asyncio_mode = "auto" +asyncio_default_fixture_loop_scope = "function" diff --git a/src/python/rtclient/__init__.py b/src/python/rtclient/__init__.py new file mode 100644 index 0000000..68c8f47 --- /dev/null +++ b/src/python/rtclient/__init__.py @@ -0,0 +1,837 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import asyncio +import base64 +import uuid +from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable +from typing import Literal, Optional, TypeGuard, Union + +from azure.core.credentials import AzureKeyCredential +from azure.core.credentials_async import AsyncTokenCredential + +from rtclient.low_level_client import RTLowLevelClient +from rtclient.models import ( + AssistantContentPart, + AssistantMessageItem, + AudioFormat, + ClientMessageBase, + ErrorMessage, + FunctionCallItem, + FunctionCallOutputItem, + FunctionToolChoice, + InputAudioBufferAppendMessage, + InputAudioBufferClearedMessage, + InputAudioBufferClearMessage, + InputAudioBufferCommitMessage, + InputAudioBufferCommittedMessage, + InputAudioBufferSpeechStartedMessage, + InputAudioBufferSpeechStoppedMessage, + InputAudioContentPart, + InputAudioTranscription, + InputTextContentPart, + Item, + ItemCreatedMessage, + ItemCreateMessage, + ItemDeletedMessage, + ItemDeleteMessage, + ItemInputAudioTranscriptionCompletedMessage, + ItemInputAudioTranscriptionFailedMessage, + ItemParamStatus, + ItemTruncatedMessage, + ItemTruncateMessage, + MessageItem, + MessageItemType, + MessageRole, + Modality, + NoTurnDetection, + OutputTextContentPart, + RateLimits, + RateLimitsUpdatedMessage, + RealtimeError, + Response, + ResponseAudioDeltaMessage, + ResponseAudioDoneMessage, + ResponseAudioTranscriptDeltaMessage, + ResponseAudioTranscriptDoneMessage, + ResponseCancelledDetails, + ResponseCancelMessage, + ResponseContentPartAddedMessage, + ResponseContentPartDoneMessage, + ResponseCreatedMessage, + ResponseCreateMessage, + ResponseCreateParams, + ResponseDoneMessage, + ResponseFailedDetails, + ResponseFunctionCallArgumentsDeltaMessage, + ResponseFunctionCallArgumentsDoneMessage, + ResponseFunctionCallItem, + ResponseFunctionCallOutputItem, + ResponseIncompleteDetails, + ResponseItem, + ResponseItemAudioContentPart, + ResponseItemBase, + ResponseItemContentPart, + ResponseItemInputAudioContentPart, + ResponseItemInputTextContentPart, + ResponseItemStatus, + ResponseItemTextContentPart, + ResponseMessageItem, + ResponseOutputItemAddedMessage, + ResponseOutputItemDoneMessage, + ResponseStatus, + ResponseStatusDetails, + ResponseTextDeltaMessage, + ResponseTextDoneMessage, + ServerMessageBase, + ServerMessageType, + ServerVAD, + Session, + SessionCreatedMessage, + SessionUpdatedMessage, + SessionUpdateMessage, + SessionUpdateParams, + SystemContentPart, + SystemMessageItem, + Temperature, + ToolChoice, + ToolsDefinition, + TurnDetection, + Usage, + UserContentPart, + UserMessageItem, + UserMessageType, + Voice, + create_message_from_dict, +) +from rtclient.util.id_generator import generate_id +from rtclient.util.message_queue import MessageQueueWithError + + +class RealtimeException(Exception): + def __init__(self, error: RealtimeError): + self.error = error + super().__init__(error.message) + + @property + def message(self): + return self.error.message + + @property + def type(self): + return self.error.type + + @property + def code(self): + return self.error.code + + @property + def param(self): + return self.error.param + + @property + def event_id(self): + return self.error.event_id + + +class RTInputAudioItem: + def __init__( + self, + id: str, + audio_start_ms: Optional[int], + has_transcription: bool, + queue: MessageQueueWithError[ServerMessageType], + ): + self.type: Literal["input_audio"] = "input_audio" + self.id = id + self.audio_start_ms = audio_start_ms + self.audio_end_ms: Optional[int] = None + self.transcript: Optional[str] = None + self._has_transcription = has_transcription + self.__queue = queue + + def __await__(self): + async def resolve(): + while True: + message = await self.__queue.receive( + lambda m: ( + m.type + in [ + "input_audio_buffer.speech_stopped", + "conversation.item.input_audio_transcription.completed", + "conversation.item.input_audio_transcription.failed", + ] + and m.item_id == self.id + ) + or (m.type == "conversation.item.created" and m.item.id == self.id) + ) + if message is None: + return + elif message.type == "error": + raise RealtimeException(message.error) + elif message.type == "input_audio_buffer.speech_stopped": + self.audio_end_ms = message.audio_end_ms + if not self._has_transcription: + return + elif message.type == "conversation.item.created" and not self._has_transcription: + return + elif message.type == "conversation.item.input_audio_transcription.completed": + self.transcript = message.transcript + return + elif message.type == "conversation.item.input_audio_transcription.failed": + raise RealtimeError(message.error) + + return resolve().__await__() + + +class SharedEndQueue: + def __init__( + self, + receive_delegate: Callable[[], Awaitable[ServerMessageType]], + error_predicate: Callable[[ServerMessageType], bool], + end_predicate: Callable[[ServerMessageType], bool], + ): + self._receive_delegate = receive_delegate + self._error_predicate = error_predicate + self._end_predicate = end_predicate + self._queue = [] + self._lock = asyncio.Lock() + + async def receive(self, predicate: Callable[[ServerMessageType], bool]): + async with self._lock: + for i, message in enumerate(self._queue): + if predicate(message): + return self._queue.pop(i) + elif self._end_predicate(message): + return message + + while True: + message = await self._receive_delegate() + if message is None or self._error_predicate(message) or predicate(message): + return message + if self._end_predicate(message): + self._queue.append(message) + return message + self._queue.append(message) + + +class RTAudioContent: + def __init__(self, message: ResponseContentPartAddedMessage, queue: MessageQueueWithError[ServerMessageType]): + self.type: Literal["audio"] = "audio" + self._item_id = message.item_id + self._content_index = message.content_index + assert message.part.type == "audio" + self._part = message.part + self.__queue = queue + self.__content_queue = SharedEndQueue( + self._receive_content, + lambda m: m.type == "error", + lambda m: m.type == "response.content_part.done", + ) + + async def _receive_content(self): + def is_valid_message( + m: ServerMessageType, + ) -> TypeGuard[ + Union[ + ResponseAudioDeltaMessage, + ResponseAudioDoneMessage, + ResponseAudioTranscriptDeltaMessage, + ResponseAudioTranscriptDoneMessage, + ResponseContentPartDoneMessage, + ] + ]: + return m.type in [ + "response.audio.delta", + "response.audio.done", + "response.audio_transcript.delta", + "response.audio_transcript.done", + "response.content_part.done", + ] + + return await self.__queue.receive( + lambda m: is_valid_message(m) and m.item_id == self.item_id and m.content_index == self.content_index + ) + + @property + def item_id(self) -> str: + return self._item_id + + @property + def content_index(self) -> int: + return self._content_index + + @property + def transcript(self) -> str: + return self._part.transcript + + async def audio_chunks(self) -> AsyncGenerator[bytes]: + while True: + message = await self.__content_queue.receive( + lambda m: m.type in ["response.audio.delta", "response.audio.done"] + ) + if message is None: + break + if message.type == "response.content_part.done": + self._part = message.part + break + if message.type == "error": + raise RealtimeException(message.error) + if message.type == "response.audio.delta": + yield base64.b64decode(message.delta) + elif message.type == "response.audio.done": + # We are skipping this as it's information is already provided by 'response.content_part.done' + # and that is a better signal to end the iteration + continue + + async def transcript_chunks(self) -> AsyncGenerator[str]: + while True: + message = await self.__content_queue.receive( + lambda m: m.type in ["response.audio_transcript.delta", "response.audio_transcript.done"] + ) + if message is None: + break + if message.type == "response.content_part.done": + self._part = message.part + break + if message.type == "error": + raise RealtimeException(message.error) + if message.type == "response.audio_transcript.delta": + yield message.delta + elif message.type == "response.audio_transcript.done": + # We are skipping this as it's information is already provided by 'response.content_part.done' + # and that is a better signal to end the iteration + continue + + +class RTTextContent: + def __init__(self, message: ResponseContentPartAddedMessage, queue: MessageQueueWithError[ServerMessageType]): + self.type: Literal["text"] = "text" + self._item_id = message.item_id + self._content_index = message.content_index + assert message.part.type == "text" + self._part = message.part + self.__queue = queue + self.__content_queue = MessageQueueWithError( + self._receive_content, lambda m: m.type == "response.content_part.done" + ) + + async def _receive_content(self): + def is_valid_message( + m: ServerMessageType, + ) -> TypeGuard[ + Union[ + ResponseTextDeltaMessage, + ResponseTextDoneMessage, + ResponseContentPartDoneMessage, + ] + ]: + return m.type in [ + "response.text.delta", + "response.text.done", + "response.content_part.done", + ] + + return await self.__queue.receive( + lambda m: is_valid_message(m) and m.item_id == self.item_id and m.content_index == self.content_index + ) + + @property + def item_id(self) -> str: + return self._item_id + + @property + def content_index(self) -> int: + return self._content_index + + @property + def text(self) -> str: + return self._part.text + + async def text_chunks(self) -> AsyncGenerator[str]: + while True: + message = await self.__content_queue.receive( + lambda m: m.type in ["response.text.delta", "response.text.done"] + ) + if message is None: + break + if message.type == "response.content_part.done": + assert message.part.type == "text" + self._part = message.part + break + if message.type == "error": + raise RealtimeException(message.error) + if message.type == "response.text.delta": + yield message.delta + elif message.type == "response.text.done": + # We are skipping this as it's information is already provided by 'response.content_part.done' + # and that is a better signal to end the iteration + continue + + +RTMessageContent = Union[RTAudioContent, RTTextContent] + + +class RTMessageItem: + def __init__( + self, + response_id: str, + item: ResponseItem, + previous_id: Optional[str], + queue: MessageQueueWithError[ServerMessageType], + ): + self.type: Literal["message"] = "message" + self.response_id = response_id + self._item = item + self.previous_id = previous_id + self.__queue = queue + + @property + def id(self) -> str: + return self._item.id + + # TODO: Add more properties here + + def __aiter__(self) -> AsyncIterator[RTMessageContent]: + return self + + async def __anext__(self): + message = await self.__queue.receive( + lambda m: (m.type == "response.content_part.added" and m.item_id == self.id) + or (m.type == "response.output_item.done" and m.item.id == self.id) + ) + if message is None: + raise StopAsyncIteration + if message.type == "error": + raise RealtimeException(message.error) + if message.type == "response.output_item.done": + self._item = message.item + raise StopAsyncIteration + assert message.type == "response.content_part.added" + if message.part.type == "audio": + return RTAudioContent(message, self.__queue) + elif message.part.type == "text": + return RTTextContent(message, self.__queue) + raise ValueError(f"Unexpected part type {message.part.type}") + + +class RTFunctionCallItem: + def __init__( + self, + response_id: str, + item: ResponseItem, + previous_id: Optional[str], + queue: MessageQueueWithError[ServerMessageType], + ) -> None: + self.type: Literal["function_call"] = "function_call" + self.response_id = response_id + self._item = item + self.previous_id = previous_id + self.__queue = queue + self.__awaited = False + self.__iterated = False + + @property + def id(self) -> str: + return self._item.id + + @property + def function_name(self) -> str: + assert self._item.type == "function_call" + return self._item.name + + @property + def call_id(self) -> str: + assert self._item.type == "function_call" + return self._item.call_id + + @property + def arguments(self) -> str: + assert self._item.type == "function_call" + return self._item.arguments + + async def __inner_iter(self): + while True: + message = await self.__queue.receive( + lambda m: ( + m.type in ["response.function_call_arguments.delta", "response.function_call_arguments.done"] + and m.item_id == self.id + ) + or (m.type == "response.output_item.done" and m.item.id == self.id) + ) + if message is None: + break + if message.type == "error": + raise RealtimeException(message.error) + if message.type == "response.output_item.done": + self._item = message.item + break + if message.type == "response.function_call_arguments.delta": + yield message.delta + if message.type == "response.function_call_arguments.done": + continue + + def __aiter__(self) -> AsyncIterator[str]: + if self.__awaited: + raise RuntimeError("Cannot iterate after awaiting") + self.__iterated = True + return self + + async def __anext__(self): + return await self.__inner_iter().__anext__() + + def __await__(self): + if self.__iterated: + raise RuntimeError("Cannot await after iterating") + self.__awaited = True + + async def wait_for_completion(): + async for _ in self.__inner_iter(): + pass + + return wait_for_completion().__await__() + + +RTOutputItem = Union[RTMessageItem, RTFunctionCallItem] + + +class RTResponse: + def __init__( + self, + response: Response, + queue: MessageQueueWithError[ServerMessageType], + client: RTLowLevelClient, + ): + self.type: Literal["response"] = "response" + self._response = response + self.__queue = queue + self._client = client + self._done = False + + @property + def id(self) -> str: + return self._response.id + + @property + def status(self) -> ResponseStatus: + return self._response.status + + @property + def status_details(self) -> Optional[ResponseStatusDetails]: + return self._response.status_details + + @property + def output(self) -> list[ResponseItem]: + return self._response.output + + @property + def usage(self) -> Optional[Usage]: + return self._response.usage + + async def cancel(self) -> None: + await self._client.send(ResponseCancelMessage(response_id=self.id)) + # We drain the queue to ensure that the response is marked as cancelled + async for _ in self: + pass + + def __aiter__(self) -> AsyncIterator[RTOutputItem]: + return self + + async def __anext__(self): + if self._done: + raise StopAsyncIteration + message = await self.__queue.receive( + lambda m: (m.type == "response.done" and m.response.id == self.id) + or (m.type == "response.output_item.added" and m.response_id == self.id) + ) + if message is None: + raise StopAsyncIteration + if message.type == "error": + raise RealtimeException(message.error) + if message.type == "response.done": + self._done = True + self._response = message.response + raise StopAsyncIteration + if message.type == "response.output_item.added": + # TODO: This can probably be generalized and reused (similar to the input item pattern) + created_message = await self.__queue.receive( + lambda m: m.type == "conversation.item.created" and m.item.id == message.item.id + ) + if created_message is None: + raise StopAsyncIteration + if created_message.type == "error": + raise RealtimeException(created_message.error) + assert created_message.type == "conversation.item.created" + if created_message.item.type == "message": + return RTMessageItem(self.id, created_message.item, created_message.previous_item_id, self.__queue) + elif created_message.item.type == "function_call": + return RTFunctionCallItem(self.id, created_message.item, created_message.previous_item_id, self.__queue) + else: + raise ValueError(f"Unexpected item type {created_message.item.type}") + raise ValueError(f"Unexpected message type {message.type}") + + +class RTClient: + def __init__( + self, + url: Optional[str] = None, + token_credential: Optional[AsyncTokenCredential] = None, + key_credential: Optional[AzureKeyCredential] = None, + model: Optional[str] = None, + azure_deployment: Optional[str] = None, + ): + self._client = RTLowLevelClient(url, token_credential, key_credential, model, azure_deployment) + + self._message_queue = MessageQueueWithError(self._receive_message, lambda m: m.type == "error") + + self.session: Optional[Session] = None + + self._response_map: dict[str, str] = {} + + @property + def request_id(self) -> uuid.UUID | None: + return self._client.request_id + + async def _receive_message(self): + async for message in self._client: + return message + return None + + async def configure( + self, + model: Optional[str] = None, + modalities: Optional[set[Modality]] = None, + voice: Optional[Voice] = None, + instructions: Optional[str] = None, + input_audio_format: Optional[AudioFormat] = None, + output_audio_format: Optional[AudioFormat] = None, + input_audio_transcription: Optional[InputAudioTranscription] = None, + turn_detection: Optional[TurnDetection] = None, + tools: Optional[ToolsDefinition] = None, + tool_choice: Optional[ToolChoice] = None, + temperature: Optional[Temperature] = None, + max_response_output_tokens: Optional[int] = None, + ) -> Session: + session_update_params = SessionUpdateParams() + if model is not None: + session_update_params.model = model + if modalities is not None: + session_update_params.modalities = modalities + if voice is not None: + session_update_params.voice = voice + if instructions is not None: + session_update_params.instructions = instructions + if input_audio_format is not None: + session_update_params.input_audio_format = input_audio_format + if output_audio_format is not None: + session_update_params.output_audio_format = output_audio_format + if input_audio_transcription is not None: + session_update_params.input_audio_transcription = input_audio_transcription + if turn_detection is not None: + session_update_params.turn_detection = turn_detection + if tools is not None: + session_update_params.tools = tools + if tool_choice is not None: + session_update_params.tool_choice = tool_choice + if temperature is not None: + session_update_params.temperature = temperature + if max_response_output_tokens is not None: + session_update_params.max_response_output_tokens = max_response_output_tokens + await self._client.send(SessionUpdateMessage(session=session_update_params)) + + message = await self._message_queue.receive(lambda m: m.type == "session.updated") + if message.type == "error": + raise RealtimeException(message.error) + assert message.type == "session.updated" + self.session = message.session + return message.session + + async def send_audio(self, audio: bytes) -> None: + base64_encoded = base64.b64encode(audio).decode("utf-8") + await self._client.send(InputAudioBufferAppendMessage(audio=base64_encoded)) + + async def commit_audio(self) -> RTInputAudioItem: + await self._client.send(InputAudioBufferCommitMessage()) + message = await self._message_queue.receive(lambda m: m.type == "input_audio_buffer.committed") + if message.type == "error": + raise RealtimeException(message.error) + assert message.type == "input_audio_buffer.committed" + return RTInputAudioItem( + message.item_id, + None, + self.session.input_audio_transcription is not None, + self._message_queue, + ) + + async def clear_audio(self) -> None: + await self._client.send(InputAudioBufferClearMessage()) + message = await self._message_queue.receive(lambda m: m.type == "input_audio_buffer.cleared") + if message.type == "error": + raise RealtimeException(message.error) + assert message.type == "input_audio_buffer.cleared" + + # TODO: Consider splitting this into one method per type of item. + async def send_item(self, item: Item, previous_item_id: Optional[str] = None) -> ResponseItem: + item.id = item.id or generate_id("item") + await self._client.send(ItemCreateMessage(previous_item_id=previous_item_id, item=item)) + message = await self._message_queue.receive( + lambda m: m.type == "conversation.item.created" and m.item.id == item.id + ) + if message.type == "error": + raise RealtimeException(message.error) + assert message.type == "conversation.item.created" + # TODO: Use input item wrapper + return message.item + + async def remove_item(self, item_id: str) -> None: + await self._client.send(ItemDeleteMessage(item_id=item_id)) + message = await self._message_queue.receive( + lambda m: m.type == "conversation.item.deleted" and m.item_id == item_id + ) + if message.type == "error": + raise RealtimeException(message.error) + assert message.type == "conversation.item.deleted" + + async def generate_response(self) -> RTResponse: + await self._client.send(ResponseCreateMessage()) + message = await self._message_queue.receive(lambda m: m.type == "response.created") + if message.type == "error": + raise RealtimeException(message.error) + assert message.type == "response.created" + # TODO: Need to verify if there is a way to correlate the response.create message with the + # response.created message + return RTResponse(message.response, self._message_queue, self._client) + + async def events(self) -> AsyncGenerator[RTInputAudioItem | RTResponse]: + # TODO: Add the updated quota message as a control type of event. + while True: + message = await self._message_queue.receive( + lambda m: m.type == "input_audio_buffer.speech_started" or m.type == "response.created" + ) + if message is None: + break + elif message.type == "input_audio_buffer.speech_started": + item_id = message.item_id + yield RTInputAudioItem( + item_id, + message.audio_start_ms, + self.session.input_audio_transcription is not None, + self._message_queue, + ) + elif message.type == "response.created": + yield RTResponse(message.response, self._message_queue, self._client) + else: + raise ValueError(f"Unexpected message type {message.type}") + + async def connect(self): + await self._client.connect() + message = await self._message_queue.receive(lambda m: m.type == "session.created") + if message.type == "error": + raise RealtimeException(message.error) + self.session = message.session + + async def close(self): + await self._client.close() + + async def __aenter__(self): + await self.connect() + return self + + async def __aexit__(self, *args): + await self.close() + + +__all__ = [ + "RealtimeException", + "Voice", + "AudioFormat", + "Modality", + "NoTurnDetection", + "ServerVAD", + "TurnDetection", + "FunctionToolChoice", + "ToolChoice", + "MessageRole", + "InputAudioTranscription", + "ClientMessageBase", + "Temperature", + "ToolsDefinition", + "SessionUpdateParams", + "SessionUpdateMessage", + "InputAudioBufferAppendMessage", + "InputAudioBufferCommitMessage", + "InputAudioBufferClearMessage", + "MessageItemType", + "InputTextContentPart", + "InputAudioContentPart", + "OutputTextContentPart", + "SystemContentPart", + "UserContentPart", + "AssistantContentPart", + "ItemParamStatus", + "SystemMessageItem", + "UserMessageItem", + "AssistantMessageItem", + "MessageItem", + "FunctionCallItem", + "FunctionCallOutputItem", + "Item", + "ItemCreateMessage", + "ItemTruncateMessage", + "ItemDeleteMessage", + "ResponseCreateParams", + "ResponseCreateMessage", + "ResponseCancelMessage", + "RealtimeError", + "ServerMessageBase", + "ErrorMessage", + "Session", + "SessionCreatedMessage", + "SessionUpdatedMessage", + "InputAudioBufferCommittedMessage", + "InputAudioBufferClearedMessage", + "InputAudioBufferSpeechStartedMessage", + "InputAudioBufferSpeechStoppedMessage", + "ResponseItemStatus", + "ResponseItemInputTextContentPart", + "ResponseItemInputAudioContentPart", + "ResponseItemTextContentPart", + "ResponseItemAudioContentPart", + "ResponseItemContentPart", + "ResponseItemBase", + "ResponseMessageItem", + "ResponseFunctionCallItem", + "ResponseFunctionCallOutputItem", + "ResponseItem", + "ItemCreatedMessage", + "ItemTruncatedMessage", + "ItemDeletedMessage", + "ItemInputAudioTranscriptionCompletedMessage", + "ItemInputAudioTranscriptionFailedMessage", + "ResponseStatus", + "ResponseCancelledDetails", + "ResponseIncompleteDetails", + "ResponseFailedDetails", + "ResponseStatusDetails", + "Usage", + "Response", + "ResponseCreatedMessage", + "ResponseDoneMessage", + "ResponseOutputItemAddedMessage", + "ResponseOutputItemDoneMessage", + "ResponseContentPartAddedMessage", + "ResponseContentPartDoneMessage", + "ResponseTextDeltaMessage", + "ResponseTextDoneMessage", + "ResponseAudioTranscriptDeltaMessage", + "ResponseAudioTranscriptDoneMessage", + "ResponseAudioDeltaMessage", + "ResponseAudioDoneMessage", + "ResponseFunctionCallArgumentsDeltaMessage", + "ResponseFunctionCallArgumentsDoneMessage", + "RateLimits", + "RateLimitsUpdatedMessage", + "UserMessageType", + "ServerMessageType", + "create_message_from_dict", +] diff --git a/src/python/rtclient/client_test.py b/src/python/rtclient/client_test.py new file mode 100644 index 0000000..902324d --- /dev/null +++ b/src/python/rtclient/client_test.py @@ -0,0 +1,424 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import json +import os +from collections.abc import AsyncGenerator, AsyncIterator, Callable, Generator +from pathlib import Path +from typing import Optional + +import numpy as np +import pytest +import soundfile as sf +from azure.core.credentials import AzureKeyCredential +from azure.identity.aio import DefaultAzureCredential +from dotenv import load_dotenv +from scipy.signal import resample + +from rtclient import RealtimeException, RTClient, RTInputAudioItem, RTResponse +from rtclient.models import InputAudioTranscription, InputTextContentPart, NoTurnDetection, ServerVAD, UserMessageItem + +load_dotenv() + +run_live_tests = os.getenv("LIVE_TESTS") == "true" + +openai_key = os.getenv("OPENAI_API_KEY") +openai_model = os.getenv("OPENAI_MODEL") + +azure_openai_endpoint = os.getenv("AZURE_OPENAI_ENDPOINT") +azure_openai_deployment = os.getenv("AZURE_OPENAI_DEPLOYMENT") + + +def should_run_openai_live_tests(): + return run_live_tests and openai_key is not None and openai_model is not None + + +def should_run_azure_openai_live_tests(): + return run_live_tests and azure_openai_endpoint is not None and azure_openai_deployment is not None + + +if not run_live_tests: + pytest.skip("Skipping live tests") + + +@pytest.fixture +def test_data_dir() -> str: + return os.path.join(Path(__file__).parent.parent, "test_data") + + +def resample_audio(audio_data, original_sample_rate, target_sample_rate): + number_of_samples = round(len(audio_data) * float(target_sample_rate) / original_sample_rate) + resampled_audio = resample(audio_data, number_of_samples) + return resampled_audio.astype(np.int16) + + +class AudioSamples: + def __init__(self, audio_file: str, sample_rate: int = 24000): + self._sample_rate = sample_rate + audio_data, original_sample_rate = sf.read(audio_file, dtype="int16") + + if original_sample_rate != sample_rate: + audio_data = resample_audio(audio_data, original_sample_rate, sample_rate) + self._audio_bytes = audio_data.tobytes() + + def chunks(self): + duration_ms = 100 + samples_per_chunk = self._sample_rate * (duration_ms / 1000) + bytes_per_sample = 2 + bytes_per_chunk = int(samples_per_chunk * bytes_per_sample) + for i in range(0, len(self._audio_bytes), bytes_per_chunk): + yield self._audio_bytes[i : i + bytes_per_chunk] + + +@pytest.fixture +def audio_samples(test_data_dir: str) -> AsyncIterator[bytes]: + samples = AudioSamples(os.path.join(test_data_dir, "1-tardigrades.wav")) + return samples.chunks() + + +@pytest.fixture +def audio_files(test_data_dir: str) -> Callable[[str], AsyncIterator[str]]: + def get_audio_file(file_name: str) -> AsyncIterator[str]: + samples = AudioSamples(os.path.join(test_data_dir, file_name)) + return samples.chunks() + + return get_audio_file + + +@pytest.fixture(params=["openai", "azure_openai"]) +async def client(request: pytest.FixtureRequest) -> AsyncGenerator[RTClient, None]: + if request.param == "openai" and should_run_openai_live_tests(): + async with ( + RTClient( + key_credential=AzureKeyCredential(openai_key), + model=openai_model, + ) as client, + ): + yield client + elif request.param == "azure_openai" and should_run_azure_openai_live_tests(): + async with ( + DefaultAzureCredential() as credential, + RTClient( + url=azure_openai_endpoint, azure_deployment=azure_openai_deployment, token_credential=credential + ) as client, + ): + yield client + else: + pytest.skip(f"Skipping {request.param} live tests") + + +@pytest.mark.asyncio +async def test_configure(client: RTClient): + original_session = client.session + assert original_session is not None + updated_session = await client.configure(instructions="You are a helpful assistant.") + assert updated_session is not None + + +@pytest.mark.asyncio +async def test_commit_audio(client: RTClient, audio_samples: Generator[bytes]): + await client.configure(turn_detection=NoTurnDetection()) + for chunk in audio_samples: + await client.send_audio(chunk) + item = await client.commit_audio() + await item + + +@pytest.mark.asyncio +async def test_commit_audio_with_transcription(client: RTClient, audio_samples: Generator[bytes]): + await client.configure( + turn_detection=NoTurnDetection(), input_audio_transcription=InputAudioTranscription(model="whisper-1") + ) + for chunk in audio_samples: + await client.send_audio(chunk) + item = await client.commit_audio() + assert item is not None + await item + assert item.transcript is not None + assert len(item.transcript) > 0 + + +@pytest.mark.asyncio +async def test_clear_audio(client: RTClient, audio_samples: Generator[bytes]): + await client.configure(turn_detection=NoTurnDetection()) + for chunk in audio_samples: + await client.send_audio(chunk) + await client.clear_audio() + + with pytest.raises(RealtimeException) as ex: + await client.commit_audio() + assert "buffer" in ex.value.message + + +@pytest.mark.asyncio +async def test_send_item(client: RTClient): + item = await client.send_item( + item=UserMessageItem(content=[InputTextContentPart(text="This is my first message!")]) + ) + assert item is not None + + +@pytest.mark.asyncio +async def test_remove_item(client: RTClient): + item = await client.send_item( + item=UserMessageItem(content=[InputTextContentPart(text="This is my first message!")]) + ) + assert item is not None + await client.remove_item(item_id=item.id) + + with pytest.raises(RealtimeException) as ex: + await client.send_item( + item=UserMessageItem(content=[InputTextContentPart(text="This is my second message!")]), + previous_item_id=item.id, + ) + assert item.id in ex.value.message + assert "does not exist" in ex.value.message + + +@pytest.mark.asyncio +async def test_generate_response(client: RTClient): + await client.configure(modalities={"text"}, turn_detection=NoTurnDetection()) + item = await client.send_item( + item=UserMessageItem( + content=[InputTextContentPart(text="Repeat exactly the following sentence: Hello, world!")] + ) + ) + response = await client.generate_response() + assert response is not None + assert response.id is not None + response_item = await anext(response) + assert response_item is not None + assert response_item.response_id == response.id + assert response_item.previous_id == item.id + + +@pytest.mark.asyncio +async def test_cancel_response(client: RTClient): + await client.configure(modalities={"text"}, turn_detection=NoTurnDetection()) + await client.send_item( + item=UserMessageItem( + content=[InputTextContentPart(text="Repeat exactly the following sentence: Hello, world!")] + ) + ) + response = await client.generate_response() + await response.cancel() + + with pytest.raises(StopAsyncIteration): + await anext(response) + + assert response.status in ["cancelled", "completed"] + + +@pytest.mark.asyncio +async def test_items_text_in_text_out(client: RTClient): + await client.configure(modalities={"text"}, turn_detection=NoTurnDetection()) + await client.send_item( + item=UserMessageItem( + content=[InputTextContentPart(text="Repeat exactly the following sentence: Hello, world!")] + ) + ) + response = await client.generate_response() + + item = await anext(response) + assert item.type == "message" + async for part in item: + text = "" + assert part.type == "text" + async for chunk in part.text_chunks(): + assert chunk is not None + text += chunk + assert part.text == text + + +@pytest.mark.asyncio +async def test_items_text_in_audio_out(client: RTClient): + await client.configure(modalities={"audio", "text"}, turn_detection=NoTurnDetection()) + await client.send_item( + item=UserMessageItem( + content=[InputTextContentPart(text="Repeat exactly the following sentence: Hello, world!")] + ) + ) + response = await client.generate_response() + + item = await anext(response) + assert item.type == "message" + async for part in item: + if part.type == "audio": + audio = b"" + async for chunk in part.audio_chunks(): + assert chunk is not None + audio += chunk + assert len(audio) > 0 + transcript = "" + async for chunk in part.transcript_chunks(): + assert chunk is not None + transcript += chunk + assert part.transcript == transcript + + +function_declarations = { + "get_weather_by_location": { + "name": "get_weather_by_location", + "type": "function", + "description": "A function to get the weather based on a location.", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string", "description": "The name of the city to get the weather for."}}, + "required": ["city"], + }, + }, +} + + +@pytest.mark.asyncio +async def test_items_text_in_function_call_out_chunks(client: RTClient): + await client.configure( + modalities={"text"}, + tools=[function_declarations["get_weather_by_location"]], + turn_detection=NoTurnDetection(), + ) + + await client.send_item( + item=UserMessageItem(content=[InputTextContentPart(text="What's the weather like in Seattle, Washington?")]) + ) + response = await client.generate_response() + + item = await anext(response) + assert item.type == "function_call" + assert item.function_name == "get_weather_by_location" + + arguments = "" + async for chunk in item: + assert chunk is not None + arguments += chunk + assert item.arguments == arguments + + +@pytest.mark.asyncio +async def test_items_text_in_function_call_out_await(client: RTClient): + await client.configure( + modalities={"text"}, + tools=[function_declarations["get_weather_by_location"]], + turn_detection=NoTurnDetection(), + ) + + await client.send_item( + item=UserMessageItem(content=[InputTextContentPart(text="What's the weather like in Seattle, Washington?")]) + ) + response = await client.generate_response() + + item = await anext(response) + assert item.type == "function_call" + assert item.function_name == "get_weather_by_location" + + await item + assert item.arguments is not None + assert len(item.arguments) > 0 + arguments = json.loads(item.arguments) + assert "city" in arguments + + +@pytest.mark.asyncio +async def test_function_call_fails_await_after_iter(client: RTClient): + await client.configure( + modalities={"text"}, + tools=[function_declarations["get_weather_by_location"]], + turn_detection=NoTurnDetection(), + ) + + await client.send_item( + item=UserMessageItem(content=[InputTextContentPart(text="What's the weather like in Seattle, Washington?")]) + ) + response = await client.generate_response() + item = await anext(response) + + async for _ in item: + pass + + with pytest.raises(RuntimeError) as ex: + await item + + assert "Cannot await after iterating" in ex.value.args[0] + + +@pytest.mark.asyncio +async def test_function_call_fails_iter_after_await(client: RTClient): + await client.configure( + modalities={"text"}, + tools=[function_declarations["get_weather_by_location"]], + turn_detection=NoTurnDetection(), + ) + + await client.send_item( + item=UserMessageItem(content=[InputTextContentPart(text="What's the weather like in Seattle, Washington?")]) + ) + response = await client.generate_response() + item = await anext(response) + + await item + + with pytest.raises(RuntimeError) as ex: + async for _ in item: + pass + + assert "Cannot iterate after awaiting" in ex.value.args[0] + + +@pytest.mark.asyncio +async def test_items_audio_in_text_out(client: RTClient, audio_files: Callable[[str], Generator[bytes]]): + audio_file = audio_files("1-tardigrades.wav") + await client.configure( + modalities={"text"}, + input_audio_transcription=InputAudioTranscription(model="whisper-1"), + turn_detection=NoTurnDetection(), + ) + for chunk in audio_file: + await client.send_audio(chunk) + await client.commit_audio() + response = await client.generate_response() + + item = await anext(response) + assert item.type == "message" + async for part in item: + text = "" + assert part.type == "text" + async for chunk in part.text_chunks(): + assert chunk is not None + text += chunk + assert part.text == text + + +@pytest.mark.asyncio +async def test_items_audio_in_text_out_with_vad(client: RTClient, audio_files: Callable[[str], Generator[bytes]]): + audio_samples = audio_files("1-tardigrades.wav") + await client.configure( + modalities={"text"}, + input_audio_transcription=InputAudioTranscription(model="whisper-1"), + turn_detection=ServerVAD(), + ) + for chunk in audio_samples: + await client.send_audio(chunk) + input_item: Optional[RTInputAudioItem] = None + response: Optional[RTResponse] = None + for _ in [1, 2]: + item = await anext(client.events()) + if item.type == "input_audio": + input_item = item + if item.type == "response": + response = item + + assert input_item is not None + await input_item + assert input_item.transcript is not None + assert len(input_item.transcript) > 0 + + assert response is not None + item = await anext(response) + assert item.type == "message" + async for part in item: + text = "" + assert part.type == "text" + async for chunk in part.text_chunks(): + assert chunk is not None + text += chunk + assert part.text == text diff --git a/src/python/rtclient/defaults.py b/src/python/rtclient/defaults.py new file mode 100644 index 0000000..c17b43f --- /dev/null +++ b/src/python/rtclient/defaults.py @@ -0,0 +1,18 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + + +from rtclient.models import AudioFormat, ServerVAD, TurnDetection, Voice + +DEFAULT_CONVERSATION: str = "default" +DEFAULT_TEMPERATURE: float = 0.6 +DEFAULT_VOICE: Voice = "alloy" +DEFAULT_AUDIO_FORMAT: AudioFormat = "pcm16" +DEFAULT_VAD_THRESHOLD: float = 0.5 +DEFAULT_VAD_PREFIX_PADDING_MS: int = 300 +DEFAULT_VAD_SILENCE_DURATION_MS: int = 200 +DEFAULT_TURN_DETECTION: TurnDetection = ServerVAD( + threshold=DEFAULT_VAD_THRESHOLD, + prefix_padding_ms=DEFAULT_VAD_PREFIX_PADDING_MS, + silence_duration_ms=DEFAULT_VAD_SILENCE_DURATION_MS, +) diff --git a/src/python/rtclient/low_level_client.py b/src/python/rtclient/low_level_client.py new file mode 100644 index 0000000..fc2ef31 --- /dev/null +++ b/src/python/rtclient/low_level_client.py @@ -0,0 +1,140 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import json +import os +import uuid +from collections.abc import AsyncIterator +from typing import Optional + +from aiohttp import ClientSession, WSMsgType, WSServerHandshakeError +from azure.core.credentials import AzureKeyCredential +from azure.core.credentials_async import AsyncTokenCredential + +from rtclient.models import ServerMessageType, UserMessageType, create_message_from_dict +from rtclient.util.user_agent import get_user_agent + + +class ConnectionError(Exception): + def __init__(self, message: str, headers=None): + super().__init__(message) + self.headers = headers + + pass + + +class RTLowLevelClient: + def __init__( + self, + url: Optional[str] = None, + token_credential: Optional[AsyncTokenCredential] = None, + key_credential: Optional[AzureKeyCredential] = None, + model: Optional[str] = None, + azure_deployment: Optional[str] = None, + ): + self._is_azure_openai = url is not None + if self._is_azure_openai: + if key_credential is None and token_credential is None: + raise ValueError("key_credential or token_credential is required for Azure OpenAI") + if azure_deployment is None: + raise ValueError("azure_deployment is required for Azure OpenAI") + else: + if key_credential is None: + raise ValueError("key_credential is required for OpenAI") + if model is None: + raise ValueError("model is required for OpenAI") + + self._url = url if self._is_azure_openai else "wss://api.openai.com" + self._token_credential = token_credential + self._key_credential = key_credential + self._session = ClientSession(base_url=self._url) + self._model = model + self._azure_deployment = azure_deployment + self.request_id: Optional[uuid.UUID] = None + + async def _get_auth(self): + if self._token_credential: + scope = "https://cognitiveservices.azure.com/.default" + token = await self._token_credential.get_token(scope) + return {"Authorization": f"Bearer {token.token}"} + elif self._key_credential: + return {"api-key": self._key_credential.key} + else: + return {} + + @staticmethod + def _get_azure_params(): + api_version = os.getenv("AZURE_OPENAI_API_VERSION") + path = os.getenv("AZURE_OPENAI_PATH") + return ( + "2024-10-01-preview" if api_version is None else api_version, + "/openai/realtime" if path is None else path, + ) + + async def connect(self): + try: + self.request_id = uuid.uuid4() + if self._is_azure_openai: + api_version, path = RTLowLevelClient._get_azure_params() + auth_headers = await self._get_auth() + headers = { + "x-ms-client-request-id": str(self.request_id), + "User-Agent": get_user_agent(), + **auth_headers, + } + self.ws = await self._session.ws_connect( + path, + headers=headers, + params={"deployment": self._azure_deployment, "api-version": api_version}, + ) + else: + headers = { + "Authorization": f"Bearer {self._key_credential.key}", + "openai-beta": "realtime=v1", + "User-Agent": get_user_agent(), + } + self.ws = await self._session.ws_connect("/v1/realtime", headers=headers, params={"model": self._model}) + except WSServerHandshakeError as e: + await self._session.close() + error_message = f"Received status code {e.status} from the server" + raise ConnectionError(error_message, e.headers) from e + + async def send(self, message: UserMessageType): + message._is_azure = self._is_azure_openai + message_json = message.model_dump_json(exclude_unset=True) + await self.ws.send_str(message_json) + + async def recv(self) -> ServerMessageType | None: + if self.ws.closed: + return None + websocket_message = await self.ws.receive() + if websocket_message.type == WSMsgType.TEXT: + data = json.loads(websocket_message.data) + msg = create_message_from_dict(data) + return msg + else: + return None + + def __aiter__(self) -> AsyncIterator[ServerMessageType | None]: + return self + + async def __anext__(self): + message = await self.recv() + if message is None: + raise StopAsyncIteration + return message + + async def close(self): + await self.ws.close() + await self._session.close() + + @property + def closed(self) -> bool: + return self.ws.closed + + async def __aenter__(self): + await self.connect() + return self + + async def __aexit__(self, *args): + await self.close() diff --git a/src/python/rtclient/models.py b/src/python/rtclient/models.py new file mode 100644 index 0000000..399f767 --- /dev/null +++ b/src/python/rtclient/models.py @@ -0,0 +1,706 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +from typing import Annotated, Any, Literal, Optional, Union + +from pydantic import ( + AliasChoices, + BaseModel, + Field, + SerializationInfo, + SerializerFunctionWrapHandler, + model_serializer, +) + +from rtclient.util.model_helpers import ModelWithDefaults + +Voice = Literal["alloy", "shimmer", "echo"] +AudioFormat = Literal["pcm16", "g711-ulaw", "g711-alaw"] +Modality = Literal["text", "audio"] + + +class NoTurnDetection(ModelWithDefaults): + type: Literal["none"] = "none" + + +class ServerVAD(ModelWithDefaults): + type: Literal["server_vad"] = "server_vad" + threshold: Optional[Annotated[float, Field(strict=True, ge=0.0, le=1.0)]] = None + prefix_padding_ms: Optional[int] = None + silence_duration_ms: Optional[int] = None + + +TurnDetection = Annotated[Union[NoTurnDetection, ServerVAD], Field(discriminator="type")] + + +class FunctionToolChoice(ModelWithDefaults): + type: Literal["function"] = "function" + function: str + + +ToolChoice = Literal["auto", "none", "required"] | FunctionToolChoice + +MessageRole = Literal["system", "assistant", "user"] + + +class InputAudioTranscription(BaseModel): + model: Literal["whisper-1"] + + +class ClientMessageBase(ModelWithDefaults): + _is_azure: bool = False + event_id: Optional[str] = None + + +Temperature = Annotated[float, Field(strict=True, ge=0.6, le=1.2)] +ToolsDefinition = list[Any] + +MaxTokensType = Union[int, Literal["inf"]] + + +class SessionUpdateParams(BaseModel): + model: Optional[str] = None + modalities: Optional[set[Modality]] = None + voice: Optional[Voice] = None + instructions: Optional[str] = None + input_audio_format: Optional[AudioFormat] = None + output_audio_format: Optional[AudioFormat] = None + input_audio_transcription: Optional[InputAudioTranscription] = None + turn_detection: Optional[TurnDetection] = None + tools: Optional[ToolsDefinition] = None + tool_choice: Optional[ToolChoice] = None + temperature: Optional[Temperature] = None + max_response_output_tokens: Optional[MaxTokensType] = None + + +class SessionUpdateMessage(ClientMessageBase): + """ + Update the session configuration. + """ + + type: Literal["session.update"] = "session.update" + session: SessionUpdateParams + + @model_serializer(mode="wrap") + def _azure_compatibility(self, next: SerializerFunctionWrapHandler, info: SerializationInfo): + serialized = next(self) + if not self._is_azure: + if ( + self.session is not None + and self.session.turn_detection is not None + and self.session.turn_detection.type == "none" + ): + serialized["session"]["turn_detection"] = None + + return serialized + + +class InputAudioBufferAppendMessage(ClientMessageBase): + """ + Append audio data to the user audio buffer, this should be in the format specified by + input_audio_format in the session config. + """ + + type: Literal["input_audio_buffer.append"] = "input_audio_buffer.append" + audio: str + + +class InputAudioBufferCommitMessage(ClientMessageBase): + """ + Commit the pending user audio buffer, which creates a user message item with the audio content + and clears the buffer. + """ + + type: Literal["input_audio_buffer.commit"] = "input_audio_buffer.commit" + + +class InputAudioBufferClearMessage(ClientMessageBase): + """ + Clear the user audio buffer, discarding any pending audio data. + """ + + type: Literal["input_audio_buffer.clear"] = "input_audio_buffer.clear" + + +MessageItemType = Literal["message"] + + +class InputTextContentPart(ModelWithDefaults): + type: Literal["input_text"] = "input_text" + text: str + + +class InputAudioContentPart(ModelWithDefaults): + type: Literal["input_audio"] = "input_audio" + audio: str + transcript: Optional[str] = None + + +class OutputTextContentPart(ModelWithDefaults): + type: Literal["text"] = "text" + text: str + + +SystemContentPart = InputTextContentPart +UserContentPart = Union[Annotated[Union[InputTextContentPart, InputAudioContentPart], Field(discriminator="type")]] +AssistantContentPart = OutputTextContentPart + +ItemParamStatus = Literal["completed", "incomplete"] + + +class SystemMessageItem(ModelWithDefaults): + type: MessageItemType = "message" + role: Literal["system"] = "system" + id: Optional[str] = None + content: list[SystemContentPart] + status: Optional[ItemParamStatus] = None + + +class UserMessageItem(ModelWithDefaults): + type: MessageItemType = "message" + role: Literal["user"] = "user" + id: Optional[str] = None + content: list[UserContentPart] + status: Optional[ItemParamStatus] = None + + +class AssistantMessageItem(ModelWithDefaults): + type: MessageItemType = "message" + role: Literal["assistant"] = "assistant" + id: Optional[str] = None + content: list[AssistantContentPart] + status: Optional[ItemParamStatus] = None + + +MessageItem = Annotated[Union[SystemMessageItem, UserMessageItem, AssistantMessageItem], Field(discriminator="role")] + + +class FunctionCallItem(ModelWithDefaults): + type: Literal["function_call"] = "function_call" + id: Optional[str] = None + name: str + call_id: str + arguments: str + status: Optional[ItemParamStatus] = None + + +class FunctionCallOutputItem(ModelWithDefaults): + type: Literal["function_call_output"] = "function_call_output" + id: Optional[str] = None + call_id: str + output: str + status: Optional[ItemParamStatus] = None + + +Item = Annotated[Union[MessageItem, FunctionCallItem, FunctionCallOutputItem], Field(discriminator="type")] + + +class ItemCreateMessage(ClientMessageBase): + type: Literal["conversation.item.create"] = "conversation.item.create" + previous_item_id: Optional[str] = None + item: Item + + +class ItemTruncateMessage(ClientMessageBase): + type: Literal["conversation.item.truncate"] = "conversation.item.truncate" + item_id: str + content_index: int + audio_end_ms: int + + +class ItemDeleteMessage(ClientMessageBase): + type: Literal["conversation.item.delete"] = "conversation.item.delete" + item_id: str + + +class ResponseCreateParams(BaseModel): + commit: bool = True + cancel_previous: bool = True + append_input_items: Optional[list[Item]] = None + input_items: Optional[list[Item]] = None + instructions: Optional[str] = None + modalities: Optional[set[Modality]] = None + voice: Optional[Voice] = None + temperature: Optional[Temperature] = None + max_output_tokens: Optional[MaxTokensType] = None + tools: Optional[ToolsDefinition] = None + tool_choice: Optional[ToolChoice] = None + output_audio_format: Optional[AudioFormat] = None + + +class ResponseCreateMessage(ClientMessageBase): + """ + Trigger model inference to generate a model turn. + """ + + type: Literal["response.create"] = "response.create" + response: Optional[ResponseCreateParams] = None + + +class ResponseCancelMessage(ClientMessageBase): + type: Literal["response.cancel"] = "response.cancel" + + +class RealtimeError(BaseModel): + message: str + type: Optional[str] = None + code: Optional[str] = None + param: Optional[str] = None + event_id: Optional[str] = None + + +class ServerMessageBase(BaseModel): + event_id: str + + +class ErrorMessage(ServerMessageBase): + type: Literal["error"] = "error" + error: RealtimeError + + +class Session(BaseModel): + id: str + model: str + modalities: set[Modality] + instructions: str + voice: Voice + input_audio_format: AudioFormat + output_audio_format: AudioFormat + input_audio_transcription: Optional[InputAudioTranscription] + turn_detection: Optional[TurnDetection] + tools: ToolsDefinition + tool_choice: ToolChoice + temperature: Temperature + max_response_output_tokens: Optional[MaxTokensType] + + +class SessionCreatedMessage(ServerMessageBase): + type: Literal["session.created"] = "session.created" + session: Session + + +class SessionUpdatedMessage(ServerMessageBase): + type: Literal["session.updated"] = "session.updated" + session: Session + + +class InputAudioBufferCommittedMessage(ServerMessageBase): + """ + Signals the server has received and processed the audio buffer. + """ + + type: Literal["input_audio_buffer.committed"] = "input_audio_buffer.committed" + previous_item_id: Optional[str] + item_id: str + + +class InputAudioBufferClearedMessage(ServerMessageBase): + """ + Signals the server has cleared the audio buffer. + """ + + type: Literal["input_audio_buffer.cleared"] = "input_audio_buffer.cleared" + + +class InputAudioBufferSpeechStartedMessage(ServerMessageBase): + """ + If the server VAD is enabled, this event is sent when speech is detected in the user audio buffer. + It tells you where in the audio stream (in milliseconds) the speech started, plus an item_id + which will be used in the corresponding speech_stopped event and the item created in the conversation + when speech stops. + """ + + type: Literal["input_audio_buffer.speech_started"] = "input_audio_buffer.speech_started" + audio_start_ms: int + item_id: str + + +class InputAudioBufferSpeechStoppedMessage(ServerMessageBase): + """ + If the server VAD is enabled, this event is sent when speech stops in the user audio buffer. + It tells you where in the audio stream (in milliseconds) the speech stopped, plus an item_id + which will be used in the corresponding speech_started event and the item created in the conversation + when speech starts. + """ + + type: Literal["input_audio_buffer.speech_stopped"] = "input_audio_buffer.speech_stopped" + audio_end_ms: int + item_id: str + + +ResponseItemStatus = Literal["in_progress", "completed", "incomplete"] + + +class ResponseItemInputTextContentPart(BaseModel): + type: Literal["input_text"] = "input_text" + text: str + + +class ResponseItemInputAudioContentPart(BaseModel): + type: Literal["input_audio"] = "input_audio" + transcript: Optional[str] + + +class ResponseItemTextContentPart(BaseModel): + type: Literal["text"] = "text" + text: str + + +class ResponseItemAudioContentPart(BaseModel): + type: Literal["audio"] = "audio" + transcript: Optional[str] + + +ResponseItemContentPart = Annotated[ + Union[ + ResponseItemInputTextContentPart, + ResponseItemInputAudioContentPart, + ResponseItemTextContentPart, + ResponseItemAudioContentPart, + ], + Field(discriminator="type"), +] + + +class ResponseItemBase(BaseModel): + id: Optional[str] + + +class ResponseMessageItem(ResponseItemBase): + type: MessageItemType = "message" + status: ResponseItemStatus + role: MessageRole + content: list[ResponseItemContentPart] + + +class ResponseFunctionCallItem(ResponseItemBase): + type: Literal["function_call"] = "function_call" + status: ResponseItemStatus + name: str + call_id: str + arguments: str + + +class ResponseFunctionCallOutputItem(ResponseItemBase): + type: Literal["function_call_output"] = "function_call_output" + call_id: str + output: str + + +ResponseItem = Annotated[ + Union[ResponseMessageItem, ResponseFunctionCallItem, ResponseFunctionCallOutputItem], + Field(discriminator="type"), +] + + +class ItemCreatedMessage(ServerMessageBase): + type: Literal["conversation.item.created"] = "conversation.item.created" + previous_item_id: Optional[str] + item: ResponseItem + + +class ItemTruncatedMessage(ServerMessageBase): + type: Literal["conversation.item.truncated"] = "conversation.item.truncated" + item_id: str + content_index: int + audio_end_ms: int + + +class ItemDeletedMessage(ServerMessageBase): + type: Literal["conversation.item.deleted"] = "conversation.item.deleted" + item_id: str + + +class ItemInputAudioTranscriptionCompletedMessage(ServerMessageBase): + type: Literal["conversation.item.input_audio_transcription.completed"] = ( + "conversation.item.input_audio_transcription.completed" + ) + item_id: str + content_index: int + transcript: str + + +class ItemInputAudioTranscriptionFailedMessage(ServerMessageBase): + type: Literal["conversation.item.input_audio_transcription.failed"] = ( + "conversation.item.input_audio_transcription.failed" + ) + item_id: str + content_index: int + error: RealtimeError + + +ResponseStatus = Literal["in_progress", "completed", "cancelled", "incomplete", "failed"] + + +class ResponseCancelledDetails(BaseModel): + type: Literal["cancelled"] = "cancelled" + reason: Literal["turn_detected", "client_cancelled"] + + +class ResponseIncompleteDetails(BaseModel): + type: Literal["incomplete"] = "incomplete" + reason: Literal["max_output_tokens", "content_filter"] + + +class ResponseFailedDetails(BaseModel): + type: Literal["failed"] = "failed" + error: Any + + +ResponseStatusDetails = Annotated[ + Union[ResponseCancelledDetails, ResponseIncompleteDetails, ResponseFailedDetails], + Field(discriminator="type"), +] + + +class Usage(BaseModel): + total_tokens: int + input_tokens: int + output_tokens: int + + +class Response(BaseModel): + id: str + status: ResponseStatus + status_details: Optional[ResponseStatusDetails] + output: list[ResponseItem] + usage: Optional[Usage] + + +class ResponseCreatedMessage(ServerMessageBase): + type: Literal["response.created"] = "response.created" + response: Response + + +class ResponseDoneMessage(ServerMessageBase): + type: Literal["response.done"] = "response.done" + response: Response + + +class ResponseOutputItemAddedMessage(ServerMessageBase): + type: Literal["response.output_item.added"] = "response.output_item.added" + response_id: str + output_index: int + item: ResponseItem + + +class ResponseOutputItemDoneMessage(ServerMessageBase): + type: Literal["response.output_item.done"] = "response.output_item.done" + response_id: str + output_index: int + item: ResponseItem + + +class ResponseContentPartAddedMessage(ServerMessageBase): + type: Literal["response.content_part.added"] = "response.content_part.added" + response_id: str + item_id: str + output_index: int + content_index: int + part: Annotated[ + ResponseItemContentPart, Field(alias="part", validation_alias=AliasChoices("part", "content")) + ] # TODO: this alias won't be needed when AOAI and OAI are in sync. + + +class ResponseContentPartDoneMessage(ServerMessageBase): + type: Literal["response.content_part.done"] = "response.content_part.done" + response_id: str + item_id: str + output_index: int + content_index: int + part: Annotated[ + ResponseItemContentPart, Field(alias="part", validation_alias=AliasChoices("part", "content")) + ] # TODO: this alias won't be needed when AOAI and OAI are in sync. + + +class ResponseTextDeltaMessage(ServerMessageBase): + type: Literal["response.text.delta"] = "response.text.delta" + response_id: str + item_id: str + output_index: int + content_index: int + delta: str + + +class ResponseTextDoneMessage(ServerMessageBase): + type: Literal["response.text.done"] = "response.text.done" + response_id: str + item_id: str + output_index: int + content_index: int + text: str + + +class ResponseAudioTranscriptDeltaMessage(ServerMessageBase): + type: Literal["response.audio_transcript.delta"] = "response.audio_transcript.delta" + response_id: str + item_id: str + output_index: int + content_index: int + delta: str + + +class ResponseAudioTranscriptDoneMessage(ServerMessageBase): + type: Literal["response.audio_transcript.done"] = "response.audio_transcript.done" + response_id: str + item_id: str + output_index: int + content_index: int + transcript: str + + +class ResponseAudioDeltaMessage(ServerMessageBase): + type: Literal["response.audio.delta"] = "response.audio.delta" + response_id: str + item_id: str + output_index: int + content_index: int + delta: str + + +class ResponseAudioDoneMessage(ServerMessageBase): + type: Literal["response.audio.done"] = "response.audio.done" + response_id: str + item_id: str + output_index: int + content_index: int + + +class ResponseFunctionCallArgumentsDeltaMessage(ServerMessageBase): + type: Literal["response.function_call_arguments.delta"] = "response.function_call_arguments.delta" + response_id: str + item_id: str + output_index: int + call_id: str + delta: str + + +class ResponseFunctionCallArgumentsDoneMessage(ServerMessageBase): + type: Literal["response.function_call_arguments.done"] = "response.function_call_arguments.done" + response_id: str + item_id: str + output_index: int + call_id: str + name: str + arguments: str + + +class RateLimits(BaseModel): + name: str + limit: int + remaining: int + reset_seconds: float + + +class RateLimitsUpdatedMessage(ServerMessageBase): + type: Literal["rate_limits.updated"] = "rate_limits.updated" + rate_limits: list[RateLimits] + + +UserMessageType = Annotated[ + Union[ + SessionUpdateMessage, + InputAudioBufferAppendMessage, + InputAudioBufferCommitMessage, + InputAudioBufferClearMessage, + ItemCreateMessage, + ItemTruncateMessage, + ItemDeleteMessage, + ResponseCreateMessage, + ResponseCancelMessage, + ], + Field(discriminator="type"), +] +ServerMessageType = Annotated[ + Union[ + ErrorMessage, + SessionCreatedMessage, + SessionUpdatedMessage, + InputAudioBufferCommittedMessage, + InputAudioBufferClearedMessage, + InputAudioBufferSpeechStartedMessage, + InputAudioBufferSpeechStoppedMessage, + ItemCreatedMessage, + ItemTruncatedMessage, + ItemDeletedMessage, + ItemInputAudioTranscriptionCompletedMessage, + ItemInputAudioTranscriptionFailedMessage, + ResponseCreatedMessage, + ResponseDoneMessage, + ResponseOutputItemAddedMessage, + ResponseOutputItemDoneMessage, + ResponseContentPartAddedMessage, + ResponseContentPartDoneMessage, + ResponseTextDeltaMessage, + ResponseTextDoneMessage, + ResponseAudioTranscriptDeltaMessage, + ResponseAudioTranscriptDoneMessage, + ResponseAudioDeltaMessage, + ResponseAudioDoneMessage, + ResponseFunctionCallArgumentsDeltaMessage, + ResponseFunctionCallArgumentsDoneMessage, + RateLimitsUpdatedMessage, + ], + Field(discriminator="type"), +] + + +def create_message_from_dict(data: dict) -> ServerMessageType: + event_type = data.get("type") + match event_type: + case "error": + return ErrorMessage(**data) + case "session.created": + return SessionCreatedMessage(**data) + case "session.updated": + return SessionUpdatedMessage(**data) + case "input_audio_buffer.committed": + return InputAudioBufferCommittedMessage(**data) + case "input_audio_buffer.cleared": + return InputAudioBufferClearedMessage(**data) + case "input_audio_buffer.speech_started": + return InputAudioBufferSpeechStartedMessage(**data) + case "input_audio_buffer.speech_stopped": + return InputAudioBufferSpeechStoppedMessage(**data) + case "conversation.item.created": + return ItemCreatedMessage(**data) + case "conversation.item.truncated": + return ItemTruncatedMessage(**data) + case "conversation.item.deleted": + return ItemDeletedMessage(**data) + case "conversation.item.input_audio_transcription.completed": + return ItemInputAudioTranscriptionCompletedMessage(**data) + case "conversation.item.input_audio_transcription.failed": + return ItemInputAudioTranscriptionFailedMessage(**data) + case "response.created": + return ResponseCreatedMessage(**data) + case "response.done": + return ResponseDoneMessage(**data) + case "response.output_item.added": + return ResponseOutputItemAddedMessage(**data) + case "response.output_item.done": + return ResponseOutputItemDoneMessage(**data) + case "response.content_part.added": + return ResponseContentPartAddedMessage(**data) + case "response.content_part.done": + return ResponseContentPartDoneMessage(**data) + case "response.text.delta": + return ResponseTextDeltaMessage(**data) + case "response.text.done": + return ResponseTextDoneMessage(**data) + case "response.audio_transcript.delta": + return ResponseAudioTranscriptDeltaMessage(**data) + case "response.audio_transcript.done": + return ResponseAudioTranscriptDoneMessage(**data) + case "response.audio.delta": + return ResponseAudioDeltaMessage(**data) + case "response.audio.done": + return ResponseAudioDoneMessage(**data) + case "response.function_call_arguments.delta": + return ResponseFunctionCallArgumentsDeltaMessage(**data) + case "response.function_call_arguments.done": + return ResponseFunctionCallArgumentsDoneMessage(**data) + case "rate_limits.updated": + return RateLimitsUpdatedMessage(**data) + case _: + raise ValueError(f"Unknown event type: {event_type}") diff --git a/src/python/rtclient/util/id_generator.py b/src/python/rtclient/util/id_generator.py new file mode 100644 index 0000000..386eb11 --- /dev/null +++ b/src/python/rtclient/util/id_generator.py @@ -0,0 +1,9 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import secrets + + +def generate_id(prefix: str) -> str: + suffix = secrets.token_urlsafe(24) + return f"{prefix}-{suffix[len(prefix) + 1:]}" diff --git a/src/python/rtclient/util/id_generator_test.py b/src/python/rtclient/util/id_generator_test.py new file mode 100644 index 0000000..e64a421 --- /dev/null +++ b/src/python/rtclient/util/id_generator_test.py @@ -0,0 +1,11 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from id_generator import generate_id + + +def test_id_schema(): + prefixes = ["test", "sh", "longer_prefix"] + for prefix in prefixes: + id = generate_id(prefix) + assert len(id) == 32 diff --git a/src/python/rtclient/util/message_queue.py b/src/python/rtclient/util/message_queue.py new file mode 100644 index 0000000..0b41d3c --- /dev/null +++ b/src/python/rtclient/util/message_queue.py @@ -0,0 +1,104 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import asyncio +from collections.abc import Awaitable, Callable +from typing import Generic, Optional, TypeVar + +T = TypeVar("T") + + +class MessageQueue(Generic[T]): + def __init__(self, receive_delegate: Callable[[], Awaitable[T]]): + self._stored_messages: list[T] = [] + self.waiting_receivers: list[tuple[Callable[[T], bool], asyncio.Future]] = [] + self.is_polling: bool = False + self.receive_delegate = receive_delegate + self.poll_task: Optional[asyncio.Task] = None + + def _push_back(self, message: T): + self._stored_messages.append(message) + + def _find_and_remove(self, predicate: Callable[[T], bool]) -> Optional[T]: + for i, message in enumerate(self._stored_messages): + if predicate(message): + return self._stored_messages.pop(i) + return None + + async def _poll_receive(self): + if self.is_polling: + return + + try: + self.is_polling = True + while self.is_polling: + message = await self.receive_delegate() + if message is None: + self._notify_end_of_stream() + break + self._notify_receiver(message) + if len(self.waiting_receivers) == 0: + break + except Exception as error: + self._notify_exception(error) + finally: + self.is_polling = False + self.poll_task = None + + def _notify_exception(self, error: Exception): + for _, future in self.waiting_receivers: + if not future.done(): + future.set_exception(error) + self.waiting_receivers.clear() + + def _notify_end_of_stream(self): + for _, future in self.waiting_receivers: + if not future.done(): + future.set_result(None) + self.waiting_receivers.clear() + + def _notify_receiver(self, message: T): + for i, (predicate, future) in enumerate(self.waiting_receivers): + if predicate(message): + del self.waiting_receivers[i] + future.set_result(message) + return + self._push_back(message) + + def queued_messages_count(self) -> int: + return len(self._stored_messages) + + async def receive(self, predicate: Callable[[T], bool]) -> Optional[T]: + found_message = self._find_and_remove(predicate) + if found_message is not None: + return found_message + + future = asyncio.Future() + self.waiting_receivers.append((predicate, future)) + + if not self.is_polling and self.poll_task is None: + self.poll_task = asyncio.create_task(self._poll_receive()) + + return await future + + +class MessageQueueWithError(MessageQueue[T]): + def __init__(self, receive_delegate: Callable[[], Awaitable[T]], error_predicate: Callable[[T], bool]): + super().__init__(receive_delegate) + self._error_predicate = error_predicate + self._error: Optional[T] = None + + def _notify_error(self, error: T): + for _, future in self.waiting_receivers: + if not future.done(): + future.set_result(error) + self.waiting_receivers.clear() + + async def receive(self, predicate) -> Optional[T]: + if self._error is not None: + return self._error + message = await super().receive(lambda m: predicate(m) or self._error_predicate(m)) + if message is not None and self._error_predicate(message): + self._error = message + self._notify_error(message) + return message diff --git a/src/python/rtclient/util/message_queue_test.py b/src/python/rtclient/util/message_queue_test.py new file mode 100644 index 0000000..764911c --- /dev/null +++ b/src/python/rtclient/util/message_queue_test.py @@ -0,0 +1,200 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import asyncio + +import pytest +from message_queue import MessageQueue + + +class Message: + def __init__(self, id: str, content: str): + self.id = id + self.content = content + + +@pytest.fixture +def message_queue(): + async def receive_delegate(): + await asyncio.sleep(0.1) + return None + + return MessageQueue(receive_delegate) + + +@pytest.mark.asyncio +async def test_receive_existing_message(message_queue): + message = Message("1", "Hello") + message_queue._push_back(message) + + result = await message_queue.receive(lambda m: m.id == "1") + assert result == message + assert message_queue.queued_messages_count() == 0 + + +@pytest.mark.asyncio +async def test_receive_non_existing_message(message_queue): + messages = [Message("2", "World")] + message_queue.receive_delegate = lambda: asyncio.sleep(0, messages.pop(0) if messages else None) + + result = await message_queue.receive(lambda m: m.id == "2") + assert isinstance(result, Message) + assert result.id == "2" + assert result.content == "World" + + +@pytest.mark.asyncio +async def test_receive_multiple_messages(message_queue): + messages = [Message("1", "First"), Message("2", "Second"), Message("3", "Third")] + for message in messages: + message_queue._push_back(message) + + result1 = await message_queue.receive(lambda m: m.id == "2") + result2 = await message_queue.receive(lambda m: m.id == "1") + result3 = await message_queue.receive(lambda m: m.id == "3") + + assert result1 == messages[1] + assert result2 == messages[0] + assert result3 == messages[2] + assert message_queue.queued_messages_count() == 0 + + +@pytest.mark.asyncio +async def test_receive_end_of_stream(message_queue): + result = await message_queue.receive(lambda m: True) + assert result is None + + +@pytest.mark.asyncio +async def test_receive_with_error(message_queue): + async def receive_delegate(): + raise Exception("Test error") + + message_queue.receive_delegate = receive_delegate + + with pytest.raises(Exception, match="Test error"): + await message_queue.receive(lambda m: True) + + +@pytest.mark.asyncio +async def test_multiple_receivers_same_predicate(message_queue): + messages = [Message("1", "Shared")] + message_queue.receive_delegate = lambda: asyncio.sleep(0, messages.pop(0) if messages else None) + + task1 = asyncio.create_task(message_queue.receive(lambda m: m.id == "1")) + task2 = asyncio.create_task(message_queue.receive(lambda m: m.id == "1")) + + result1, result2 = await asyncio.gather(task1, task2) + + assert result1.content == "Shared" + assert result2 is None + + +@pytest.mark.asyncio +async def test_polling_mechanism(message_queue): + messages = [Message("2", "Second"), Message("1", "First"), Message("3", "Third")] + + async def delayed_receive_delegate(): + await asyncio.sleep(0.1) + return messages.pop(0) if messages else None + + message_queue.receive_delegate = delayed_receive_delegate + + task1 = asyncio.create_task(message_queue.receive(lambda m: m.id == "1")) + task2 = asyncio.create_task(message_queue.receive(lambda m: m.id == "2")) + task3 = asyncio.create_task(message_queue.receive(lambda m: m.id == "3")) + + results = await asyncio.gather(task1, task2, task3) + + assert [msg.content for msg in results if msg] == ["First", "Second", "Third"] + assert not message_queue.is_polling + assert message_queue.poll_task is None + + +@pytest.mark.asyncio +async def test_polling_stops_when_no_receivers(message_queue): + messages = [Message("1", "First"), Message("2", "Second")] + + async def delayed_receive_delegate(): + await asyncio.sleep(0.1) + return messages.pop(0) if messages else None + + message_queue.receive_delegate = delayed_receive_delegate + + result1 = await message_queue.receive(lambda m: m.id == "1") + assert result1.content == "First" + assert not message_queue.is_polling + assert message_queue.poll_task is None + + result2 = await message_queue.receive(lambda m: m.id == "2") + assert result2.content == "Second" + assert not message_queue.is_polling + assert message_queue.poll_task is None + + +@pytest.mark.asyncio +async def test_concurrent_receive_calls(message_queue): + messages = [Message("1", "First"), Message("2", "Second"), Message("3", "Third")] + + async def delayed_receive_delegate(): + await asyncio.sleep(0.1) + return messages.pop(0) if messages else None + + message_queue.receive_delegate = delayed_receive_delegate + + tasks = [ + asyncio.create_task(message_queue.receive(lambda m: m.id == "1")), + asyncio.create_task(message_queue.receive(lambda m: m.id == "2")), + asyncio.create_task(message_queue.receive(lambda m: m.id == "3")), + asyncio.create_task(message_queue.receive(lambda m: m.id == "4")), + ] + + results = await asyncio.gather(*tasks) + + assert [msg.content if msg else None for msg in results] == ["First", "Second", "Third", None] + assert not message_queue.is_polling + assert message_queue.poll_task is None + + +@pytest.mark.asyncio +async def test_receive_with_complex_predicate(message_queue): + messages = [ + Message("1", "Apple"), + Message("2", "Banana"), + Message("3", "Cherry"), + Message("4", "Date"), + ] + for message in messages: + message_queue._push_back(message) + + result = await message_queue.receive(lambda m: m.id in ["2", "4"] and len(m.content) > 5) + assert result.id == "2" + assert result.content == "Banana" + + result = await message_queue.receive(lambda m: m.content.startswith("C")) + assert result.id == "3" + assert result.content == "Cherry" + + +@pytest.mark.asyncio +async def test_receive_with_always_true_predicate(message_queue): + messages = [Message("1", "First"), Message("2", "Second")] + for message in messages: + message_queue._push_back(message) + + result1 = await message_queue.receive(lambda m: True) + result2 = await message_queue.receive(lambda m: True) + + assert result1.id == "1" + assert result2.id == "2" + assert message_queue.queued_messages_count() == 0 + + +@pytest.mark.asyncio +async def test_receive_with_always_false_predicate(message_queue): + messages = [Message("1", "First"), Message("2", "Second")] + message_queue.receive_delegate = lambda: asyncio.sleep(0, messages.pop(0) if messages else None) + + result = await asyncio.wait_for(message_queue.receive(lambda m: False), timeout=0.5) + assert result is None + assert message_queue.queued_messages_count() == 2 diff --git a/src/python/rtclient/util/model_helpers.py b/src/python/rtclient/util/model_helpers.py new file mode 100644 index 0000000..679b66f --- /dev/null +++ b/src/python/rtclient/util/model_helpers.py @@ -0,0 +1,14 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from pydantic import BaseModel, model_validator + + +class ModelWithDefaults(BaseModel): + @model_validator(mode="after") + def _add_defaults(self): + for field in self.model_fields: + if self.model_fields[field].default is not None: + if not hasattr(self, field) or getattr(self, field) == self.model_fields[field].default: + setattr(self, field, self.model_fields[field].default) + return self diff --git a/src/python/rtclient/util/model_helpers_test.py b/src/python/rtclient/util/model_helpers_test.py new file mode 100644 index 0000000..d09f410 --- /dev/null +++ b/src/python/rtclient/util/model_helpers_test.py @@ -0,0 +1,23 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from typing import Optional + +from model_helpers import ModelWithDefaults + + +class Bar(ModelWithDefaults): + foo: Optional[int] = None + bar: Optional[float] = 3.14 + baz: int = 42 + + +def test_with_defaults(): + instance = Bar() + assert instance.foo is None + assert instance.baz == 42 + + +def test_serialize_with_defaults(): + instance = Bar() + assert instance.model_dump(exclude_unset=True) == {"bar": 3.14, "baz": 42} diff --git a/src/python/rtclient/util/user_agent.py b/src/python/rtclient/util/user_agent.py new file mode 100644 index 0000000..0994218 --- /dev/null +++ b/src/python/rtclient/util/user_agent.py @@ -0,0 +1,11 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import platform +from importlib.metadata import version + + +def get_user_agent(): + package_version = version("rtclient") + python_version = platform.python_version() + return f"ms-rtclient/{package_version} Python/{python_version}" diff --git a/src/python/rtclient/util/user_agent_test.py b/src/python/rtclient/util/user_agent_test.py new file mode 100644 index 0000000..76244ab --- /dev/null +++ b/src/python/rtclient/util/user_agent_test.py @@ -0,0 +1,12 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import re + + +def test_user_agent_schema(): + from rtclient.util.user_agent import get_user_agent + + user_agent = get_user_agent() + regex = re.compile("ms-rtclient/\d+\.\d+\.\d+ Python/\d+\.\d+\.\d+") + assert regex.match(user_agent) is not None From 0d74545209f02f9b14cc67a34094fdcedb3b03f3 Mon Sep 17 00:00:00 2001 From: Andras Horompo Date: Mon, 25 Nov 2024 12:14:38 +0100 Subject: [PATCH 2/4] Main, readme, requirements + speaker_output files added --- src/python/README.md | 116 +++++++++++++++++++++++++++++++++++ src/python/example.jpg | Bin 0 -> 47574 bytes src/python/main.py | 66 ++++++++++++++++++++ src/python/requirements.txt | 8 +++ src/python/speaker_output.py | 53 ++++++++++++++++ 5 files changed, 243 insertions(+) create mode 100644 src/python/README.md create mode 100644 src/python/example.jpg create mode 100644 src/python/main.py create mode 100644 src/python/requirements.txt create mode 100644 src/python/speaker_output.py diff --git a/src/python/README.md b/src/python/README.md new file mode 100644 index 0000000..1421c66 --- /dev/null +++ b/src/python/README.md @@ -0,0 +1,116 @@ +# RxAI: Real-Time AI Conversation with Python and PyRx + +RxAI is an advanced real-time AI conversation system developed using Python and PyRx. It integrates cutting-edge AI capabilities to facilitate live, interactive dialogues. The system is designed to leverage the power of OpenAI's language models in combination with reactive programming, enabling fluid and context-aware conversations. + +--- + +## Features + +RxAI comes equipped with several powerful features that make it a versatile solution for real-time interactions: + +- **Real-time AI conversation**: RxAI processes and responds to user inputs instantly, creating a seamless conversational experience similar to speaking with a human. +- **Function calling**: The system supports dynamic function calling, allowing it to execute specific actions during a conversation based on user intent. For example, it can initiate system commands or trigger other functionalities. +- **Reactive Python implementation**: RxAI leverages reactive programming paradigms using RxPY, making it highly responsive to changes in user input, updates, or any other asynchronous event. This ensures smooth handling of multiple streams of information in real time. + +--- + +## Screenshots + +![Screenshot about a joke and calling 'goodbye' function.](example.jpg) + +--- + +## Project Structure + +- `rtclient`: Provided by Microsoft ([link](https://github.com/Azure-Samples/aoai-realtime-audio-sdk/tree/6779885d3aaa2ddbed2bbc5dbba74da8cddffca1/python/rtclient)), this package offers low-level components to support real-time AI conversations. It includes: + - **Session Handling**: Manages conversation sessions, including starting, updating, and ending sessions. + - **Message Management**: Defines and handles various message types exchanged during conversations. + - **Testing Utilities**: Includes `client_test.py` to ensure correct behavior of the real-time client. + - **Model Definitions**: Provides models such as `SessionUpdateMessage` to represent various data structures used in the system. + +- `conversation_client.py`: Manages the conversation using `RTClient` and `RxPY` to facilitate real-time AI interactions. It handles: + - **Session Initialization**: Configures conversation parameters such as turn detection and input transcription. + - **Event Handling**: Processes various types of events including audio input and AI responses. + - **Function Calls**: Manages function calls initiated during conversation, and notifies when function calls start and finish. + - **Subscriptions**: Allows external subscriptions to events like transcription updates, audio updates, and error notifications. + +- `microphone_stream.py`: Captures microphone input for real-time audio processing, providing an audio stream for interaction with the AI. + +- `speaker_output.py`: Manages audio playback for the AI's response. It includes: + - **Audio Playback**: Buffers and plays back audio data. + - **Clearing Playback**: Clears the audio buffer and stops playback as needed. + - **Dispose Method**: Closes the audio stream to release resources when no longer needed. + +- `main.py`: The main script that initializes and runs the conversation client, serving as the entry point for the project. It handles the overall flow of the AI conversation. + +- `main.ipynb`: A Jupyter Notebook version of the AI conversation demo, useful for interactive exploration, debugging, or showcasing features step-by-step. + +--- + +## Installation + +1. **Create a virtual environment (e.g., `your_venv`)**: + + ```bash + python -m venv your_venv + ``` + +2. **Activate the virtual environment**: + + - **Windows**: `your_venv/Scripts/activate` + - **Linux/Mac**: `source your_venv/bin/activate` + +3. **Install requirements**: + + ```bash + pip install -r requirements.txt + ``` + +4. **Install `rtclient`**: + + ```bash + pip install -e . + ``` + +5. **Set API Key** + + Set `OPENAI_API_KEY` in the `.env` file. + +--- + +## Running the Demo + +- Run `main.py` with Python: + ```bash + python main.py + ``` +- Alternatively, open and run `main.ipynb` in Jupyter Notebook. + +--- + +## Tips + +- To exit the conversation, simply say **"Goodbye"**. This triggers function calling to quit the application. +- Avoid feedback loops by muting your speakers or using headphones to prevent the AI's output from being picked up by the microphone. + +--- + +## How It Works + +RxAI operates by integrating real-time audio input with advanced language models to facilitate dynamic and context-aware conversations. Here's a step-by-step overview of how the system works: + +1. **Audio Capture**: The `microphone_stream.py` module captures audio input from the user in real time. The audio is streamed to the `conversation_client.py` for processing. + +2. **Audio Processing and Transcription**: Using `RTClient`, the captured audio is converted into text transcriptions. These transcriptions are managed through reactive streams using RxPY, enabling efficient handling of updates and modifications. + +3. **Real-Time AI Interaction**: Once the transcription is available, the input is passed to OpenAI's language model via the `conversation_client.py`. The AI processes the input to generate a response based on the current context of the conversation. + +4. **Function Calling**: Depending on user intent, the system may identify the need to trigger specific actions. This is managed through dynamic function calling, where certain commands are executed automatically during the conversation. Examples include ending the conversation or interfacing with external systems. + +5. **Audio Response Playback**: The generated AI response is either played back as audio using the `speaker_output.py` module or displayed as text. This ensures a seamless and interactive experience for the user. + +6. **Reactive Updates**: The entire workflow leverages reactive programming to handle changes and updates in real time. This means that as new input or events occur, the system adapts instantly, making the conversation fluid and natural. + +7. **Session Management**: The `rtclient` module provided by Microsoft helps manage the session lifecycle, handling configurations, event subscriptions, and maintaining consistent communication throughout the conversation. + +Overall, RxAI combines advanced language processing with real-time input/output handling and reactive programming to create a highly interactive and responsive conversational experience. diff --git a/src/python/example.jpg b/src/python/example.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e4b5caa378030fb6a5453a5038a0b5c09c7e361b GIT binary patch literal 47574 zcmdSAcT`i+7bY5{Ne#U>=}HsnMIs_iM5IV>QiRY#CqO9DdlgiA??pP&dlBh9NJ&CR zAOV3W#mVo@d-K+M|IB-D%~~^abI)J7IoaPm=j^lhx4-jm<=+;7>ZO{d8UPOu4*=Y-FMkH-zTM~#P1 zjrVT|zzG205#p@hhU11(g5dwBTx_XzO+V>RySAl&x=0%}5< z2ja>^w7TzzxjpEfd`K<4&-1*YpI&bk%`0K!8Ad|Nz{teR!pAQl`0$aWl(dYjoV>~l zRW)@D&6oNHZw-x%O-ya=?B6>$f}Om)eSH0({sG|;kx|hxv2khXA2TvPeg2YFR9sS8 zR$ftA)!5YB(%RPktz%$tXn16FZ2bG&{KDeW^2#b4fkbWZ?C$L!9AeJSe_vc)VgLO7 z4=y|a{(pt_-;n)3a8cuM-6J5tCm{Y0F1&kwxQ0(nK=?qMh(=kL_?-tW_mdCz>7J(+ zHuRJ5Na&&IZ9Hd58F(cTe3<`0`#+HVzXKNb{|niF1NML8f&<9$@o5#l5Z1huXk~!mEx+=VWwjo3Lz`>(7hmDsS-)5`N*^aK)6A4rRH zXXWqsiKEJ$smsV+hl~ZhGuB|aKbqX3O7=zFVbg;$(*FQWb#0Xw7CF zA+0rhDWWShCfo+AXE*)K(IR19CLEMl)=Fso;NRCxUiFT_EDcLyQ`u#$4Z?jJuP9j^ z;~q*^@ZOtlwo7uvpRA~VjbZF#hYY%+O4AMI>ZZekr=-Lll$$qC=^Bdj3blvH3E&e@ znbtzRE?qI8I4K~p$?C>|kW5_jp9@w_d4D=y4+qaNSZ_v>?1*|L9p3hDqY`NRR?uA} zBCwuH87YEqJ}(TJ*Iuxn+so2vc!lEecl;=SxlWxzLrGE#iD#jhN=Nth=@&tz&(-sf zb>YrRY>n+vXXufBt7C0NkERqTr&uk(pMw5a==u>hX45gi3mZS^;Vgd~x**vtQdcgq zK4NqJSe(Sb#a{jM7zr^!sJ2%y1D2-}Hxig5zrRz-2-hdBFTjTLM|!W6=Dlt-!k9|9 z#F%|ai=PI}5a;SzI1B@$jq*uh;6Oy8+TG}z1>x5C==qy@772=eFJ4E@eqt`_C?@`v z9{d}zXyiY@kTVEqzS$kwsmk}G=Ver$eWD*zA;H#IpS8n>Wnv~SjzCpEj>j{go{nkTsY92Rxy2+k>m-fSNwddP5BvAZd~YOqz~+QH4LUqjv#u(D?tj5Xf84eqjBXp& z3Gbmr-__R-B0#iQoypcP*XMAgnbr~5$`@amEg|VM^LOjL@MR7@RY0IybUpMXBr;e4 z>kNwLZqTh(sR6@}hcm{SJ3MHHh22t%zlT?g9hr+rXSQ%17U|iy=QFs8T$pmH!Ya5M9yE!fe6E{kzRq=WnD9! zhkx1_`ejZ>HoHe8@|~;_Hza4fEXr>b;x)Uj%?^~9n&<1)KOXFu^B<3&<8?B#)v0&; z`_SQ?2Ge(-6Z89kG+4DgBxN+D##!dl4>z}r=+uZ;vTe|Graka>kK^=Q}ay}&LakBL?0f5$+C zA(FT9YMI|f^fjA$rE7+0cO)?iYESS6QY0n;0Vjo0qv2AV@f(!b3?!=pkQ&;!F?C7d zVVfEF@QeMEgg@e6=$`a)cA~&zY!2l(M?Yc}%|VZ88)$Gv`in{CQ|uf4b=|q?fMN3G z1_`(ymmGU~is#&uvEBw|bZH}l%tlOiHX3RVX%7Q(=TEbC5myqNI@ZJh%KrojH64Jg zCA#*GsD}bcOLA;NI)XN-VeTfaVJqyf@f6)D0wF_W^(2R!_%M3J1dzkA?Qd^1%N-aI zH?OPfv8UC|z#sUQd6d{+Iq&vpP_O`4R)`%&5Z)FkF+p<^p!uy0XC+t7Hol+65 zqED9PO1SxCQT727J*>orKJE#J_}Tjvuf1NDz6_|#09WvJC0=ETHNUvC3`l8b=Dpw# zmFxjIT0WfZ;kTMbUUxoKH(az$)?S2uT}fh zadpF^YA_C|Xr|Ff{n1tS4F`eAVhLR$ z=uyPvXymUhRR}Wv7-HVo)z#V1(w+qWz65`U`cjk#S9_YF0pRfJ|Cx=vipWN_4R2CY zn8_=37jBvyEQ8)>U!7~D5?FM~h`wwS1~6?ePvVRZMBv>1#qz)@^`^7wNvQ+sGr#Jb z_t*mwhzI9Hd#4K*hZ%O6iB0AuWPqKgPkyR5h;ittm+p*rNn`49`F!CMg`f92sSuoi z#LdCAQ?X#0IKX}vzCEH_Wu(KRE$)J~R$}mpPV436+|zk=4&aX}`fpPT(h@TZmU7KA z!irYaW?0j|D1G#AKk`-?KX9Xo&95KQ^`Owdlo^r#YkGFyO|l-2RhZU7-ES|XkOj~j zI(qrG`>zezpV-lq_fy<9N+-(XP*hxLIrgFVKLF<_BA?k}atpz>yK%XToGo)!2@jhM zBYUXePh9v@V&in+NQZ#^<>?#DSR-8L5iy7l!%$303oKz++^rT_1 z*;f(iCGq88Ej=3POUZcbh_I|abw9{~Ti)p31H)=O0%L`x4Nb!}ZfGTS-! z*J+7Yv)Tl?&WHv`Tz5XpdlPuE$q92>=z93hwsg%|-{}*>DN!Cd1H-Niy7{tP%v95l2N8w+1oIS+#bBHz8HOHpu_d}YA2RKQv{6}i8j6(sUd3NaA z(xZbPQq26_Up{7LN;@*s$*~c6kI0d|4VJ{t58Mf&rQ=!6_XAFc@IoZ7{$53Uf04w# zZ-eDF5yx>pvDs!HPy#4!=d__1z9Ew1SkxnLlCM!zWK~Wa1mVMfGrJ3+&M(p~wPnYvC*@y$y8rC#hsV$_&2f;R?ET@^50(wckfg8B z`ZwsSIG6$oM2b~0Zk^u>3)J=+KqAlJNWK+Qp^}E2M(HQL@+p*$g{Bm3F|@BR#C`0< z&x159pjl$nmWMJaG3;vNMzJ!kBohvyo=*t@p#FaVQXd^+m{;^5OX+CjCyR#8IE6q? zY85Y^@t6mf@!|%B~$`1ddvt zGdTEHNQPb2n1~WIbl~4wS>$1F4<8ZBby?EWOw$@-@%q|WGa7{`Pk%X*IeSW5c{Sd-#0u<2(6$1 ziZF9qY`}7~Q7w!a;}HGph8Dv*5O7)3L4u*r=v}!WsolvU$Rg2-NO@7WC4Ex(TKrGw zJ4ev{lP*nIWG}<33*4nn{$zq-P!w z`!)nQO>wBilW#D0qkxzoF@#&_8)hoAPCeshn&VAV4=VSizYn0%sQU-#ld+_ozaxe& z5BhB=%?h`KP`BnwQjLQvzP6p&ZyZSQRa2~NxrmaSCOKusBSGZQ?5&6mz#>c)f;iTB zxs94lO}yP$TsVW7kPmDyf|^A&zlAJZKWi(d^rdV^tO(TP!1;46!CrQew< z6**UTnvvt|GJ2TWj3WaKnRr;+%kx(^hdwQHFtSkWenJL{QU4j3cr#y`%0)y2LkQ%v zxpMS9(GnG(7l>QKdLesBVIXwxa5r_=i?-wH{nXWiW%1Qlv3|1FQ&prp25g8Dk`### zbKj;{Pm%n&=w+m^-{0PxEbZgmk!-)G9VclUaw+v$)tlgco;rzyoHuxZXd%73;UHU8 z3{fO9QIz8O(#{ zXv|N@QR%75p$@ai?}9*S=tN&HPWL z@N89Po_Z_~X@SGGp8(Iy3=wgn5`I65lk(;Pi_}#VOeUnpkpZJlIP2)Ja5C;W8Dj&> zapkbZDz1}5|P+q8)9i72a0fMI&;hr#21Kf1UjU6lZjwW2SW4=oVg3Ol$f9f!A z6n-N4>P6L^r`ni4LGE`Vg&dX(ZD-VIL`xJl(V1d<=~4{A3Zd9)k4@S{aTZD#?)DtZP6;$w0&F`!vKF! zoDzkVBL`y-hA^tL2wVJe{xqcNba%ES=Qd6M$}iR?;iGLzUG~Qm|9Yi*Tg*Q|V*NXG zW_{5?j}khnzG!orGq>DPiTQ3KZ^t%hwkyK;MKKtF`XM`xFNOMu_e+Ty{p%PlGx7=U zkG5Lh?po30;Cz4PdC2yB1C6}{{bAdwfc`LN4;usX{Ui$agv}tiOAPB>OWOKLIK9^ zwNV$k&lp{M?eiJTp~aT7X1X)ANy+aRs-#%%@^(r1 z*j}S+BJyD?U8S|GPUFPM?sdYnl)^Qo1sgYn^ zHJjVZFcL)vcD#CJ>^RzQK#37O{ozz{nr`1TX!tZ5)!j}hCE)tf#_X!${dyQt9{}`e z`i=a;-}0C)GDLou6_uhwIdlL-b@cbL+&Pfcbj5ra5GVKf7%1%I7x^cf3onJPO)4Ej zW{&P1s*ldQm&QqNy#IGOx>>)>;vgWScH@t!G{1BtGU|mwKgjaGaMa?C*E%C;XV(6+Uyqe1bQjYmq#p6AAAf*rZl2cdja%LcCbYDvhOG z+b~NXnvs6yt~~Vw$i-%uJTx7CWl7lZb=kXHO^gmJp~tze#MKy6;)4BC^o4|4~3`IgG_^U122!y`DY#G>S&Hu8zY&u}*jBO&N)7VmljN98y#=ABasK z%4Eo6%X6if+5y2fcspZCs{vZH^4zE9y+VM;&xcE6+^i>CFR()Ksa54^NG8 zVr%4cs!RuvJC(%oGaQRdzYijW*&{*-u<>uZFqTnSw}~Se9$<*uWvY41i529N=<0oZ zF!3&10_GV^uOX z@Q$yK&0Gv_4wpL^N}VeX zmi1PxOvf7gJ@DzFKiQ%PyrtF-H#rp{HKvDX%}SHiTyxh2QOQaEsqe^R&2f|WVZ5pR zPnBKRa&}wq#9@4Mzjfl|N1_o=ZC~}~w)i$QqK~sSS{CGjsuIqh^Wa>1tXaMOQq`eV z(^8YOGxC9b-M;*8En)?BYo<2u#uu1vzX)BkD(HC}f6LlB19 zLwU?!J22WMex^?o7k>@}*6thgGtf}CRq`9`H=eYb|+;*jY`1Gp_ z`Q=}Q$ft}J3%`jlj8SekTDA^+e`a0Xb^~MIsgj%Mr19v7rz9N?k)xP+@rU+{x{0?w zC3J~zK^QEpux-zEsKAZjWd;nqyTs^$gSFx}>9J=_Jg1b;1nBVg7L2U6fKdU-LrfRh z6=K5Pxk75K*OrBqhp&$aJ7^;ba3lW%N?N5Uwd}On1d6G-&>=ba>sDKYP1+k;+C3Y* zexf-P;sD6Z(&UeldJAAnIAPlME&x7k?8<)K6?U?I(;35SVhh7>`MAx-0p#hn!dS(% z=&)krt}T{Mde*8oHFtlS`-3?y`H4 z*zDe}@wSclhxBJ0x~hS=fD{tu%s2#!Hrf#GopC+(`L?t>|7;=6EcW#HS}yScbkVFW zYa)kbeYuCh5x&>Y{`{|8mjVJTd;<<5&uT4Xj`;&{O(GOmJ9|7%z+DM=sECJ4ZIa?H zEUYde7{mB~0FtzyI%Ulpj4g3@NTyw18N4=Q26DN9twJnq#poo!FLI&i>_0 zE8D!i;ZL-KWd}VTW|_vZK#8a~)lB_=+I*)3@u&9dl`-o5)yMb)R~uY7Pi932HVrzx z47!0!rsp@l3_igK>!u}m($9OQ0VSO+pP+}Y`mKOar{(8bAq*x?5Q zV0VQz!k!~?a&lY-f_n|geg6QF91NK9P*_lvv;5M>wj~zj_Eycrvz~0OPmWnukGShf zyh-g86Doe;?F4gRT3ZknGG<@rPJ<`FwPry{cc6!yQ}J?ZnR-U4i|#>3q@BlP2#V!6 zDec!H0z)h!CyAw|@!cvzgmn)ZnpINp{lC_f##0pPN%Q7OjMFRnpxgA&O6f&MAh)l} zil7>yY!R!M{w&1avd|TcnEj)56ZKY{cOKDwog|Z`nk41m`{1pCU0MJ@Xc2;oDS%Wf zb>;7+EEtZ4;1;0^5STbbu{MNUmVEPU`n$v12XVp)eZB&5fU`KEW$)hACga}O8@`Hl zX!f9~V(=q)$19m-i9c;L3_?4{8fIW(Haf-BinAM;z`^k1U@1${hvU2&vb$*L# zm`9SU`NQ*iVgGSJ&bcLIE0=O&$daeIaY4#igk^QzPbavk5%hzoe|wb2O-@!8e<2Rg zU(`K;Dj(=2!_uOND&G}3XDRynUR`3;5yNH&qPB+ACxkoE6Ys_$7vpe~*8}%G(PevGQQGrnj*j5>*ls(!*NYzXMnhn!lkc=cg~_X> zD~2;$!9(}KBp^=(GE{KZLT2zZx}$BVmny{W;NCiitm*FV;n_QKhOhiGlR|kh-_^hGVv!D4|(H2zbZ=;UW`2%QTAYaRmGT#g+G(!4HLuhmT|W`$0k-4R8&nel89HRlknW{-6-ecJvD^zFaIO zf?B&w)A3~DlY2I&;C*{uHvpBy!cxR0o{tu5J)Lx6y<>oHB4Vw-+|oH+OTn~zmdjt9 zAhI+yT5br6r^!oBgSTs7R1d~g)PeC*V$4p#>`CXUnl-=ckDGfk(Y2+EUq)7KKGr?< zS{y(T%MRNeS>5x&v1CNo>I=>J3@b;b@nD8F7Rfo+Yhx~d?LBp7<|!3hHzGfqP{ob7 z6WrqHyrZ%@zMnlD%&{c)`>4KQV+B0_nm_E-S6NyWmNEQ?dNyL=30rJSd68(*e}G|W zzhYI4Sagf4(^TJ~grn~#TB)%Z!S|$hMk&9l`Z6>%nRqI2!OL9SRW9w;X#}L#A^RVI zdK#Ha%5Lw|?mAM;W-X)2|T_d(CruYc?* zr8F=4Bo!mHB{dq;e5PY_MQk(Jq?!ovT697{sSQJY7 zKcFc+%V%s9qOy+)%Z=6eqt(AAcHQc|!rcEz(IDn)qdZDB%>x$t zHpj-B)D;jc>y7BYH&1j+Ee1q3AP36RZ z4SM~`{p%o?(hKJm)DHn-9%cv~4)PT!qG22b2@Ro9WGpZIWZss!IV62~H1Tr~IPs+7 zSOAD z-*P+pKxmaR{{ifJdIfu4q$xMpWyzuqG7}ia{js97WI!Vw9%PYR$#2H$7a*dLVQ%G~}9pLy{_kJkmNsj+j&8 zQR{2K1LeBx%Q#GT)(sDi4RE}hzywO!<4wzHZlmUT!Uf!Umx9wUUuB`7 zr&S9q5OMOht0iuir=L?&ubPK4;7L|D3dnq9skF?ucyV;KAh!KB`G>uMfhv>1i#POi zlyL=k2hW&3;sgZ&jQ!K}8(1xag*xw3*5R$ragm#Xt0g@P6N^GwXxWP-%h5;VuPK3# zJo3_!4#|qF{{UpH{aJfaH7*RUq6(TVUz!#8n;JU5B`$ucMYxx$5y=^h6Kwg^GZ%$E zy&hJQL61gFX?->-dB8dmW9&q~6{Vsk zhIY8aDBcDmwbX6z=THhxT`@12%`X0yw?Dx$LJ;&g={mWfJ{MQ$uZ3v}MBT_#-vBWV zF)%I2JEBB{h0^IYKdW;r#O=qsT!tsyXKK56o@b(RQs=<<3ly@R!8u?xeoQ{&_f-^^ zSJRKwdY%8&<1;&>2otd6TT2td+GfHKmmZn}Rzt zeBPDOvfcahSxzm1ySNW)RCvhi7CUgE+PH*N%QbhO+if8Ey zN-XA%8*{dzUTYOw>y7QoRuFQA*1Ebb%W=zczfB}Lc^>92G4qDF9!iV^MFj((UWUn< z%f~R+YA1MYLZ^$X=z}maGn0sKUWy)VS=%Y}Dug3izO}5JoLHW|UYagf4C`x*&C%K6 z(z3&=nY&^4HZ7@{9XA$@B|eRb#|rFM0C6iRT-jTHpOaRJ<8YJujv4w7V2m-DGkZ37 z>>}2zI)B3ZjdsBF@!?^V1T{rh?i#xNz9uSb5%v~WxaNe3w4)a`#BgaD96EApMB~KZ zm(&O!D-kMBC(oAw`LyOK7)|4<#tV7u*kOAV1RnUJZMOF01SS?#u8LqxLU@U+S0;?u z+fQ?fZ-!|Cy!Ib-VP}hBGPkA8rD|zM{daT?O3abSw;yAM-pe@{Kt87PNKpSIw_6U! zsv|)$io_JieHr(Ic?%x3t5^qo3+~2@cXYp^G#W|%0SqV1Cjo(+$v3Wnz=)P0Ms3Vh zgj5j29rWf{K^Z-LX?`WddXl)!J;2ARLCo=PM1#4Wttbv4v=oewoqj&!a^q8g4cEOJ zX~yu!@Sch+jP@|hF6+1XEA{&$RFe&pVbHYoI+~GZ+h;OgzIu(*pVQ-1I4tEu(-h{K zXsXdlHx5Q>3acxc(B@rurQ@@{)i2RA|zEU6%H9%M-DNPRBQWTx)dFVUV!nbR^T zx9WdH3MR((p=>_&UmOd7CA|m_@EtFj^8?TPjW(j5^r(le(Lx#u(?XQ?itXgiP7@qC zxPPbg*KZ;(InT+zk13Cl0@6d93>RCp)t*|t>1cPzx?}2UZg)Zbc)RkM0PyTl`2zm1 z4bJ9DH0`K7W8}$=ei1fv*Hu9sBb_Xo!Cy+iZ`kzyLFV)DUAwVuF@c$-`T;8%ti#r) zAy)wd98CpHn>%667I|*LGKXN8V%E1#eT5mC+DaRKZcEnBy9I5*%_YMgZZwYNirkRL z=}zfPvAu>E6=g81@v6BhMN5qy3Q+xY*<$ATHoY zC2fjzx^eRSmCMrOPH8tzSsq^79uF%K-DjL?H}u$91cU-6+m<}!$An=FKRIauQmp-< zk%5~+* z_mYsKWs8G7Q&#F&-o%GBkwf-`daBSVa;J6h9`7?|%}4%BT$o`8oVCg@HEcAZo)#vz zP=rxyDa4{iX%DG1?du%n6;LVr~G?2T3Z%Y zmf?c|Iq>eDT6>=$h+^j4v5pAJmcUa!MUIpI>0!t!{B)25fNd+srki2vQk~1e64M+E z@H_Pm!w)l>RA#nkJEN4k?I5xG_*vY{J5uiqb{DD5uo36= zh~?I__=+!nhM6LA-gOehhfC(iSaPNvz_dHjCG}>95?8E4H!v9c?zsr+*9oShQu^=kimzgMKbX%nNn!@VV z*OqkM4qM@uA%3jizcFw8nB*dT+TkA(8B7Pid4~PK5s9d$;#*1)p}3u@4{}@@nOdjy z9x-l{`H_>r)7T&swY~Kpy2xDT!(X@F!u`LekuYtK>istPZF&0hO>o#8^1F%g{q3`% zIpLVT^xUm&z_V$_8!(!4V1s(8%LuW9O@)ABy;;Gc+TBYzA|j$MW&XzO`8PT|QeL5| z6u?U&DFD(Hby_I&1*q-~vt^l5Y}7OL#ELV-VY8OK*Lra0%n&+im@-lg z^Y&MkeMx17X8Ew)hJM+$3Y}fWBT9+%tTR%8AZ`o)->C_$s1Jz|LYEO&VctN-TraI} z=*~&f@nVxf?@z6b8oZA->}Efg5Ky;6*fH>~fH=UVt1U`8A0-yY!G%d2sYXKXZCdzy ztXM!@)BGPqzEQJD@lRp+Syk?yVjU)(6DIF>TPZ9OEMtc4S<~rFo71NEeyaaQg*~q{ zB;%ge5A~0lA4uOSgANzm*j)d7$8~REhCGyRDzK0lvf_Yt>NG9Zo;u9KlqtuCJjOUZ z{{hOrvnVG3-m2qvb;1;`#-_aqK* z?0sz1H+%DNu|`bN;<**4W)NGZe3f>*V{O#>{zR9j`(|H2@@HH6;s2yjpy)fEgASv< zUV5ykv1qozqK|^v82=0XdnM0K$hF*v^B$KU3YnQ>V-Z$NFmWtSaHZyItEI$gw%KX{ zUs3h?R+{qU{%m=&J1XrZ>Z{Z?E{UkSo5@o`M-LqbtXa&6UbJ*2O~Hq2E<~i6Y}e=l z$-@l3d8Ix3rtRnGN2nF={(#D1e15tZdw(;z*dgjyhtL9YZG`&J7=^*%NUtaay(pR&}? zizUah%jw0tE7w2x;Su;h*sSprF(FYfi9!oi%3LJH>l2N-?Uvr=g=VV;3$tHbjK((y z^wArvCbJ*R=>fZ)^@%fwR5y>Y?;c@WkA{L-@-~jWtIgIq$Vgn$9#S)#O{8gGC;Qqa z1kU#e!tCZ01GgY#Seqk*gSlm4}Q0(Q5^` z+LiOpjL1%V!+WofZSTJk&UT3ND}f?@S8b98OG85tnfb}b$qY^bcA2!wep}gNT-ll( znJ*qR328+>UvY^K>VtiIm*#I3)l01iD#a3y^iW`)I2pp-dR)uHt~A2rwp;U<$;d}} zW7zM@UT-A?ZUa332B#u~5^diwTJkOk|JI;l_3v$@KXudOvVJCTw#>|X^4VQ>T)`Pf zkgnSMoH9cFbiQI-1XM6YvC=K?s?AqEuNq2Cj7oB=e2v;l`5`_w{qhG^AEPqx4?y9n z=vr>N>3m6hIHN+Dja-)=pkJtcYc3hP!x~C$RrpXk`0dS8BGhFWx(Ql7(D8xe;o^hY z8&Cz9cKvYVVZEq*_5 z{RHw@Y0b2$Cuxb3LY5Q!EwOd?t-^=Cp8*d!A8vC9#nmmpUW8F_4&<5+oU1PIVSay_ z9k;D2S!s=1aotGh92KCrA+GonC2sx}5C|HqkBCK0hxUjsN%m519OVf&EQ)Y7-%V?l zWjwHc75X9~ieu;XQ&k^$@~=_RLR|hI*(wgE!CDRdZFHhN@-wZ-lOU&h3=v0$bhgM7 z>UQ!hOYJIgVw1}btnS?yZGE%uaMgdia17!4Lj8&~6~A3vq@4pgkY7=+h88zy96#?%T8PsGeuU`A3^%zl3kd1d$AzMs9XgtzVyMWR~*S_?ABLkDphfu z{PoK+yyp%O9ZC-cu>Ge0f&B+S@+$E+XmMgqdZ`pu%Qd}gMUrDg`jccoOkwVFxzbco zfU2;-T1V{KNFHG6jtXlv5D;9ARO>+>bY(KvHEThUo#5>a`qmBc7|Jhrw;u3NsT*3P ztF7_v$GeBdR)=lNOY*ReSH8Y&GC*F#?{5Ca3u#nrltl7I?8P^H^CB?fQfS)Q|D+2Rc7*OENt0HNZ28udysx5No+{r4L3$T;`9PSQ+pbd__0ab zwQy~my||zMtU7z~Nf`9!^8i%KTipIFF`Q3%^Aur4H(9sR7Ii905mj1hJg8s}f3=n; z$L)`zZnxKCl1S3vy5M55q{aG&prbE~&~EWQUXY4MhnYqUZLNRKPF%O`#mDfr?KqV8 zl@H|}r7cCVd&SU>LlG^tndY+Au(_v-!&2J} zXmT~9Ne(m3)b_fP+C3tnL)(8U#Z{g z`jxq8Lp{=zS`K^2usa1zfuwoQKt~=tO?~$9`?_SKjM;_GljQ+US zVN!I$7(E(wqlzM$g;^nm98W}Q^tB{R*Zhlb*+m7ORH}J>I8*2UnXsV5b^BRS+m<7e zDCu=~wUyZ+PJTKP3tXkh*)Zr+P~v6oXX=;39V(F-(ZR=N59d)~Mq^wrb}-W6&JgyI zPxcTplg&vNkUP@rQ5RQAWc0_|`y4HEjv=jtwaF338%M9Z(uW5vvMx1?UtG(qlyxK_ z9Y3tMH;6|7609!@aObU3(fcsP=z-1)qOSI(W@D+?2hJfvzXtTyY^PA2kw>F*ja|fB zxmrk>1j$~CgL`7f6wj?5@#n^kbTPcQ!F%vL>^0t&kv`@e$)Ai78jY=h1}vRSC^vm= zo&QyPn!#X@#5=;oLw~5FV*C$q|L%bnx*o}&h=d*o|7?9ClkH?)kR`3AZoY+jG|I-| z@r)LV8t4I{vEAX%F^N;%Ar9^jyz@1rzfQ@Kqc!+azUqN?ibhjBCFVLrfLo{0Hx9+v z1g!0dzEi+%WLayaROwQhYi*s)EB~GsBR1Q0&`lbLq>S|X4VcXQMe;F-vaGkN{xZUk z`oMI!gkeV2jjd^DD($=d*g$xzB6(hbQR`uE3n=nL3dm}qUJXSaPFT$G&(|m_$g41X zDb0-4eLm9k)tVIjy~2R@tidIeQb7AQTT(80@Y+>kbLFR|^TUK4tzvo?4iwiiPF_2JndwD5#>!RQ zrt4mV!d*mMeF)D`n?kA&-iuRUd59B-LrBj;SezeYlSkxFYF!%nT5fNybcG9uNzWtz zeKZo`QzQix`4Np^FX*27yiW*l#rZNT-$u(u9H z`ysSGo4|cb_J$c3@4$m&-P-rxGWOoA=s(B;6RCkh1-G|3#ADHey6C%T6orf-hVY~m zOqAj4E@ zH6EzhVGOE9-|HnR28u(U5l z&3H*&?p+d3oZ!_9w^s$RzV8}#GPWsJr<6?QPSQs8o4%!pYrNNF5;G?vKz75e;P``c zFi&hGO56c=l+7L+t^RHP_}y9Yd79>fB&jDr2}kq7DW)9TIKaSc&%J}d8}@Q&@^F12 zGd$0&Hk##pM;8eE6g>X|RmdV80Tg3dYvRt05W7~6bLF0SZexmMjBkHePjy!`W zn{f4Unj2R{w~oa%WjexDN&7X*ltFdL__@#v!>6b(GNDAM@VxeVhWZL6ap;*AXm!57 zb$N!>iW!?Xa^HtX;P0yBktXl8MCyp*EHhV9|F{HB_^gW3*>>~(`slX zO(2<;tMGcsV~yv-d_SU$Z@MS*sO`AObG)lK9%F$OSk`{|%a&@Zk*ug>BoE?GGqz|W z36?7O#r`MmsN+NPt!-2(L}pkFAKKqVQVnf<8n9-D?k+KuFgx)5Izd!QPSF9}P#m*% zd)Z2j-41xWu7aA5?%~^45}H@^psX)TG#iRV-Y=g)#BE1X$0UBEQ4DmCE6|Yf8n=A> zOz|9fH3-7KRpS0S*W29NqO@3M=A`78x_x|}?)$^d1y$xXT~GtL2X|cF5NM3H=NP4^#lqp35F??EtwuRNwNh#5ny=x)6=hVrF^1lb>0s86 zxr4Lq0y9QAra!xL7i6eWKVp|u$533`Z-_gC=ah)hIUrW4W7L|-qwil~ zhRgd?siBpRemdl?{sZXA@dhrc-{yE<*Ha$_Q_d^eiQV{@m_8kXdx6Uom`f5>vhau2`2}bq?B%zuUv1`lxN`3htI(esJV{=Dt zq=OL{s?L5&ve+e6Fvt;K>MBQcUu}K$^jll2x_KPn^zJe1STOMl{N`mxa~C$7p(tAW z)3N*dA!xOh;~(Jn7u)xIPqqtE_VQ#WVJM5ekz}R=OLMh?L@ggdF3)*M}GwC#)_p+gMvUlrc2Cn)R2_HJh5P9rB+V z2guq_+ol>g7KI8QbYyJ&7xvyWsHyO6_obs!A_CGuilWk_O9z!EA_7V;Dj+2kkrp5j zI!G@90@8bz-a;s$iu5L-CiEr=B}NE%)_=dB&b%M?K4;FEeLkES@`0JKn5?Yztoym{ z>-yaZFv_#=nK{|!-vSX^sHdDs-*^jmT9Zq3%oZ|JmVFoabz@ecQC^zKtWe% z?XBIfylH@7vpjV3O>3%#s-KKOiIT14WGCj>4^4woDSbn0OeLK@xZi6R8k%H27b{ft+KP+RTRi6wknyEh;8^U0d~aj=!C zRj)EHL};K31ZT{H!1psP$HP|~jC9$@IEp%(lGS*PPVU9_-? zvwsB0DjY69D}2a$_rw$%AjS>s*+>FLtu| z#J}Vqfb>sj-?vtfoYjU~Q2FdWosbUGsX}?X++Yz^FGO1%xU%(mY<<1D(C|GXwDm+L+4whY=FZh^;;YG1 z6~JlJeI-;6Z^e$mZE_FI95mYN+hV?MKKsPn^O&|cJ$7v_x3`OjK#dmba}hIbinN%o z<_F^S;QbCEh+uQZ0AJ<`NJf3>>)zB!>$}98r`fxxxQ{C&K2QqZR=35+$F>v(UC`C3wY8kkY7uo-jgdS!dcgoCP4Lg)YS>rs1Yt00#z!|t%gkZr=wt>tG(LJ^cYwda#*{UIkoH) z2dBeDyT4_Re52(dSjn>x@A0snxz#(V*yB>eXRAa0x*vT8wDSxNB5Cm0)M^|F#LMsQ zpI$b--Yz=Gxb_Uko2%Iw+bLHmoJ(D`sHS+$;Wos5NI=Z#r2X6&uYxk4Yj#*!zMgjd?TMB# ztqhKzeqBk3r8oC>Veep+%>m+8KQ!eN%uY;ejKkbQZD-C?@1K0{vHjq?uef}Z951g)<6K` zyFxP~6I2?#15!*9o?ne0OLlo$*=KCIu5-y862Vcmb~sR^jzS0BzoyTM=lR@Z`8uJCN#Xa18mqw{L$woGB*9 z%& zM=%i1+)9X|O;|%nR{sja)i5FrgEYpD(O4jbZcQ$!;iJB*0Zn;l*LU$7WFVRy<+}ui z3k?Tl&gS`=w>oi#dPeQjq5>gGPo_+5Y=!e;D}_GACDLmu=}F@v%>gSu?qa|!J_Y7g zymu2U`$4GVK_;8c-CwtAiWm~L)=ux0PCy;=W&6Q;jQ`O*gy#9C#Y9}I-9$Ziev66r zy{xfG)b_o|A0Pz@Qa|Q;*wyD^(|sjP$SB)SJ0FY^IUP0I!yIq_oO~;3E?H;yOwG4h zZH(sj3!D5r`WE~^TjmMIJ5H&uR`HnN@zVEhQjBC>j6kxlK+YLjdVjT%IswjwlYBjW z7DshrVZLaoJg+O08J|HbUf&n~BFD+1ZFqeXD2>T#dGkPJcQ0c-7qzT(YNNo{!W<7j zv1kduvZDK5arKX63nod^xaRV9u`8lA_vY0Yz6*qzJA&I|e5JDQO`E2u*QM)C2hO`g zcSoM{Neq;;8eCTx2EM5BqqAI8Wv^eXe7$Dr{zKPI>_y2A@0!U_oD9r=>A~Xus>@Rh z4AqY8p8%hGvTw&i8J*8(+DkIZAuz`b%ojTBQm-fty+|cz$wGEWdMHnonUH%Q&{v?A;zw4R-N|ai`l01b=`h%O<_E zI&32Su>PI%!mC=38xms82#E&|D?Y6vv`20lv95e{yX@uQ`a?KM3fLoJ|&|oy<#VS@f z1J&R+DR}+io>Z)^V48S*ScFm{eIPUsT&W))Z zIu5q~NM6s=pY*-SqV=w5CfPWN-Xxl1VOI=pkuI3rL$@Xu9Nj=DUa!F!cw;eq`C{B6 zV99+Cj8X;>HL*T@Zn3#Hnz){O?QFq3 zQP-zj8lG8%6)F1k-J-GTb7C~Jwt`3Zk(re|p?s|?3dYHgOGWOt-%D{$6Qit>hst_L zc9*en$T9?ViP0tNc>(1st^Ap_!oJk}NLhDd@+N3T{(Bsbrb&Ho3~SvW=J80+2Vmnxn4!9bOS%|cYgOD0r$;f@g*{Eg z3XGRJxPGRKuhaTk#y(QmX6~N1#*V1Y91#Q7jv9Vh{AkH+WLT{4^ZqgXkmZJZXyZ-2 zJpRXdARSz3uabE`53}+;C+>$h14J&m96oB_XT8v^t9Ik*CzW{}z5QzHzyQ#P?gl}p7_6_FEOJ7edHy<88X=z{g32f9_^Dt#!mO6 zQ1nrRJWJHoV@K|r9RW5yeqPf9=|)$4CR4{KJu&w%@z(|xX#EAo%lE{{LK5FMj(b+D zb5a2^$X8ucZdK0Glw$T?9?rP_Ple@iQKa2N8;o-uu;>-aX(Dxiy_-rH+!75=@)Gti z15c|zTz(R+K@h!tZC-Bau{KNU+rh`xwQ=ma#a&3T2>E6JIuWlc`#R*$@Kn?2YirH- ze5p5u@4k<{FDDR$$oWFL?r98Pfr9=+)kJXPBzAKi;FJ)>B|5(TwU9uQyA4zJZv=-k zK_Xud{`#b+i4u0Be-Xd|Xo1VLE8eVXMzhTON1k5yRWry`Z{AmW6BhOl zC|2`8S!FdRJvrQy@OPNF=UghiD_BdSJBE&2b6!!&qp~w6pB<>lr=N65efl6xFFC&SM3WOZdb&fc?W`ir znp_~;BH@o#GmZyu*$hW1ZDskFEq?xe6S<`+{HYr8E8#_@K=wzD*qj!86#|<$3zg*9 z$((l5Ln|qwEJW+BMKIrAq(}-#mta_m)LjmlpdOvA3s|S~e?4(SHNarR?6|`j}JVHB4wr)xjL4!-fh@*#d`8Q z7z0NG*Eha)VT#P3C=NYJzSN zBnxc@$=ut*E1Y1mbU`EG6Y3k!gBsrED;z`Q2J32KFvdt&SPm7gHU`d#>iR6)Byw*> zqiMF*?19TxJF9K;BlYW0<~zO}Gp@rZDhblH)cgX1{~MgZiCWQadS=l?@BFPnJJ%B_ zcTrgdO6nUD5_;s)r*4se_AlhJLOU&ONj;DmeiT)GYj4|P@^BFEowQZ^+d%alOsc#m z@o(K4w-5;LGr5HYGU1=wrGJ)9?Ow6ZIhP~_?z0XVX!zSXkZzS=!HFYIu zy5iW}9=8<8)X`BdxhOnNTbIhaE^w>*#O6-uD2XrUqhSxJk4O$3;aCQK^}89LyGiJa zGINcl+#0zw0#pp^Tc>Le7j5TS+@%Z#mOuXJz8Tnb+7hNrYe5v8B-*n?JA00d>KZ@E z8Q7CzWL3PFs^a%~r9UuWyto?=He5GGc2NWqdwNOS6sXD^PxoZb)H+geo|tfBJs-vUC(&`j?Z7K z90cs#*+V3L>bLF)EfU{g*YyY;_Eph@cft})n_p4?lCcdNjGj8`ku0uyH(T5irw=4Gz2`1D&*Q?ETE#5qDc?mvzH zDm@KJUlR=C!=?9$Q8}as^0|O}&tIGuI2ku|yA(S#iwE`AQC-xq{G^xY&i*=}s>qLQ z<`!s*c4OEvLRIPS3fGEOmkLw}|H2j*>z&!+1)>LAThop)rrW7oIYwdv$>JZp1qG7* zlKtmR(Y4X?UE$E{n1t&K{Z1&iD0prYykhJ1an`2;-XON4M968aB?07*_9SrNoGl&B z;ZvHWrZ5L7`9}SiV0>uf6S@Xp0iPw?@g@@Jrn&=o8ZbkDNhnybGI89XOSE1tp2=JP zHdLnpQaPaYpQ}Z}Xn#RQ1~a)oS2)$}_ywFzA(ivRUwtp%`_u<|mlB(#%89Ae>Un+p z!{ZxN+R|<&lT9Gdo}}Y>aokuIj9zike8}v9;T?uH6EcZwig>)D^Jwwu4M_3>n}|CczloJww6ZI#+q3^job)2(L6J~LOf3gt#ZuF? zPWTZ7raiT(RopJV*jukFoWKzlr^mY3mhb{#7jO6-A-HP$Z%=tqV(;MGr$#WR5T8S< zmxo=))5Cu~S}v9==b*9|BB;yUp4;t+;B`y0{l4a5FTk5>^p9kR{fSN=-;3@mM;_mQ zB);6U{IzBymzhLql~9<|mkC!})6!{c{YGELpiSOXACOJ#)&E#__N>XS=y{^`Fgh1e z(mKnMNc@vMFdaRQ#QCx2-S-i^O(7d0I?T?LDi03lA%$zdW3YQFL%ioIvPy=G#^=|N$AMN|zl)5=X z({6J0v+gMFC8v;Yl^RK+-z(~IDdUM|6>4Q}N0eS~Xq7I;eAIA|ObT+~T8)k#_gv9W z7okaKv+aWh%-ub|5!t)ifXMP(WhhAUvS!)4Vws`)#?3P!%GV5U3)6a26A0OSpZC^9 zYj}8Hm77%Eg)pt#I7!qmTQEWGrO8YHN|{$#pHJd9y5ajIbUzl@y2nUSZQZr#ivaFz z{p&T>){b)=+LDfVZ6DF!$PwT7tbdjV0T&a9V00w{z}{i&A52+9dDsm!Nwrl<5&4P<;G?Y)2NW35}L=UG}i+V3hiw&n~Bb zn_^JFGBy^G=(x5-h+Y0CpKSB~WGi4->1NL>^b0gadsU^I;{*?;Vc?{M?Z{3d=nVQ!oFogYm@p@CwmMDivL(8&JG(zJ>9 z;dp^H3cXMytc+cHeLfXY7}Nt?NA|u9t?fx7sVrVSkxu1E<)iz!;MfyY9^pxAGpHy5 zhRzL#HARLue&wBQVn#%8X+CdDKEFR&uj6!J;CYirI+FkXWPBubG4u#`;{<7J);~1i znQg3_;c9pBbWgs>7%ceH{rg?Gx!kzHvIBWaL2*-4+W8)b!_8sEvATiKO7X|!C(>s^ zSA+O{j85(~m@TBw>EmwoW1Pq0;bi84V8+J5vE=6Z#);y=?<;PA^j*z-2&25ZxodrM z{#o=%ua)9V!Hno=(^zNWv&wfiX{b`}XE_dRv>R~9Fzfo=Iw<2Ll&6@mZGj6vJNuoA zKD2D)Ft_ANzTQNGc=;k(C@S$jZJ@D=4E0M@#T4C%Gm-Zk!_HC{l{;1WvMhQoG+N)E z$q#LiH))n*%I2qF4ebPtJ-z6X+jGK-;(}qe;G%E+N=y8HzR~gdMo)WC{fuQlaO2s; z=LG*eM;HjowXR3|kHpd3@q7bN+kWlMi!bTWCTg*K8s^jwsRBL83sIP${;F@(*eT;; z^BS<1+;9~w(c-85C@V?Hrqm{VN?}c=Nauw!J3_x;IX@!aO=l_ifQueAG&fha|ICk1CU6x7V>|H)eqk1GumKI-2D>=v`1g%J_o#9&P~gn4t^0ywJsO7FFuV( zNwD(M#s-}wL{I}O)T+6BWGGavXvf=c79U=2>M&mm>7{sZH0pNaj{Q~n~EDOv2{eUIDX$i^^E+u(1;+Sm_Tfwzf5 z^KdywZFZ12T?wY$jenIsitr`G*eMN*afY4MI%pt~a!Z#*8!R8uTKK-qS zA&>iFN!Gms;hEdK1nL5!>J*f`9CFW6?`2G1=FfA*=JOfcvZAu}=r&dS;~s7kffD&x zDkTT<9*5c@sP={QZW@9M7hebln!g+l;snzv4_^%HoI-%rjPmdco}Jx|VdC@aUuOe8Wu5W^DP zsPr7~zbw`p$W8FIsaV#e8I)e{tG=V9|Dz(^dEM87zQf;`d2Kux&~&qkA;#7d#dD|2 z68nAP4)O!J{vOPI6B$$+X>4;$1~(EPO=9&7Re4m5Vy!r4Yos2WwKjGvNIBfSeY_{+ zy0s#Aje=i0E?+P5?Zei^ydIae$tMtR;?&4{`C*^H0% zqYgrKgLcigKcSz_T#sN`4!Lo^`ej_1)Ae#4A)&vRXRq?eJ6xCU?-gB4Q1hqkCSy)Nlh05(+Pt}U(Jds>C$(3PR*z6b2)?s*(P^0=CX9#v|;lPOl%O`E^%O}h|5 z(-EHY+FBz?pPc7o%qh@MlIJe2)s-^)kn2@8LLuV*(Co=nqq~?0Kd@C$Na?PS{te(1 z54;;7C;j@XFY$A_XiF_UDfc5=-SW1O4cp)x2_fo%RgmJnE}BynPJ$x<7m(jK$Uv}87Hj=eS=-X;e)b&rrC}aGPmzxuY*#v! z^}pvH-8gl-y#o2*hN@o?QP7&L_VU72sW`N^&+_H0=$+Pe*a|tvW+=uE{o?eP3sQ#7 zNL^GvJoi^vbgViOSsIk??6VVy3+^#89J+&-wgPVVI_zo>LK>$sV7|;A;G_gYy4@>Q znq8_3NZyRy(7E-5e8A_sn%tw`sNd#Zt* zjYaS_HFaPcbME9GrYAQ2{KOhF$e{0<@7}n5bemTB2qum_3WCaaQCbiB!TpL4zC)rM zB}{&;Xgb$kZ$2TVpBQfX!!UNOdjMe2`vx$drXP9*pw%#=GJ=y{X|q|#OyA~qODkRq zr}ux!`|*}zs(a63a@D5lj+Ci)%=CJXK#&a2E+#_Gs=@SKc;`G^Mfldju4F~Z#wc~# zPj6_)$OsG6v!4XgoPu69wl;5`Ivp9+c1Fvx+?!h7ya_z9FVs+lGdR=P!>$JB0VNj9 z-fvXgTF1e5+h2F$ejgCn0%C5phJdQ(;C8DAt#Sk6-7!#Mkf6L-hu`*hB&a0CL$ZkUZ(!N+{i}+tpvtDeJI=YgJJsc(sVhY>Bj%tYkbL zgAPQKoPn63Z>P_?NFlIpyu2u|e8)+?a#5T6M`Fz46K}#SK6pR)mH=wB`!m9(bvwUq zJvaJ_5$6MV4(SOWR(t(sgR!T^3;=lyj2-EPt%=xW6dem0Xw*tY-fvb?JC(grEZ?iLar7ZF~tvh&?E?oE+!$~M3n z8vYSFG{Cyep-H}dexJNK5~G)bbzSdLlLnJ_%AWK##p8 z{4O>=*Q46XTGI>k*&?BFQKQ4-R$S5Ap%N?TD^^eXKZz{QKvqD*vUl`w!|;Gs_KLG@ zBldM?`oG}TZoWE-wzZX`{!KN+BO@U+4OHfi7jZoO$~OSErV^7CH@5goyH`oNySHtQ z!w1y{Hc_(sqRVO~-B!wmtH`oEkUcEH>~_YQ0)VUb?=f)nzKvQ&G|#ihx1d}FlO;aY z^fFm#%u>E~p&5FwUBiMIOn{*spc?b=H+Y_d@k5;im4h6gtW9X1P2lXfo|;7}6X~AW zc7Fmr;pF**;Q%+wj9XJSU%%80;McShMohZq?u8`{yF6OTOKN+*tr$=F7OIFFNC)iC z<~jT7@L7?ggL`!m_q{$>Zo{*#Xvs46UiDr*>;mj zJcqmi-SUy6_=EPv6JMYg6R-|CpoR6mDY4jHwYjh-`u+V88u*VyYF)WM}5mHRxne$JY5V3Rd9*AwX=VCD%V8;%mcyR1{?n=q^>XcX07hb^5{uG|AgXv2Kva7F zu#2SSm-WEPwP6)B!`oUGibgy8v}*ixDL*=H`6|9$Ou>hpC<_uU26%q40{hF#7j4ss zLHs!3>?D=y^2&NVF8fj2xW1KwItlYCQIwER9ptR2U$keY0M6uzs!~tggd1Cb$hgtG zvl0I!o?OV|$0O-(eN}e$Q`DP*V*Z_G<1}_(a$A-^j$Zd4Z%YX4hmK#6bDw519|D;% z)RL3@r@guR|40z^jyYn|@;A9&zUc@ZqM(gnqY4ME7Qe0muoM9d3Wlo`S}!<+8d@3{ zRyh@1z1kzWPCxX{BQ9f^L^fwaW`B(|)X6Btw?nv|^dgSX7zSbliSt7RKoGH+eq=0lqBlTA}E zef}q~0KdQ{^tVi&q%8$Z#E|n3;0|oqp*WAeEsD`!MX9t z3IIJR&dYc}>~&^6Bcjq0_sqT-7k|C^x?Bh1y-qE+{4M_LHW{!sT1VxwN#$}u@)d1d z(2IwR16OuU$W^sU;WQUTUh1zRs8xvI+x5|IW%#z$K0b0=s^n!C_ClgAoVdA*L6u3w^6g zmn-fGm@|CEJV2)+%3{P=z~@&@=s3+ozlai4YSp=N3m%}0|1=*F#Xheg?V)90z3F== zND%|%N(04+FFOdG>y_iTEK!>8r;0ufk|}77u-|{ZayWt@8R4|Wa|uNQ(Z}C`V$c_Q zG2HSn2JI4=uL*8$u5*>{Ti?{#^Lg>~9Ugxm<9@qdwJ|yLv&ziBAlzctz=>v#e>|*F z%PLX%zKXjnZez-Q>hlo3Bs8S<`M5xf(!v z2K?t))|Y*d+X(%&ao)Iz)0iIMN>48U75})8|II#;4v~jp1+hx`U z?48B(Qb5P}^7h}K92_!n9q#k)<4x{NuPEkxQXKyzTAFK3EUEF0aK_AqXg>T1K5jkA~?^v2iiIipQ1jPE#vBv9^9l{ewcrAZXz0a$AER8;WMO_Fy*P9{| zfW5jE);_9HD9%G;R*ZM%wz-n3>C)noAV(_d!RrXCYY$ed)nxnjwE~&r-Qlyi&rpgg{*Y+Ze$g>DOnlD_ia(%%0Q2+&fmGcCA>1|5w1+ zQL*nP6KUN#=q9Y!6a!3qIaVIIEDcaPO6r28Z)uIUcvHD|09v5D5u$b-Os9jc2}XZC z1;C%Diy7PL`T<6c@4jkjqtdD`TBA7B>^7TEEW(*cjIFT=7;56f@hFs7G@|FS^fHTI z8?UD@d?FG1b+xJPPs7A%%@e?0jNm4|1{QaqGPKISOvfJ`F5f%RIlh|Q{LAfY*ro9D zu*2`Iy^=Z@;venvENSJWP)cZZAzDXBW)*%5IEk&DZUrQ#^ z9d1z^b(o@Cxz%_9MAmHD>+rnnPHN;{3$=0wMr}2hU+8n0_Yarppmj_dITC;# zdlMHGO|#Z^u+Gg%<)yyv?~93q#bLRsLl8cw9_t*)JPnsdXHxRMM=&?Dr!Tqbl~G1?n1v<_q{Q z#jYDCH|~=!H(8@3pJc9L7Czo6VCVNZw+0 z|B*D_-O9xpxslP`SHi`cKlVbppCIjoKMOQtBqDh;`J8*oYVt+FUeR}r5#;^uPEB2Qn;r5i^YwFftS$(%CMq~H6D zxAR;=%8C9OUYq{{AYN)&2HnIA3B^8_k-3DIylGsyU?A!sa{M=AY<$J06gv&o$AOW# z3@)cxxhw>8gAN>J+!ya#PG-tYT$z>vHqI^r2IjhvG(penLwAIJl;u&&EDt*HdUOj4 z@|c}_;61Vo8VR8F&z|?s4h@Y0%qCxEtu2-`%uYQj{ZsNro3&S4sO5)Z=+NN_9TVwk zOLUVt3~;Rtt}x-E*qbYB_A_m_`?xuAGDQu0GQDrE-8P+*wzJoua7*}fxWT#Rdlaqw zXeTqWSBprR2hq^I)vKiKB^lsR9#DaCq_E@o$tPg&QsYKA$$eJA%;Wa^_$;_wM4H*0Bj%ST{7D*3QO9{osn;NxlHT`|13c1 zpa5)X0A>!(${wq8mpHv#gqAW7O#Xj!z|;Tzae`Jzod45l&+$-T23W20>4mw@$f5Gt z@85TV4Rj*RgP4Z8?S?qXJc3oZmDB?q*MYthC(eE#H_E6v)3a^Tvr?f0bc5iYSKr#@ z9Znk|GSp4U_JM{&Tv&YE`VrQ;8wj4OuohuEi_@dvN;dYkUUWKovm#-c+3iJ6Q_rF+ z5H^w$Veda9LkttXk`I*C7D($YAjaJ2cg#-xcw(SD6bRmyNw?E+M#__w2iZ^bC~}@X zongNJ(f^xA22K!t)C;2|R1R3((lzf(^J`9^C>ej<<*?D*)^0@U5-a}1gscc8)yhVZ zsvo4P_%IrvfCzO@K_AT49!Xa}5#!Me;PjR^c$($t62JU=^!u^I>VQ1x%eHj?D^hj; z$X*!jt{G2HE)!I}ZL0Co$FzASZ?bX3#Y6~YKFqhdoh+lkLpneal*cdNZiero@;c&j zVh{KeS9K}|f&`B1z-LS56ou0$R?QaceWNNJt~9Xb4xjSk2E8xyNI*IFYa!VA?f?)# zm^f&HkquWkN|v#3u6o2;e2%TdJis!AMcNnKynfaow)DJyqiAl;K+V{<^0#E`D0@} zf>h9{Okj-+azLlNjyx}}Nx5CFoaIZOAyrM`Q?I&DGt}ivo)m&?rRGDF5M+U31F3Qy z&}H<O;vmWBg*w(~y5NmlA~agE8S=NR7W;W-w(SlvU3!h1o=HU)?nT*AmSn$dP&N&l& zVI~uB4_@6ovKy{L6puEz(@yX?Go^$K;Kws+Yx>YLd_rKl_IC~j+`-AfH|i1oA+?2Z zi;{#BT1u__K%K;gzdwgzlBia&&&>@Gn{7Nw_a3dl{UTbJXFl;gb`yDm(s-Rr{Zr?y zw-Q(d6B)09wk4{~L-p`tz|&qQt@T1IYwNG+;7FI#cB#dU&9XfD`Ioq@BztVB5w-}y zF#X%Ba%AUsK29hJpXD&``d+{B8&_!|=?VAst=0-5J#{Ri0izCg5!6P^-X7yu8Zk9j z?7cQ<9FTWt?a@!-l?%FhC>E|LSxYcBnS!+tg{lwyV+7h{ru*gXM>5&GzZ+AtEyWhm zGFUd%9*TeqfMEXfDY%dzkzd{@H=x+cu|20mRA5HrcedFZM+4SghDH}&dQk0iEKn`Y z9R@0asCMK(B&>;W!{`^yo$B^kWZB~gh}MoAWqu>O=(`N0xnD$IdSvEYte3>th{ZJV z&=8a@sjYF(Zkb!I=v7>na5!wreKUW?s}bAwd+b7!4DKCGa?YrJS>D*GGSIFU=LWsj zo*r{3ML6B9<#@e75glc7k}M|hC` zFi~+`qs~z~`A0%a5b9Hw%7j3b^7i~rC+_>&Obc(cI?Q|{>tR~6pUbPq4m6a5SAThD z>YmLtZ7HfOd|y;p`Wuryycyi_5%oztDRftr9r196$OR)+=EmY13YQu?J|pG|;Ngb3 zN%}7D-Co?<4qmPPbXJN1(e88Knt7R9<0y-gCcN63f7J}Rd&7?l*pi=B{rK<5txD8D zWI4p=l|PrcUi-9I;SDR)@$Cq83bN})B(Wja@X{LpYw1CuLZE%(hFK+O!q9hs0@U2> zs^^%vdZ>5s7ctWCczNq^G4{`>y#>VHq;UcO`=3_sc!JB>g#t8m?9F|%JyNx=q59r# zkpG&Xa$zz`3q^NY_Z$Bsc{eFlx%~c*jW!-k22a;ctuU#L7uGW>wGo?mf-@}ZFR6$6*g6~ih)OQp_>=YU7@$JNY* zK(5{DHRqIjc&n?tRNOnMU&XZ+cQr;are} zLuGZr9!=4`ia8dYV)($aTdc^2RGzDN%LclkZ@YYPH>0Wrcam8Wi}z zgaFvw?_on4N~ER(WC$bdec%be{q z{G0P$NK@kLx^8x$twLQ6>-sm?XJsye*IKr{F?KWs5@RrXT8Q_R4s|C(Em;(jJ``z5 zZr#e37;NxVRps?`E+z2Gw&9sgJGWZ-Os1-<|+RR}i&9eIM z#!g4xmbPf6j<=Z%RNIDKDO+#Xf5*-~{Uuf?z*4C!nb}cI>)2i+OhHJD0>h zSB9)Y8HFA^b|uRJ&K0l+hl*pzxZZ0+#VVcB&ug+hiuE3XlhkhYh;tSMBn^FJqU(VNReaB(d#qVriifEhlKhm_zBO7Yh5Z zdX2EIk4>utSAb=dZ{g@uV4g;XoGnHyGKIJJAIBNlE+=1mCP4eqw^j-}-mlApJXeWU z6jzC?mFA(?*JT&^as8q@l>XWs@9mNapu-W4ovSMl%|hq$BmM3r z`4s=!b%C{8&eME$y_+>cy3{h-d;frBUzK%57;p_KjEAXUvmtuGtP+I)uSUm1>8vPw ziSj6|ln3!rzubGCcCex2<&OOh^9d;%M2=Q058fMk=m6$Hal>9b;X5XFA{LfA7PKCppd*C`h)FzL*)0QZN zWsZz-5xtw$b0v#&)M<*=!1lf4YYK}!@;(+^+Ww{)#%c}s=yXNe?&iwK^Oj-Eg4 zH7)rIqPbjW@zr>(nl-&M9e0-GgGFDqxOD->p|wzT!YHsJ_ts7;GGkP(k5rjB`hCmH zF0Y?88@9DKp!KLt+fv^+u8NB%r*9;kKoO<&!eC8v!RRR04McO}+(8aoiEc9Ma?;G> zD=b}Te35;8=qTdRKayb!T-Rl!Sl4O(e)|kU*&*}KzyKkpobd8*4R6|%zmoEZsaD}< z^c(9H$wfIYm%`x2cwZ%O63#6K&Wq{dvILK#M_pH@t}y-``_l8gj_jtKUB8PjxEa)x z*f^7xUQxSTSONbXuxN)n0^^_DFG^Z)+R{GA;psNs-R5u#@gBPmOZL9H~b$7 zs}#_VEXlyo$s$>HF)?%m z0WR1%8T9h|c_TvNWi#P5n;;+gFZb-(10z^HBGL=TkDg11K-KddtsYL-xRy>g-Ha)7 zJb4mUe_Q;%ok+;_IBgz_`Amo)2LE}j62$KC5#sjhKTXc+_bqAUReaVxo6T$LKNb(| z1e#y>$J$J6S59|1>Cl1ta!9+Th|G8qm0p>av_um?_il{v%6vb=H%e9# z@-K5zM`1~hEYl_ygj6pP`K!kox&H1qU5u7O1d?TSyv>b_w!s;Y^_1z(%fcoQH3&qM zF_jW=uMxm{Wz;2kl|4}nlj(VQnV6xa^!)3|FA?armn_P)B+GSGeB^M{Em5x`_` zj$BWmf_S``hfDjNuLPH5OH2bFCM) zP?<-Ve$|O%F0i!N&s1u?8k8dC1SO; zZrVO2?0I?nw}M)X{R62f9RUff$-7tg{I|c+c6wldG>C7p!-I7Y&7Lf3Ea`ow6Ce%M zr&&9+=$$>$+Gzgj`bul5*ds@iEH{s6J6TPnDS=q`ao>X7G}%FYYmff|YlO#T4UW>T z|8SF$=)T1=BFLrTK;C#+Bmz|_3!I%^BxK_Eb+Aia@68u0-<;fw;At34E_59JM-r5q zziY+2u3&jrdyMOFf+Fz*B;h21Ndg$-F1JVJ`-!?=lJv1cQCjF4KRS%A45aO1}@dMi@5NiKRp zdq{olqf!uekZ0Zv1o!m%Y}2L0z@jpuXgM5L@AnDDk z$_t@>ZV5}xalkr{ZN%0)6f5uPkVb2IFQ(tkxjR7RJ8DD-z|u)4gyRTve^hbSG5m-b z_8%iU*eW$Ut?5hK6YY(kB;!9A-2?*mdI5&Bj~c?64!VTtW?*_WlF)pj!Jfx?0fnT8iLw3E-P zIbfbOnkuq+)P?F7VgwI$Lgae2V4qn{9aUrj1PY0mGj}L7@41dzv6yV_@uK_^WGp~F z@KG^xV9tsp1u1V-h!$ElTYQu3`@VBh+L033&#KaqVXyW639HwOTOg=WWa!ny-5gS& zqF!0moN!&#!*R!mCUN?pl1D^pu#&u9!4%&@j}NUn{pWemxm1wQfpkVS^d*hKynVRO z@a+uyyGXumwzolvl1mems}X*;GcXBJR-BVf8LoGxrD+H*g2}U~c5i%WQiZEXhq=^h zZGp^Yx{ueoi@2A6L`<)7=2(7-Yskm+q2>`e`kuo3PI=lr( zHy*60WA9e{Jbm2Vqi9K;^SunH?OO>G?I*qGXK_#QGi~sBxPAan*m}tHo9`D&*qg>rRD1CsqNl-y{(ZcmHZcG;Yvh(d?pNl8>8m3C3mqqb;3N3I0U+8aV zbSsIC;KznS;Np44!c98U1fv+K09(;FMvskR=FC<`XE_;ogPv%ma3K!`lZ)t@h9C*+ zf=sdoIyV(<29*Sm6_>Nfx#5eXH!yO<{3qPd*8sf zKmae>fMaOg$k~;{1${j}5XYZ1)bHxV)A(j)l~qw}eNpMQf^VaO0$A+(=to=AxyL_U zbto-6X0z&muHxMqNJ`JV)xGAK)79(NYPR!Y(py<&`+OKV{dj0A>MOYng(1 z3OMJFh(aI=h>dCTqW;Y7)5R4h>ep?K44-Z-^M%|Ajrh=8r(DQ92J4L^r2iv%cGM3e z2mm!L22cPzJD>J|F9enqS9s=&zFXhcs7G?Qk^MNiO#GmpxR6+$u=Y^zBA_y=f#8o; z5=2Mgt~%)(+}~9=B|Dxg$Gn5l8dAo(`rd1EVC4*05`HR}Lt3?ZgZR82Fmg$clMqUg zK4v9Z?q!34UABnZvfk<9H3OUIRIP_HsSlDu$BQ!E`dt{{AosOV4`e#3Hj;P7Tz@JK zQ3Xt<6-I*dzbv+ERf#8U*2?j7{U7a}XFS{Q+xMfWnkltcv~*gfMpcT|s-hG{t+Z5) zm_-FKYHvkRTdRs%Rk35nsJ%yHC9y|H%+&f_-{1f7eZQ{je%(*62cG4Dd~<$}^E{8^ z^Lc+jw6HiBvr~Wo+zgr_y9_WmA`a%H1z^uo$rlEH_GM|_xGz@MCIs>TDy<^s@c7~2 z*DDsy&10ohhob~@wTNB=&}R!yE1pdI$k;5qCXWE65zm#g?9=NN z4qWrsYsXe5CB(T@C%nASs1J$qJl5*rLb09kh;84Mx}BJXD_02J@wpua4sqXwXJ%%q zf6V$6vT77P@ee5n#{Bo@8!X|3)sSkvu=j#=G3~rdoEiSJQ-`&Bgj-g657cM&5-LHz z?<;DIolk%~5C6xG3BBNR&=?9BzC7su@NELBRp>Z*#8vYVJ})?t+7bCxQ$!{}yUKN1 znkM?9AEB=CztJI- zs4?PI6n26D;hE=7jG&S;-;3r9)zdk(`{lCqpO{n0T9UoxORFOC42b>xir{9h zDpCbeopd-TPTi5i(eO59+jD^}%*XJ^gEmQl#@$te8VwIFq-{hDG*{tf{C$1E;j)Tx zET+&(xDu+xDOzW>Uu$yCutp9gc2^I{x;CkdY8{2@l2$hQFh^|m&Ghk}Rj6@UJ(T6p zN->k*Vy`}j#_*5IpbLvRcc~z;&HNNaGZHusX@d8uF4{&uEK!BKVZT*Ey?gFSrwhjk z8`959bL8lyHN!jLqeyx3g~O9JD{+!sCjKPu*-58wSarX+BT=Lh^$gZ$nnM3EYcW_V zKD_Av@S=XH171K$Dj-w4ypga!d0?s6(R*av5Rl-_{M8IA%CRSs^rS#y%k43(K1jjk zQ-&)hm(d>RX>CLODwHaV*1a=O8CD73r*)aLc0>6x917li7Gmt`)aYZ~qZ-h3CoAhT zkpg1yU_pe&$@4~1$7q|Khi!47okb^)wH4M2ah-`eLEX0N-ju9oZVazc4t+BzFcK*% zmycg87BJLJ&>fE}J@$-c_$890eizY;d;KXZc2$N7fC^RVg#;`VJJ$LR)Mq?H8;c z^#8o`Nv!9i4Gq)YdYaG4Q(do{k$4zb)42eKa6M;@lWXUznJJtJ@2#{h9B`?>Z;?Dx z;ErAGIGQ-;LTI;djoDQ!<2$^>W2A@C= zdvWhSyfKeei=Br-LqRj2mLI?V%`GDMwTZRG=ERG99k_!2f?CVo_ngc^)DsWa2Hs!k z-+k}HvrFFf$$9S`2zK0)jD?|OS38~MRjU)8TC#2}pE&PV4y~LuAN9p6qga1BE83jY zDfUs={)=%bxmc zGCmhm!2ske-;6Pl86`Ss&;%Gy(5YkSs!K9A5EcE(Af6jA74P-N> zjuJ+KV}Q)uQ$Bnkx1*9E*(J@cSnpxnLiho^j#%+E<)VRFb{R-2W$X2z-)JDlaLy8` z12o?Q@dr6W#v|sBk|r5f;VZyJ>RcDy~rL<08E|$xQ!PQwls`7 zyK}5PURO6BH#?T5QnnI6ytuWr{5bZNv%nKN?~^xH2%B@x`6h<7uRFk6qS$bFPw*^E-*ZfQeePPen1-Pi!t6gqrSGKjO&83bk zrfnf_*(kCha4{mAsZUCxW&Lajlc-=UeADt}K=-|_FG!5bICm8)r6L*g5tW+JGl(w}YCSI$lq<-v0!OuxaH5cffbJHh8N~=dVnNZ@r3nVo&Vc z9S~gGeWSS>HCt-F(n{lO6Jn8_&{YQ3A4Oi}ih*90Ybm0dGtjy@ z?naCcIRXDYs$g~=0;V){jlGb9{VsE-dB#%^*>ERZ@%hg-lcL3V1RxT-Gkz|HG-@(u z)GCa+c|I_*pLsvVM~+3XC>WqE`1M)79rfqkvm%ShbLv}aSf}p(5#N3D0*ZQ- z7=BVOEMR}3Vyn7HlOV|sL&{C0l%o|1eGaef`4VZd8bi$OcH0dpU7Gqn>sF#(PKVxx z<~<40H|=NmV?#o$Moyq@T$nX4y%^q>G=caC{+b3IX$5BgVl7sqIF+SPW3lc$?h4Kf zySfamH_p3HI@1&0J=4m&wDzB-`u}SRLvuc9rK}k67I=rjbSNKyzVf*ynHlF-fr6z>9WRq9(KYZ_ z+5iYu+2T7QA5wYJpYaPPh6^2l$mW7f6jvZT?Kzwc5%}!H*MYOl&lgEq-2KAShuo6) zT&q`_*ba~<=Olu+#z>>vqWi&3CvdS2Ytte z)B>L(iw?sfi&EpST!4=Wl0B)DJVwotM#8jJxZ}g8B+#_qHqCT0qGz%VJ94h_TJSwy z!d!F_$Mgkt)T~ldFLNt^@K6q%xDA1`_$>RYcx-L|3j@Zcr`UmlxA$Kp3?|A~Dz^g< z+XA6!0OQcZ|4E%3dJ#;}erddW$cM;Jl2$Fz6^`s;NKBf($$j%$)2|c!c{-{4uxEIm znZNHWou%4FjQ(B}uk6RJ6L~P2>pj&rXwY}+)9LGfKuPtBfhR1mz;)GOS_nUa&Vu4z zb|ZzrDD$fsAzZH(zakoDpJmGp)#-gC7+$||1i>DiKa~sl2V^}7#Ke9r!4y&dAf;g- ztdtq;aoL9wgH#3UTTxy`u!Tpq&Q}!Li&MSTW&;~o{C~h0Pj8Zyz+UVj`V$AP&AjSVsS$Z!ZRz(&kCS?c@)uZtnK%Z@8t=FbM})a zqvjZ~syjV*T;6>*dE4cCVPXm}!`kI(;w=rIlfYq~73>I&A>EqHb2jID=BYrmnlWSj zj(Eo7%WS_~9$g5!bajdb1innZ86ul|3G+?6 zwYA9~u1TfKWaC;AU%T&WSej|wBvRIf1{*q6)iF83?w0ekV?%zs?~0I{3W%zw9HL3N znNNY?`_HfCYLB1|i{{`{4{T9_SEm+7%kpY#-vKTTGMO4$s;Sa(t8He3YR}(g^PGMX zE%|LNBd5^z5H9X`vEgUhfaYF4GpdFo%F6HV#iq>;*rjs_MO{e<(f%7tsZtqzRrw$w zk)?2Trqj#oFZyy8!-c)D`%<;2HE-9d_aG((Z`If%Ct_^dYnjY z^b`)x_;<+oawk@ICA=eD+2hAbYb>qtpbV5`+F@@o6^wz#J@ zJkDxmWypfm6+vJvdfaT`LLN-gbQehsZ|zM73;8WkFGq_`AYDt53eL)%4`kCq%4Gzs zyNK6D4j9dQ%FS#wMDI54UH_9x?IxvBcaytywHL5$HUA*nZ50-7niXtU;k$tJOLONt z8)Hviix54NOW)$6c#g^kP8tuo)IH9HD0Z*EBaXUP|2?+uXPaPrHA)+NEh!-|KLMo~ z#_%W+zsQ%%Z+TXXvC1JI0WOM}r#pI7%)LahlHo_bNb)ch%)+sSIsk&u>w>VpMGvn4 zb{a_xXS2|+F+&5Ss#xe1hQV0x?%Xu@+4or?S-d!}ILY>71z*j`&V6_6O|Nmp1(S=&Er(~<7U9sbN19<*{HI9%oQi34fkPG7kd+=V*Bb6 zF68%)(dL3OpsZwd*hAWrQ$Oc$j=pL#|LjrPEsk$+ozw6!RES9IW{rRNw+tK^$P85g zCZGQ2Uw}=Z{TLQ~de83L3_t`ES2j;MfZvYd`_@?Pa5p9eSri09?^y7IkgF!BGdjZw z;*N_L06>}X<3?{5JL4QU=b?+pP^LSp^?;dl-*sz_z{m&#W;LAG(lER)mQtG`(z?*m zxd^o3-Hvym*+#d91Wh$?fX#_|^^WMDa{)vkMo?n()O*q29P>UE>l(Ht1jsZBE#|H_ zO>_*7d`Z%}o}`}_{NyEj78C1$F&U_aNW~vWE}+~9T+xEDSk5{ctQ{TOR(JfGx%Y#I z(;mI*!zJ2A`x|E?&-IK!{yBSm{gPu+vcC=lTCVGjLKotfo-eibP-IFy4WpSs!xY$FeMe z(zeVmCwWu%tcFOnn0Y>OL^~{U3`Tj|;W#fanzTJ;98OrbbKrGudGNRE8VE{RXP=VOc^zVPe%+Lg;*{=(b3hV)oRozV1Dv104y z*G}XXeiK_9EKG@!l1PUEUy%rEP_HQ?&<%U}b7HsO*L?mV6CDBW)+c;XFf_?s`vu3Y;|#tzQCJU5#tq=lb*e(bFtI9o8#0g45I5zU_4hiS{4Wh zIZiFiqXb$8vK5cZ@*GsKcNd(+PvkmwJM%O|r3Wv^+)Yl5H3}q^0a6P|{6fcof9X~7 z+OQX;R3@ga_{m3I_QccX*B@FFAH90kUJ#9w4ps*tN7VhiLu@FI7WJadK#f(&?N|}4 z_pw*p!FZI-*;n?k=SoK@;(vZ>-@BlA8lu-6K5ns5|4OpzFu@yAlS_T!TIKbum2 zLj={zL{1<)G-*(d>s)j~f(j-TsE67*m6c8CvF$Rkos^tW0)j`l?!|o@KGp~j>B%6o zKe+7l_Q((h%m+7%2-yd-z~?#9HHqRV7^{H#+qeR$9#?PPxnACg6AgMEjdT7%NZ<06G(s3fot!p=ON zQRd0P_pH1DmFTf#7h?&SsttE$h`~xmp|s7rnyV8Gxo%%a#nw3|xXS1l9bAp#=`PSC z?JP=B;Lb%ia)C@Q@VvzaXY#bA<$N_cN!#a%I49e(J`0HEmJ>^TPldwQAjJiiQ;T}i z(5Z;r`+3O7chX8-QEh-gJ(u*$;81o?8~LpALXo>^%2LEwr$6T-^CCcaWZ{PKdjJyn zJ+oloo2dgUM)HsNtCaV@jd+-1mrgt9rjw^0iuM9VfUlB~zMqtt-;z{<$JeDdLW~`v z<#SgeL;W7j)w)(`nEua#hHB8u(rJ>o0`@EM~+46|x!ZZf(ex<_yHTW$5M@5@ll z%XfUO3+q_(~S;V2bTsT)_$_a*XG^e^Oq*A=T-T3S&X479pzk zMGPa~yy>%u77?@eaSUw$Iyy%A3Zm`wkq@p3qzz5h^S0x!B`_AhvF_6R2lRCQ8{3mh zz0F;98Mmm1n{OknGYO66dPNi))v;eRstu!OvDxK%(e?T&7!dMn!quT@LLBGjWpd`d5o!-7TP>3*p3{+5D z#-#R{$~=!5kA~pG%2|BD3w!S8W$RcE#iI9>>R=l8R+1WsjHgAdHdQg*y7Z)`iMrFfvNfYS9!htt!@cqJ&0?9>KOn}MjUmk`r+pYB z#T%2xyIff{F1bJLLR0m)TOvA1hpfg|7cblGWO5x6e2@<1yBVE2^nk`Y{~5eR_I};% zmbd-eCxr@K4>#F@9tuL@v8PJ6NJ{9M3=UQlCF|tAr2AE+U2PYPU=}qfQ)9XG-F7*T-;D zqMCwk<|0W8QrmPu#|H59`yK&jS%-wn2rEgZCk&(`hOkmjo^HcqJZQ|1%V^UxQ^pQC6G$VdoaZ z56p8F!89%+E#W)4HWI3d6!S8yXm|E{D6zXBXL8q>A(7pn^b5$cw3M5J&b?x5sNh;0z&D^`dF9?U76#(EQ(OgzNkHWPPrTTo;FSBJ$Rb5du zc6UaKkfH&6{_G}@Ql!TCK?OC%u6<11nIj@%sV-^M{zYj2TYn-tRS37=3W<9M{EvCa z$CNhZjvn44Q_^5CRArq(yL{$)zxO>KNA`P#S+7LGK)IrYI_`!Ta?PghnAc}d7!$$| zgAK2IJIRIpp2@Tc{a6F-H1f&dt|jv9E?NY#VE3YLQP=B*`0huS%7Ph!Fv8w3`yT^^ zy*l;qfn4xh>eEL*%u65Ks1VG5btr!*?#21V&Vwc-McQ0@{4%%~7c4romd65^P=Ey7 zZRG7z$24Be2v2kIn=h*5P5G^{B3_9{l^;`uvi4i~ zO+b38__va8;cs-*PG| zEyHCJ-7(xDd?FntmpUn{Vn+Xf-hU<6c4?vA7wu0?ynykZ*#^=5Fs6!Ko(<#ccJYsO zj>8p~i%KemS*!Pjhi>Ju>{(Fiy^0VLfea32ED88HmW_2+3F3aG8`0xh(B;NiT~Sa| za0-9cO7+ZPG+7v_-$GuPMzs+AW;n-|B6`!d2y&rnPhj`n6Z4h*c!*f^FTmYyh?{S^ zR>G{>o_F{69N|m^@_WwR1O?-V=hBONRRGHSH6?(psixUDy4s+9qVe%v(znT~3%_pt z?tKD=#((~IsSJApRELNN*|KM=dc*sQ{=x>8yj}|tlk<8%2SEH6&sH-`GP<)``(@3F$JPv>Aq zhM822oVpfXZ%Pd59Mu%xlU7>meL9*Tx{7nLISO%4D6?#!;0dBbp{c+LL_n-7j_J$! zcx4MVnK@S*pj1{(ZU53Z!6MD6k&$>cJ7IiuBpYh!=hDj zwjNigj-i60Y0bT^i?l|t=!USCATPjQXXeo0l@nF!$hsjdqAdz#hRi+rv*en5MjY6& zcmY%we6~MQp_nh-vrO6S1Hs<**Eqp0TK)A#X?ZNmr8l;(N>kj212@K3|NF&@Hx^K|XTWxKs+ac?VS+Ie zNQ+#0Sxql<;&Zy1_-0R@UL^fD&pEh`O3{c-tG}Ak<~C0|dV0H*;;Kr%ky`*ADjX<^ z`BYcE_ssGIE(cn^?bcR*2r|h9R%kK~7UztEFmGf$EDS*_`U#f755+YIOJz-%~m3j%J~~g+!N(&>-(ZZs7%Cl)m`+nDzRQ5*LRQQQQLr< zkv?5f3z3!Zc1rSqqk42PeDQHI=A*+%>!!2K8_M6S`hRI`6th2PWvxc9xSk7d_yUMB z$u{Xnjt$1y{t{o|V7ua1rh+yUv9GYGiQY+t?&q?-e#8zu$?tpEW9Zs@HWck`v%aw)+U>G9>NK8j=IL;g4n_B zmqH(wr9_)Lx-P^MQB;_6(<(Uv9Dpa1G+UK!k@bST(mmlZfnY=$Cha(`D~5R~Vk?ca zuR2zmQxcV%t`=7RTY(?UHX0za;E;NJo^uWvu0Sf3K@D{AfAiAC2#8izw1$F#(W{H{ zKIu+Lolyh!*z=K`8^6e2HI202)7W0F8`w_*#z>*6s%p%!^2dDijYN za^UkRC*AGDe5WeXzN=VK(vT;z&%5`h-7aBi33oJr*oR#{;11v&zTWlqlDmc!3db4_WdoH7y3yvU7NCfBGxiKpDUSqL+piE z=<~oHt{(yzNU1IO3P$*(AmoAnN;*HbHd+?UscPr%3>`@Y(^gp&?Jiz-6Xcp3lKl-s z=(Wy+D2(}(oK>KyC^6+y-8U!hvN|qdjY_?h8^xe{EqS+?vHBD8ui{TyH+dk*$d=5R zSug__;R7IK-IYwo8hyu%h?#2*cPnN!;eL^+y7@sbwYm-9fOR|@cmLxW4HpvV|OVV0QiMui%24Vbusu$H+{glhw$9xvVsFHAD1@K~z?VnRVUFe71thz5|FhfF zWXw3!`dTHIH|Kw!?-))qejmN9CG7Puv{%pIuDbbI3H14Qi4uIKD(;M9TLOBs5M@G? zJ<4|E;bY|HQoLj}Gcr^R=s&I??a@Fs-9E{(Ho{YY`jPhXIH$M-eQ1vT<8%j+ls~&* z6EOtnGKd2bsdU!@(TI6ql~Vri#waP%1a+wK9S>enoc?0)o_cw(vfhGZC3iHa3-ZKz zM<5jl;6D`|Il@4P)3F!R!7G<E^%TSYvOE>4ex z5S-01KM{?lWVItUM!%EBNN1vWo?EEUheaFVRLH%1R3E}N;KG`el?LSxE~mnf4^Sz& z1V0hOxzeGcow`)1u*A=nafQuXj(ek3`;b@@Gguh$Q~_|KsgdPk?0lyVLSQV~6SL;L5ZC5DG`Y^^GGl@vi#s>&M0^rE7=x=6m?e&x*6S-`G!c zsi@GL5%Yt={m8<_h!B(j7p4UaRsXUc^EH=so!QH)unqdFjlcNnV;kwJI=XV6cu`3s z$^jwn-9p#M#>GJ*+O(#}M_A3M^`n%O2$kubTM&s$P_RWt2id#BU#}|%LaIsdqb)U_ z7eK)kHET6k)`sfR>8ReS|_vvPx>3?5v< z-zU~3o9U)}uM%{b#Ygb9HxR~{U_g$DuAY00(Hg%>R+DNgap1cA#A%!3@-=>Xn#jJm zwv;xeXJE$xU`El!bIM29l)Q{^#hBd)5xd*t`Y@#hyIPBme-xpmw)eY(wY#_P@1BZa zL^(W__h>7@Unv_;RK~bT#-;@H74~^tt34J{{5&KU*GR*1iQls#XLIEA1sP*Vmd8v( zuW&|vUCZIGZ8V=2M@_$F=Ko*{$>> Listening to microphone input") + print(" >>> (Say 'stop' or 'goodbye' to end the conversation)\n") + + finish_conversation_tool = { + "type": "function", + "name": "user_wants_to_finish_conversation", + "description": "Invoked when the user says 'goodbye' explicitly.", + "parameters": { + "type": "object", + "properties": {}, + "required": [] + } + } + + options = { + 'turn_detection': ServerVAD(threshold=0.5, prefix_padding_ms=300, silence_duration_ms=200), + 'input_audio_transcription': InputAudioTranscription(model="whisper-1"), + } + await conversation_client.initialize_session(options, [finish_conversation_tool]) + + loop = asyncio.get_event_loop() + asyncio_scheduler = AsyncIOScheduler(loop) + + conversation_client.subscribe_events(asyncio_scheduler) + + speaker_output = SpeakerOutput() + conversation_client.subscribe_audio(speaker_output) + + microphone_stream = MicrophoneAudioStream() + microphone_stream.subscribe_audio(client) + + microphone_stream.start() + asyncio.create_task(conversation_client.start()) + + await conversation_client.stop_event.wait() + + microphone_stream.stop() + speaker_output.dispose() + await client.close() + print("Conversation ended.") + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file diff --git a/src/python/requirements.txt b/src/python/requirements.txt new file mode 100644 index 0000000..d52c88d --- /dev/null +++ b/src/python/requirements.txt @@ -0,0 +1,8 @@ +azure-core==1.31.0 +sounddevice==0.5.1 +numpy==2.1.2 +rx==3.2.0 +python-dotenv==1.0.1 +nest-asyncio==1.6.0 +jupyter_client==8.6.3 +jupyter_core==5.7.2 diff --git a/src/python/speaker_output.py b/src/python/speaker_output.py new file mode 100644 index 0000000..c1b37ed --- /dev/null +++ b/src/python/speaker_output.py @@ -0,0 +1,53 @@ +import sounddevice as sd +import numpy as np + + +class SpeakerOutput: + def __init__(self, sample_rate=24000, channels=1, dtype='int16'): + self.sample_rate = sample_rate + self.channels = channels + self.dtype = dtype + self.buffer = np.array([], dtype=self.dtype) + self.is_playing = False + self.output_stream = sd.OutputStream( + samplerate=self.sample_rate, + channels=self.channels, + dtype=self.dtype, + callback=self._audio_callback + ) + + def _audio_callback(self, outdata, frames, time, status): + if status: + print(status) + needed_frames = frames + available_frames = len(self.buffer) // self.channels + if available_frames >= needed_frames: + outdata[:] = self.buffer[:needed_frames * self.channels].reshape(needed_frames, self.channels) + self.buffer = self.buffer[needed_frames * self.channels:] + else: + if available_frames > 0: + outdata[:available_frames] = self.buffer.reshape(available_frames, self.channels) + self.buffer = np.array([], dtype=self.dtype) + outdata[available_frames:] = 0 + if not self.buffer.size: + self.output_stream.stop() + self.is_playing = False + + def enqueue_for_playback(self, audio_data: bytes): + audio_array = np.frombuffer(audio_data, dtype=self.dtype) + self.buffer = np.concatenate((self.buffer, audio_array)) + if not self.is_playing: + self.output_stream.start() + self.is_playing = True + + def clear_playback(self): + self.buffer = np.array([], dtype=self.dtype) + if self.is_playing: + self.output_stream.stop() + self.is_playing = False + + def dispose(self): + if self.is_playing: + self.output_stream.stop() + self.is_playing = False + self.output_stream.close() From d563796a555ede0d5c26ece3d296ed7bbf3bab1b Mon Sep 17 00:00:00 2001 From: Marton Almasy Date: Mon, 25 Nov 2024 12:14:25 +0100 Subject: [PATCH 3/4] Conversation client, microphone helper files added --- src/python/conversation_client.py | 112 ++++++++++++++++++++++++++++++ src/python/microphone_stream.py | 40 +++++++++++ 2 files changed, 152 insertions(+) create mode 100644 src/python/conversation_client.py create mode 100644 src/python/microphone_stream.py diff --git a/src/python/conversation_client.py b/src/python/conversation_client.py new file mode 100644 index 0000000..bffcef2 --- /dev/null +++ b/src/python/conversation_client.py @@ -0,0 +1,112 @@ +import asyncio +from rtclient import RTInputAudioItem, RTResponse +from rx.subject import Subject +from rx import operators as ops + + +class ConversationClient: + """Manages the conversation using RTClient and RxPY.""" + + def __init__(self, client): + self.client = client + self.input_transcription_updates = Subject() + self.output_transcription_updates = Subject() + self.output_transcription_delta_updates = Subject() + self.function_call_started = Subject() + self.function_call_finished = Subject() + self.error_messages = Subject() + self.audio_delta_updates = Subject() + self.speech_started = Subject() + self.stop_event = asyncio.Event() + + async def initialize_session(self, options, function_definitions): + await self.client.configure( + turn_detection=options['turn_detection'], + input_audio_transcription=options['input_audio_transcription'], + tools=function_definitions, + ) + + async def start(self): + try: + async for event in self.client.events(): + if isinstance(event, RTInputAudioItem): + self.speech_started.on_next(None) + await event + if event.transcript is not None: + self.input_transcription_updates.on_next(event) + elif isinstance(event, RTResponse): + await self._handle_response(event) + except Exception as e: + self.error_messages.on_next(str(e)) + self.stop_event.set() + finally: + self.stop_event.set() + + async def _handle_response(self, response): + async for item in response: + if item.type == "message": + await self._handle_message_item(item) + elif item.type == "function_call": + self.function_call_started.on_next(item) + await item + self.function_call_finished.on_next(item) + + if item.function_name == "user_wants_to_finish_conversation": + self.stop_event.set() + else: + pass + + async def _handle_message_item(self, item): + async for content_part in item: + if content_part.type == "audio": + async for audio_chunk in content_part.audio_chunks(): + self.audio_delta_updates.on_next(audio_chunk) + + transcript = "" + async for transcript_chunk in content_part.transcript_chunks(): + transcript += transcript_chunk + self.output_transcription_updates.on_next(transcript) + elif content_part.type == "text": + text_data = "" + async for text_chunk in content_part.text_chunks(): + text_data += text_chunk + self.output_transcription_delta_updates.on_next(text_chunk) + self.output_transcription_updates.on_next(text_data) + + def subscribe_events(self, scheduler): + self.input_transcription_updates.pipe( + ops.observe_on(scheduler) + ).subscribe(lambda t: print(f"User: {t.transcript.strip('\n')}")) + + self.output_transcription_delta_updates.pipe( + ops.observe_on(scheduler) + ).subscribe(lambda t: print(f"{t}", end='', flush=True)) + + self.output_transcription_updates.pipe( + ops.observe_on(scheduler) + ).subscribe(lambda t: print(f"Assistant: {t}\n")) + + self.function_call_started.pipe( + ops.observe_on(scheduler) + ).subscribe(lambda f: print(f"Function call started: {f.function_name}({f.arguments})")) + + self.function_call_finished.pipe( + ops.observe_on(scheduler) + ).subscribe( + lambda f: print( + f"Function call finished: {getattr(f, 'result', None) if getattr(f, 'result', None) is not None else f.function_name}" + ) +) + + self.error_messages.pipe( + ops.observe_on(scheduler) + ).subscribe(lambda msg: print(f"Error: {msg}")) + + def subscribe_audio(self, speaker_output): + self.audio_delta_updates.subscribe( + lambda audio_chunk: speaker_output.enqueue_for_playback(audio_chunk) + ) + + self.speech_started.subscribe( + lambda _: speaker_output.clear_playback() + ) \ No newline at end of file diff --git a/src/python/microphone_stream.py b/src/python/microphone_stream.py new file mode 100644 index 0000000..6b8865d --- /dev/null +++ b/src/python/microphone_stream.py @@ -0,0 +1,40 @@ +import sounddevice as sd +from rx.subject import Subject +import asyncio + + +class MicrophoneAudioStream: + def __init__(self, sample_rate=24000, channels=1, dtype='int16', blocksize=0, loop=None): + self.sample_rate = sample_rate + self.channels = channels + self.dtype = dtype + self.blocksize = blocksize + self.audio_subject = Subject() + self.loop = loop or asyncio.get_event_loop() # Use the provided loop or the current one + self.stream = sd.InputStream( + samplerate=self.sample_rate, + channels=self.channels, + dtype=self.dtype, + blocksize=self.blocksize, + callback=self._audio_callback + ) + + def _audio_callback(self, indata, frames, time, status): + if status: + print(f"Status: {status}") + # Pass the audio data to the RxPY Subject + self.audio_subject.on_next(indata.copy()) + + def start(self): + self.stream.start() + + def stop(self): + self.stream.stop() + self.audio_subject.on_completed() + + def subscribe_audio(self, client): + self.audio_subject.subscribe( + lambda audio_data: asyncio.run_coroutine_threadsafe( + client.send_audio(audio_data.tobytes()), self.loop + ) + ) \ No newline at end of file From 669de2db79f99826acd3f87c58f2fb6ee46dc869 Mon Sep 17 00:00:00 2001 From: Marton Almasy Date: Mon, 25 Nov 2024 12:16:53 +0100 Subject: [PATCH 4/4] Jupyter Notebook implementation of main.py --- src/python/main.ipynb | 102 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 src/python/main.ipynb diff --git a/src/python/main.ipynb b/src/python/main.ipynb new file mode 100644 index 0000000..2cf1f3d --- /dev/null +++ b/src/python/main.ipynb @@ -0,0 +1,102 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import asyncio\n", + "import os\n", + "from dotenv import load_dotenv\n", + "from rtclient import RTClient, ServerVAD, InputAudioTranscription\n", + "from conversation_client import ConversationClient\n", + "from microphone_stream import MicrophoneAudioStream\n", + "from speaker_output import SpeakerOutput\n", + "from azure.core.credentials import AzureKeyCredential\n", + "from rx.scheduler.eventloop import AsyncIOScheduler\n", + "import nest_asyncio\n", + "\n", + "nest_asyncio.apply()\n", + "\n", + "load_dotenv()\n", + "\n", + "async def main():\n", + " key = os.environ.get(\"OPENAI_API_KEY\")\n", + " if not key:\n", + " raise ValueError(\"Please set the OPENAI_API_KEY environment variable in your .env file.\")\n", + "\n", + " model = os.environ.get(\"OPENAI_MODEL\")\n", + " async with RTClient(key_credential=AzureKeyCredential(key), model=model) as client:\n", + " print(\"Configuring Session...\")\n", + " await client.configure(instructions=\"You are a helpful and friendly AI assistant.\") \n", + " conversation_client = ConversationClient(client)\n", + "\n", + " print(\" >>> Listening to microphone input\")\n", + " print(\" >>> (Say 'stop' or 'goodbye' to end the conversation)\\n\")\n", + "\n", + " finish_conversation_tool = {\n", + " \"type\": \"function\",\n", + " \"name\": \"user_wants_to_finish_conversation\",\n", + " \"description\": \"Invoked when the user says 'goodbye' explicitly.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {},\n", + " \"required\": []\n", + " }\n", + " }\n", + "\n", + " options = {\n", + " 'turn_detection': ServerVAD(threshold=0.5, prefix_padding_ms=300, silence_duration_ms=200),\n", + " 'input_audio_transcription': InputAudioTranscription(model=\"whisper-1\"),\n", + " }\n", + " await conversation_client.initialize_session(options, [finish_conversation_tool])\n", + "\n", + " loop = asyncio.get_event_loop()\n", + " asyncio_scheduler = AsyncIOScheduler(loop)\n", + "\n", + " conversation_client.subscribe_events(asyncio_scheduler)\n", + "\n", + " speaker_output = SpeakerOutput()\n", + " conversation_client.subscribe_audio(speaker_output)\n", + "\n", + " microphone_stream = MicrophoneAudioStream()\n", + " microphone_stream.subscribe_audio(client)\n", + "\n", + " microphone_stream.start()\n", + " asyncio.create_task(conversation_client.start())\n", + "\n", + " await conversation_client.stop_event.wait()\n", + "\n", + " microphone_stream.stop()\n", + " speaker_output.dispose()\n", + " await client.close()\n", + " print(\"Conversation ended.\")\n", + "\n", + "if __name__ == \"__main__\":\n", + " await main()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "speech-to-speech", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.4" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}