diff --git a/app/src/main/java/com/google/firebase/example/friendlymeals/data/datasource/LiveAIRemoteDataSource.kt b/app/src/main/java/com/google/firebase/example/friendlymeals/data/datasource/LiveAIRemoteDataSource.kt index 417d0d1..3fbc660 100644 --- a/app/src/main/java/com/google/firebase/example/friendlymeals/data/datasource/LiveAIRemoteDataSource.kt +++ b/app/src/main/java/com/google/firebase/example/friendlymeals/data/datasource/LiveAIRemoteDataSource.kt @@ -1,6 +1,9 @@ package com.google.firebase.example.friendlymeals.data.datasource import com.google.firebase.ai.FirebaseAI +import com.google.firebase.ai.type.AutoFunctionDeclaration +import com.google.firebase.ai.type.FirebaseAutoFunctionException +import com.google.firebase.ai.type.FunctionResponsePart import com.google.firebase.ai.type.LiveSession import com.google.firebase.ai.type.PublicPreviewAPI import com.google.firebase.ai.type.ResponseModality @@ -9,26 +12,57 @@ import com.google.firebase.ai.type.Voice import com.google.firebase.ai.type.content import com.google.firebase.ai.type.liveGenerationConfig import com.google.firebase.ai.type.Tool -import com.google.firebase.ai.type.FunctionDeclaration -import com.google.firebase.ai.type.Schema +import com.google.firebase.ai.type.JsonSchema import com.google.firebase.example.friendlymeals.data.model.Recipe +import com.google.firebase.example.friendlymeals.data.repository.AuthRepository +import com.google.firebase.example.friendlymeals.data.repository.DatabaseRepository import com.google.firebase.remoteconfig.FirebaseRemoteConfig +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.jsonPrimitive import javax.inject.Inject +import kotlin.collections.mapOf @OptIn(PublicPreviewAPI::class) class LiveAIRemoteDataSource @Inject constructor( private val aiModel: FirebaseAI, - private val remoteConfig: FirebaseRemoteConfig + private val remoteConfig: FirebaseRemoteConfig, + private val databaseRepository: DatabaseRepository, + private val authRepository: AuthRepository ) { - private val groceryListTool = Tool.functionDeclarations(listOf( - FunctionDeclaration( - name = ADD_INGREDIENTS_TOOL_NAME, - description = ADD_INGREDIENTS_TOOL_DESCRIPTION, - parameters = mapOf( - INGREDIENT_FIELD_NAME to Schema.string(INGREDIENT_FIELD_DESCRIPTION) + private val groceryListAutoTool = Tool.functionDeclarations( + autoFunctionDeclarations = listOf( + AutoFunctionDeclaration.create( + functionName = ADD_INGREDIENTS_TOOL_NAME, + description = ADD_INGREDIENTS_TOOL_DESCRIPTION, + inputSchema = JsonSchema.obj( + properties = mapOf( + INGREDIENT_FIELD_NAME to JsonSchema.string(INGREDIENT_FIELD_DESCRIPTION) + ) + ), + functionReference = ::addToGroceryList ) ) - )) + ) + + private suspend fun addToGroceryList(input: JsonObject): FunctionResponsePart { + val ingredients = input[INGREDIENT_FIELD_NAME]?.jsonPrimitive?.content + val userId = authRepository.currentUser?.uid.orEmpty() + + if (ingredients.isNullOrBlank() || userId.isEmpty()) { + throw FirebaseAutoFunctionException(INGREDIENT_FIELD_ERROR) + } + + val ingredientsList = ingredients.split(",").map { it.trim() } + databaseRepository.addIngredientsToGroceries(userId, ingredientsList) + + return FunctionResponsePart( + ADD_INGREDIENTS_TOOL_NAME, + JsonObject(mapOf( + ADD_INGREDIENTS_TOOL_RESULT to JsonPrimitive(ADD_INGREDIENTS_TOOL_RESULT_DESCRIPTION) + )) + ) + } @OptIn(PublicPreviewAPI::class) suspend fun setupLiveSession(recipe: Recipe): LiveSession? { @@ -44,7 +78,7 @@ class LiveAIRemoteDataSource @Inject constructor( modelName = remoteConfig.getString(LIVE_MODEL_NAME_KEY), generationConfig = liveGenerationConfig, systemInstruction = content { text(instructionText) }, - tools = listOf(groceryListTool) + tools = listOf(groceryListAutoTool) ) return try { @@ -69,11 +103,15 @@ class LiveAIRemoteDataSource @Inject constructor( private const val LIVE_MODEL_VOICE = "CHARON" //Tools config - private const val ADD_INGREDIENTS_TOOL_NAME = "addIngredientToGroceryList" - private const val ADD_INGREDIENTS_TOOL_DESCRIPTION = "Adds a specified ingredient to the " + + private const val ADD_INGREDIENTS_TOOL_NAME = "addToGroceryList" + private const val ADD_INGREDIENTS_TOOL_DESCRIPTION = "Adds a list of ingredients to the " + "user's grocery list in the database." + private const val ADD_INGREDIENTS_TOOL_RESULT = "result" + private const val ADD_INGREDIENTS_TOOL_RESULT_DESCRIPTION = "Successfully added " + + "ingredients to grocery list" private const val INGREDIENT_FIELD_NAME = "ingredient" private const val INGREDIENT_FIELD_DESCRIPTION = "The name of the ingredient to add." + private const val INGREDIENT_FIELD_ERROR = "Unable to add ingredients to the list" //Remote Config Keys private const val LIVE_MODEL_NAME_KEY = "live_model_name" diff --git a/app/src/main/java/com/google/firebase/example/friendlymeals/ui/live/LiveAssistantViewModel.kt b/app/src/main/java/com/google/firebase/example/friendlymeals/ui/live/LiveAssistantViewModel.kt index c15b501..ed63cc6 100644 --- a/app/src/main/java/com/google/firebase/example/friendlymeals/ui/live/LiveAssistantViewModel.kt +++ b/app/src/main/java/com/google/firebase/example/friendlymeals/ui/live/LiveAssistantViewModel.kt @@ -4,15 +4,11 @@ import android.annotation.SuppressLint import android.graphics.Bitmap import androidx.lifecycle.SavedStateHandle import androidx.navigation.toRoute -import com.google.firebase.ai.type.FunctionCallPart -import com.google.firebase.ai.type.FunctionResponsePart import com.google.firebase.ai.type.InlineData import com.google.firebase.ai.type.LiveSession import com.google.firebase.ai.type.PublicPreviewAPI import com.google.firebase.example.friendlymeals.MainViewModel -import com.google.firebase.example.friendlymeals.data.model.GroceryItem import com.google.firebase.example.friendlymeals.data.model.Recipe -import com.google.firebase.example.friendlymeals.data.repository.AuthRepository import com.google.firebase.example.friendlymeals.data.repository.DatabaseRepository import com.google.firebase.example.friendlymeals.data.repository.LiveAIRepository import com.google.firebase.example.friendlymeals.ui.live.LiveAssistantUiState.Loading @@ -20,8 +16,6 @@ import dagger.hilt.android.lifecycle.HiltViewModel import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow -import kotlinx.serialization.json.JsonObject -import kotlinx.serialization.json.JsonPrimitive import java.io.ByteArrayOutputStream import javax.inject.Inject @@ -29,7 +23,6 @@ import javax.inject.Inject @OptIn(PublicPreviewAPI::class) class LiveAssistantViewModel @Inject constructor( savedStateHandle: SavedStateHandle, - private val authRepository: AuthRepository, private val databaseRepository: DatabaseRepository, private val liveAIRepository: LiveAIRepository ) : MainViewModel() { @@ -72,46 +65,12 @@ class LiveAssistantViewModel @Inject constructor( } } - private fun handler(functionCall: FunctionCallPart): FunctionResponsePart { - if (functionCall.name == ADD_INGREDIENTS_TOOL_NAME) { - val ingredient = functionCall.args[INGREDIENT_FIELD_NAME] - val ingredientName = when (ingredient) { - is JsonPrimitive -> ingredient.content - else -> ingredient?.toString() - }?.trim()?.removeSurrounding("\"") - - if (!ingredientName.isNullOrBlank()) { - val userId = authRepository.currentUser?.uid.orEmpty() - if (userId.isNotEmpty()) { - launchCatching { - val item = GroceryItem( - userId = userId, - name = ingredientName, - checked = false - ) - databaseRepository.addGroceryItem(item) - } - } - } - - return FunctionResponsePart( - functionCall.name, - JsonObject(mapOf( - "result" to JsonPrimitive("Successfully added $ingredientName to grocery list") - )), - functionCall.id - ) - } - - return FunctionResponsePart(functionCall.name, JsonObject(emptyMap()), functionCall.id) - } - // Suppressing MissingPermission warning as we're // checking permissions before opening the screen @SuppressLint("MissingPermission") private fun startConversation() { launchCatching { - liveSession?.startAudioConversation(::handler) + liveSession?.startAudioConversation() } } @@ -147,9 +106,5 @@ class LiveAssistantViewModel @Inject constructor( private const val MIME_TYPE = "image/jpeg" private const val RECIPE_ERROR = "Failed to load recipe" private const val CONNECTION_ERROR = "Failed to connect to live assistant" - - //Tool config - private const val INGREDIENT_FIELD_NAME = "ingredient" - private const val ADD_INGREDIENTS_TOOL_NAME = "addIngredientToGroceryList" } }