From 9f88747dc29b10675f8f0e8052d7b8190a9cb306 Mon Sep 17 00:00:00 2001 From: Marina Coelho Date: Wed, 27 May 2026 13:52:12 +0100 Subject: [PATCH 1/2] Automatically calling tool via Gemini Live --- .../data/datasource/LiveAIRemoteDataSource.kt | 62 +++++++++++++++---- .../ui/live/LiveAssistantViewModel.kt | 47 +------------- 2 files changed, 51 insertions(+), 58 deletions(-) diff --git a/app/src/main/java/com/google/firebase/example/friendlymeals/data/datasource/LiveAIRemoteDataSource.kt b/app/src/main/java/com/google/firebase/example/friendlymeals/data/datasource/LiveAIRemoteDataSource.kt index 417d0d1..7c04131 100644 --- a/app/src/main/java/com/google/firebase/example/friendlymeals/data/datasource/LiveAIRemoteDataSource.kt +++ b/app/src/main/java/com/google/firebase/example/friendlymeals/data/datasource/LiveAIRemoteDataSource.kt @@ -1,6 +1,9 @@ package com.google.firebase.example.friendlymeals.data.datasource import com.google.firebase.ai.FirebaseAI +import com.google.firebase.ai.type.AutoFunctionDeclaration +import com.google.firebase.ai.type.FirebaseAutoFunctionException +import com.google.firebase.ai.type.FunctionResponsePart import com.google.firebase.ai.type.LiveSession import com.google.firebase.ai.type.PublicPreviewAPI import com.google.firebase.ai.type.ResponseModality @@ -9,26 +12,57 @@ import com.google.firebase.ai.type.Voice import com.google.firebase.ai.type.content import com.google.firebase.ai.type.liveGenerationConfig import com.google.firebase.ai.type.Tool -import com.google.firebase.ai.type.FunctionDeclaration -import com.google.firebase.ai.type.Schema +import com.google.firebase.ai.type.JsonSchema import com.google.firebase.example.friendlymeals.data.model.Recipe +import com.google.firebase.example.friendlymeals.data.repository.AuthRepository +import com.google.firebase.example.friendlymeals.data.repository.DatabaseRepository import com.google.firebase.remoteconfig.FirebaseRemoteConfig +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.jsonPrimitive import javax.inject.Inject +import kotlin.collections.mapOf @OptIn(PublicPreviewAPI::class) class LiveAIRemoteDataSource @Inject constructor( private val aiModel: FirebaseAI, - private val remoteConfig: FirebaseRemoteConfig + private val remoteConfig: FirebaseRemoteConfig, + private val databaseRepository: DatabaseRepository, + private val authRepository: AuthRepository ) { - private val groceryListTool = Tool.functionDeclarations(listOf( - FunctionDeclaration( - name = ADD_INGREDIENTS_TOOL_NAME, - description = ADD_INGREDIENTS_TOOL_DESCRIPTION, - parameters = mapOf( - INGREDIENT_FIELD_NAME to Schema.string(INGREDIENT_FIELD_DESCRIPTION) + private val groceryListAutoTool = Tool.functionDeclarations( + autoFunctionDeclarations = listOf( + AutoFunctionDeclaration.create( + functionName = ADD_INGREDIENTS_TOOL_NAME, + description = ADD_INGREDIENTS_TOOL_DESCRIPTION, + inputSchema = JsonSchema.obj( + properties = mapOf( + INGREDIENT_FIELD_NAME to JsonSchema.string(INGREDIENT_FIELD_DESCRIPTION) + ) + ), + functionReference = ::addToGroceryList ) ) - )) + ) + + private suspend fun addToGroceryList(input: JsonObject): FunctionResponsePart { + val ingredients = input[INGREDIENT_FIELD_NAME]?.jsonPrimitive?.content + val userId = authRepository.currentUser?.uid.orEmpty() + + if (ingredients.isNullOrBlank() || userId.isEmpty()) { + throw FirebaseAutoFunctionException(LIVE_MODEL_ERROR) + } + + val ingredientsList = ingredients.split(",").map { it.trim() } + databaseRepository.addIngredientsToGroceries(userId, ingredientsList) + + return FunctionResponsePart( + ADD_INGREDIENTS_TOOL_NAME, + JsonObject(mapOf( + ADD_INGREDIENTS_TOOL_RESULT to JsonPrimitive(ADD_INGREDIENTS_TOOL_RESULT_DESCRIPTION) + )) + ) + } @OptIn(PublicPreviewAPI::class) suspend fun setupLiveSession(recipe: Recipe): LiveSession? { @@ -44,7 +78,7 @@ class LiveAIRemoteDataSource @Inject constructor( modelName = remoteConfig.getString(LIVE_MODEL_NAME_KEY), generationConfig = liveGenerationConfig, systemInstruction = content { text(instructionText) }, - tools = listOf(groceryListTool) + tools = listOf(groceryListAutoTool) ) return try { @@ -67,11 +101,15 @@ class LiveAIRemoteDataSource @Inject constructor( companion object { //Live Model Config private const val LIVE_MODEL_VOICE = "CHARON" + private const val LIVE_MODEL_ERROR = "Unable to add ingredients to the list" //Tools config private const val ADD_INGREDIENTS_TOOL_NAME = "addIngredientToGroceryList" - private const val ADD_INGREDIENTS_TOOL_DESCRIPTION = "Adds a specified ingredient to the " + + private const val ADD_INGREDIENTS_TOOL_DESCRIPTION = "Adds a list of ingredients to the " + "user's grocery list in the database." + private const val ADD_INGREDIENTS_TOOL_RESULT = "result" + private const val ADD_INGREDIENTS_TOOL_RESULT_DESCRIPTION = "Successfully added " + + "ingredients to grocery list" private const val INGREDIENT_FIELD_NAME = "ingredient" private const val INGREDIENT_FIELD_DESCRIPTION = "The name of the ingredient to add." diff --git a/app/src/main/java/com/google/firebase/example/friendlymeals/ui/live/LiveAssistantViewModel.kt b/app/src/main/java/com/google/firebase/example/friendlymeals/ui/live/LiveAssistantViewModel.kt index c15b501..ed63cc6 100644 --- a/app/src/main/java/com/google/firebase/example/friendlymeals/ui/live/LiveAssistantViewModel.kt +++ b/app/src/main/java/com/google/firebase/example/friendlymeals/ui/live/LiveAssistantViewModel.kt @@ -4,15 +4,11 @@ import android.annotation.SuppressLint import android.graphics.Bitmap import androidx.lifecycle.SavedStateHandle import androidx.navigation.toRoute -import com.google.firebase.ai.type.FunctionCallPart -import com.google.firebase.ai.type.FunctionResponsePart import com.google.firebase.ai.type.InlineData import com.google.firebase.ai.type.LiveSession import com.google.firebase.ai.type.PublicPreviewAPI import com.google.firebase.example.friendlymeals.MainViewModel -import com.google.firebase.example.friendlymeals.data.model.GroceryItem import com.google.firebase.example.friendlymeals.data.model.Recipe -import com.google.firebase.example.friendlymeals.data.repository.AuthRepository import com.google.firebase.example.friendlymeals.data.repository.DatabaseRepository import com.google.firebase.example.friendlymeals.data.repository.LiveAIRepository import com.google.firebase.example.friendlymeals.ui.live.LiveAssistantUiState.Loading @@ -20,8 +16,6 @@ import dagger.hilt.android.lifecycle.HiltViewModel import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow -import kotlinx.serialization.json.JsonObject -import kotlinx.serialization.json.JsonPrimitive import java.io.ByteArrayOutputStream import javax.inject.Inject @@ -29,7 +23,6 @@ import javax.inject.Inject @OptIn(PublicPreviewAPI::class) class LiveAssistantViewModel @Inject constructor( savedStateHandle: SavedStateHandle, - private val authRepository: AuthRepository, private val databaseRepository: DatabaseRepository, private val liveAIRepository: LiveAIRepository ) : MainViewModel() { @@ -72,46 +65,12 @@ class LiveAssistantViewModel @Inject constructor( } } - private fun handler(functionCall: FunctionCallPart): FunctionResponsePart { - if (functionCall.name == ADD_INGREDIENTS_TOOL_NAME) { - val ingredient = functionCall.args[INGREDIENT_FIELD_NAME] - val ingredientName = when (ingredient) { - is JsonPrimitive -> ingredient.content - else -> ingredient?.toString() - }?.trim()?.removeSurrounding("\"") - - if (!ingredientName.isNullOrBlank()) { - val userId = authRepository.currentUser?.uid.orEmpty() - if (userId.isNotEmpty()) { - launchCatching { - val item = GroceryItem( - userId = userId, - name = ingredientName, - checked = false - ) - databaseRepository.addGroceryItem(item) - } - } - } - - return FunctionResponsePart( - functionCall.name, - JsonObject(mapOf( - "result" to JsonPrimitive("Successfully added $ingredientName to grocery list") - )), - functionCall.id - ) - } - - return FunctionResponsePart(functionCall.name, JsonObject(emptyMap()), functionCall.id) - } - // Suppressing MissingPermission warning as we're // checking permissions before opening the screen @SuppressLint("MissingPermission") private fun startConversation() { launchCatching { - liveSession?.startAudioConversation(::handler) + liveSession?.startAudioConversation() } } @@ -147,9 +106,5 @@ class LiveAssistantViewModel @Inject constructor( private const val MIME_TYPE = "image/jpeg" private const val RECIPE_ERROR = "Failed to load recipe" private const val CONNECTION_ERROR = "Failed to connect to live assistant" - - //Tool config - private const val INGREDIENT_FIELD_NAME = "ingredient" - private const val ADD_INGREDIENTS_TOOL_NAME = "addIngredientToGroceryList" } } From 8db80170d39195b5a0541aa616cac8baf79c7767 Mon Sep 17 00:00:00 2001 From: Marina Coelho Date: Wed, 27 May 2026 14:42:11 +0100 Subject: [PATCH 2/2] Fix companion object --- .../friendlymeals/data/datasource/LiveAIRemoteDataSource.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/src/main/java/com/google/firebase/example/friendlymeals/data/datasource/LiveAIRemoteDataSource.kt b/app/src/main/java/com/google/firebase/example/friendlymeals/data/datasource/LiveAIRemoteDataSource.kt index 7c04131..3fbc660 100644 --- a/app/src/main/java/com/google/firebase/example/friendlymeals/data/datasource/LiveAIRemoteDataSource.kt +++ b/app/src/main/java/com/google/firebase/example/friendlymeals/data/datasource/LiveAIRemoteDataSource.kt @@ -50,7 +50,7 @@ class LiveAIRemoteDataSource @Inject constructor( val userId = authRepository.currentUser?.uid.orEmpty() if (ingredients.isNullOrBlank() || userId.isEmpty()) { - throw FirebaseAutoFunctionException(LIVE_MODEL_ERROR) + throw FirebaseAutoFunctionException(INGREDIENT_FIELD_ERROR) } val ingredientsList = ingredients.split(",").map { it.trim() } @@ -101,10 +101,9 @@ class LiveAIRemoteDataSource @Inject constructor( companion object { //Live Model Config private const val LIVE_MODEL_VOICE = "CHARON" - private const val LIVE_MODEL_ERROR = "Unable to add ingredients to the list" //Tools config - private const val ADD_INGREDIENTS_TOOL_NAME = "addIngredientToGroceryList" + private const val ADD_INGREDIENTS_TOOL_NAME = "addToGroceryList" private const val ADD_INGREDIENTS_TOOL_DESCRIPTION = "Adds a list of ingredients to the " + "user's grocery list in the database." private const val ADD_INGREDIENTS_TOOL_RESULT = "result" @@ -112,6 +111,7 @@ class LiveAIRemoteDataSource @Inject constructor( "ingredients to grocery list" private const val INGREDIENT_FIELD_NAME = "ingredient" private const val INGREDIENT_FIELD_DESCRIPTION = "The name of the ingredient to add." + private const val INGREDIENT_FIELD_ERROR = "Unable to add ingredients to the list" //Remote Config Keys private const val LIVE_MODEL_NAME_KEY = "live_model_name"