Automatically calling tool via Gemini Live

app/src/main/java/com/google/firebase/example/friendlymeals/data/datasource/LiveAIRemoteDataSource.kt

@@ -1,6 +1,9 @@
 package com.google.firebase.example.friendlymeals.data.datasource
 
 import com.google.firebase.ai.FirebaseAI
+import com.google.firebase.ai.type.AutoFunctionDeclaration
+import com.google.firebase.ai.type.FirebaseAutoFunctionException
+import com.google.firebase.ai.type.FunctionResponsePart
 import com.google.firebase.ai.type.LiveSession
 import com.google.firebase.ai.type.PublicPreviewAPI
 import com.google.firebase.ai.type.ResponseModality
@@ -9,26 +12,57 @@ import com.google.firebase.ai.type.Voice
 import com.google.firebase.ai.type.content
 import com.google.firebase.ai.type.liveGenerationConfig
 import com.google.firebase.ai.type.Tool
-import com.google.firebase.ai.type.FunctionDeclaration
-import com.google.firebase.ai.type.Schema
+import com.google.firebase.ai.type.JsonSchema
 import com.google.firebase.example.friendlymeals.data.model.Recipe
+import com.google.firebase.example.friendlymeals.data.repository.AuthRepository
+import com.google.firebase.example.friendlymeals.data.repository.DatabaseRepository
 import com.google.firebase.remoteconfig.FirebaseRemoteConfig
+import kotlinx.serialization.json.JsonObject
+import kotlinx.serialization.json.JsonPrimitive
+import kotlinx.serialization.json.jsonPrimitive
 import javax.inject.Inject
+import kotlin.collections.mapOf
 
 @OptIn(PublicPreviewAPI::class)
 class LiveAIRemoteDataSource @Inject constructor(
     private val aiModel: FirebaseAI,
-    private val remoteConfig: FirebaseRemoteConfig
+    private val remoteConfig: FirebaseRemoteConfig,
+    private val databaseRepository: DatabaseRepository,
+    private val authRepository: AuthRepository
 ) {
-    private val groceryListTool = Tool.functionDeclarations(listOf(
-        FunctionDeclaration(
-            name = ADD_INGREDIENTS_TOOL_NAME,
-            description = ADD_INGREDIENTS_TOOL_DESCRIPTION,
-            parameters = mapOf(
-                INGREDIENT_FIELD_NAME to Schema.string(INGREDIENT_FIELD_DESCRIPTION)
+    private val groceryListAutoTool = Tool.functionDeclarations(
+        autoFunctionDeclarations = listOf(
+            AutoFunctionDeclaration.create(
+                functionName = ADD_INGREDIENTS_TOOL_NAME,
+                description = ADD_INGREDIENTS_TOOL_DESCRIPTION,
+                inputSchema = JsonSchema.obj(
+                    properties = mapOf(
+                        INGREDIENT_FIELD_NAME to JsonSchema.string(INGREDIENT_FIELD_DESCRIPTION)
+                    )
+                ),
+                functionReference = ::addToGroceryList
             )
         )
-    ))
+    )
+
+    private suspend fun addToGroceryList(input: JsonObject): FunctionResponsePart {
+        val ingredients = input[INGREDIENT_FIELD_NAME]?.jsonPrimitive?.content
+        val userId = authRepository.currentUser?.uid.orEmpty()
+
+        if (ingredients.isNullOrBlank() || userId.isEmpty()) {
+            throw FirebaseAutoFunctionException(INGREDIENT_FIELD_ERROR)
+        }
+
+        val ingredientsList = ingredients.split(",").map { it.trim() }
+        databaseRepository.addIngredientsToGroceries(userId, ingredientsList)
+
+        return FunctionResponsePart(
+            ADD_INGREDIENTS_TOOL_NAME,
+            JsonObject(mapOf(
+                ADD_INGREDIENTS_TOOL_RESULT to JsonPrimitive(ADD_INGREDIENTS_TOOL_RESULT_DESCRIPTION)
+            ))
+        )
+    }
 
     @OptIn(PublicPreviewAPI::class)
     suspend fun setupLiveSession(recipe: Recipe): LiveSession? {
@@ -44,7 +78,7 @@ class LiveAIRemoteDataSource @Inject constructor(
             modelName = remoteConfig.getString(LIVE_MODEL_NAME_KEY),
             generationConfig = liveGenerationConfig,
             systemInstruction = content { text(instructionText) },
-            tools = listOf(groceryListTool)
+            tools = listOf(groceryListAutoTool)
         )
 
         return try {
@@ -69,11 +103,15 @@ class LiveAIRemoteDataSource @Inject constructor(
         private const val LIVE_MODEL_VOICE = "CHARON"
 
         //Tools config
-        private const val ADD_INGREDIENTS_TOOL_NAME = "addIngredientToGroceryList"
-        private const val ADD_INGREDIENTS_TOOL_DESCRIPTION = "Adds a specified ingredient to the " +
+        private const val ADD_INGREDIENTS_TOOL_NAME = "addToGroceryList"
+        private const val ADD_INGREDIENTS_TOOL_DESCRIPTION = "Adds a list of ingredients to the " +
                 "user's grocery list in the database."
+        private const val ADD_INGREDIENTS_TOOL_RESULT = "result"
+        private const val ADD_INGREDIENTS_TOOL_RESULT_DESCRIPTION = "Successfully added " +
+                "ingredients to grocery list"
         private const val INGREDIENT_FIELD_NAME = "ingredient"
         private const val INGREDIENT_FIELD_DESCRIPTION = "The name of the ingredient to add."
+        private const val INGREDIENT_FIELD_ERROR = "Unable to add ingredients to the list"
 
         //Remote Config Keys
         private const val LIVE_MODEL_NAME_KEY = "live_model_name"

app/src/main/java/com/google/firebase/example/friendlymeals/ui/live/LiveAssistantViewModel.kt

@@ -4,32 +4,25 @@ import android.annotation.SuppressLint
 import android.graphics.Bitmap
 import androidx.lifecycle.SavedStateHandle
 import androidx.navigation.toRoute
-import com.google.firebase.ai.type.FunctionCallPart
-import com.google.firebase.ai.type.FunctionResponsePart
 import com.google.firebase.ai.type.InlineData
 import com.google.firebase.ai.type.LiveSession
 import com.google.firebase.ai.type.PublicPreviewAPI
 import com.google.firebase.example.friendlymeals.MainViewModel
-import com.google.firebase.example.friendlymeals.data.model.GroceryItem
 import com.google.firebase.example.friendlymeals.data.model.Recipe
-import com.google.firebase.example.friendlymeals.data.repository.AuthRepository
 import com.google.firebase.example.friendlymeals.data.repository.DatabaseRepository
 import com.google.firebase.example.friendlymeals.data.repository.LiveAIRepository
 import com.google.firebase.example.friendlymeals.ui.live.LiveAssistantUiState.Loading
 import dagger.hilt.android.lifecycle.HiltViewModel
 import kotlinx.coroutines.flow.MutableStateFlow
 import kotlinx.coroutines.flow.StateFlow
 import kotlinx.coroutines.flow.asStateFlow
-import kotlinx.serialization.json.JsonObject
-import kotlinx.serialization.json.JsonPrimitive
 import java.io.ByteArrayOutputStream
 import javax.inject.Inject
 
 @HiltViewModel
 @OptIn(PublicPreviewAPI::class)
 class LiveAssistantViewModel @Inject constructor(
     savedStateHandle: SavedStateHandle,
-    private val authRepository: AuthRepository,
     private val databaseRepository: DatabaseRepository,
     private val liveAIRepository: LiveAIRepository
 ) : MainViewModel() {
@@ -72,46 +65,12 @@ class LiveAssistantViewModel @Inject constructor(
         }
     }
 
-    private fun handler(functionCall: FunctionCallPart): FunctionResponsePart {
-        if (functionCall.name == ADD_INGREDIENTS_TOOL_NAME) {
-            val ingredient = functionCall.args[INGREDIENT_FIELD_NAME]
-            val ingredientName = when (ingredient) {
-                is JsonPrimitive -> ingredient.content
-                else -> ingredient?.toString()
-            }?.trim()?.removeSurrounding("\"")
-
-            if (!ingredientName.isNullOrBlank()) {
-                val userId = authRepository.currentUser?.uid.orEmpty()
-                if (userId.isNotEmpty()) {
-                    launchCatching {
-                        val item = GroceryItem(
-                            userId = userId,
-                            name = ingredientName,
-                            checked = false
-                        )
-                        databaseRepository.addGroceryItem(item)
-                    }
-                }
-            }
-
-            return FunctionResponsePart(
-                functionCall.name,
-                JsonObject(mapOf(
-                    "result" to JsonPrimitive("Successfully added $ingredientName to grocery list")
-                )),
-                functionCall.id
-            )
-        }
-
-        return FunctionResponsePart(functionCall.name, JsonObject(emptyMap()), functionCall.id)
-    }
-
     // Suppressing MissingPermission warning as we're
     // checking permissions before opening the screen
     @SuppressLint("MissingPermission")
     private fun startConversation() {
         launchCatching {
-            liveSession?.startAudioConversation(::handler)
+            liveSession?.startAudioConversation()
         }
     }
 
@@ -147,9 +106,5 @@ class LiveAssistantViewModel @Inject constructor(
         private const val MIME_TYPE = "image/jpeg"
         private const val RECIPE_ERROR = "Failed to load recipe"
         private const val CONNECTION_ERROR = "Failed to connect to live assistant"
-
-        //Tool config
-        private const val INGREDIENT_FIELD_NAME = "ingredient"
-        private const val ADD_INGREDIENTS_TOOL_NAME = "addIngredientToGroceryList"
     }
 }