Skip to content

agent: Add image generation appfunction #25

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
calebareeveso wants to merge 2 commits into
base: main
Choose a base branch
from
Open

agent: Add image generation appfunction #25

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b61f0f2
agent: Add image generation appfunction
Jul 8, 2026
81197ad
fix: address gemini-code-assist bot code review feedback
Jul 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions agent/app/build.gradle.kts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
plugins {
alias(libs.plugins.android.application)
alias(libs.plugins.kotlin.compose)
alias(libs.plugins.kotlin.serialization)
alias(libs.plugins.ksp)
alias(libs.plugins.hilt)
alias(libs.plugins.screenshot)
Expand Down Expand Up @@ -60,11 +61,13 @@ android {
dimension = "mode"
buildConfigField("Boolean", "IS_RETAIL", "true")

val containsRetail = gradle.startParameter.taskNames.any { it.contains("Retail", ignoreCase = true) }
val containsRetail = gradle.startParameter.taskNames.any {
it.contains("Retail", ignoreCase = true)
}
val apiKey = project.findProperty("GEMINI_API_KEY") as? String ?: ""
if (containsRetail && apiKey.isEmpty()) {
throw GradleException(
"GEMINI_API_KEY project property is required for retail builds. Pass it using -PGEMINI_API_KEY=your_key",
"GEMINI_API_KEY project property is required for retail builds. Pass it using -PGEMINI_API_KEY=your_key"
)
}
buildConfigField("String", "GEMINI_API_KEY", "\"$apiKey\"")
Expand Down
10 changes: 10 additions & 0 deletions agent/app/src/main/AndroidManifest.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,16 @@
android:theme="@style/Theme.AppCompat.DayNight.DarkActionBar"
android:exported="false"
tools:replace="android:label,android:theme" />

<provider
android:name="androidx.core.content.FileProvider"
android:authorities="${applicationId}.fileprovider"
android:exported="false"
android:grantUriPermissions="true">
<meta-data
android:name="android.support.FILE_PROVIDER_PATHS"
android:resource="@xml/file_paths" />
</provider>
</application>

<instrumentation
Expand Down
199 changes: 188 additions & 11 deletions agent/app/src/main/java/com/example/appfunctions/agent/data/BuiltInAppFunctions.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,14 +22,26 @@ import android.content.pm.PackageManager
import android.location.Address
import android.location.Geocoder
import android.location.LocationManager
import android.util.Base64
import androidx.appfunctions.AppFunctionContext
import androidx.appfunctions.AppFunctionSerializable
import androidx.appfunctions.service.AppFunction
import androidx.core.content.ContextCompat
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
import androidx.core.content.FileProvider
import androidx.datastore.preferences.core.stringPreferencesKey
import com.example.appfunctions.agent.BuildConfig
import com.example.appfunctions.agent.di.settingsDataStore
import java.io.File
import java.net.HttpURLConnection
import java.net.URL
import java.util.UUID
import kotlin.coroutines.resume
import kotlin.coroutines.suspendCoroutine
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.flow.first
import kotlinx.coroutines.withContext
import org.json.JSONArray
import org.json.JSONObject

/** Built-in AppFunctions for location and geocoding services. */
class BuiltInAppFunctions {
Expand All @@ -41,10 +53,7 @@ class BuiltInAppFunctions {
* @return The latitude and longitude coordinates of the address, or null if geocoding fails.
*/
@AppFunction
suspend fun geocodeAddress(
appFunctionContext: AppFunctionContext,
address: String,
): LatLng? {
suspend fun geocodeAddress(appFunctionContext: AppFunctionContext, address: String): LatLng? {
val context = appFunctionContext.context
if (!Geocoder.isPresent()) {
return null
Expand All @@ -63,7 +72,7 @@ class BuiltInAppFunctions {
val location = addresses.firstOrNull()
if (location != null) {
continuation.resume(
LatLng(location.latitude, location.longitude),
LatLng(location.latitude, location.longitude)
)
} else {
continuation.resume(null)
Expand All @@ -73,7 +82,7 @@ class BuiltInAppFunctions {
override fun onError(errorMessage: String?) {
continuation.resume(null)
}
},
}
)
}
} catch (e: Exception) {
Expand All @@ -96,13 +105,16 @@ class BuiltInAppFunctions {

// Check permissions
val hasFineLocation =
ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_FINE_LOCATION) ==
ContextCompat.checkSelfPermission(
context,
Manifest.permission.ACCESS_FINE_LOCATION
) ==
PackageManager.PERMISSION_GRANTED

val hasCoarseLocation =
ContextCompat.checkSelfPermission(
context,
Manifest.permission.ACCESS_COARSE_LOCATION,
Manifest.permission.ACCESS_COARSE_LOCATION
) ==
PackageManager.PERMISSION_GRANTED

Expand Down Expand Up @@ -146,6 +158,171 @@ class BuiltInAppFunctions {
/** The latitude coordinate. */
val latitude: Double,
/** The longitude coordinate. */
val longitude: Double,
val longitude: Double
)

/**
* Generates an image from a text prompt and returns the remote image URI.
*
* @param prompt The text prompt describing the image to generate (e.g., "futuristic cityscape at sunset").
* @param aspectRatio Optional aspect ratio for the image (e.g., "16:9", "1:1").
* @return A GeneratedImageResult containing the generated remote image URI.
*/
@AppFunction(isDescribedByKDoc = true)
suspend fun generateImage(
appFunctionContext: AppFunctionContext,
prompt: String,
aspectRatio: String? = null
): GeneratedImageResult = withContext(Dispatchers.IO) {
val context = appFunctionContext.context
val apiKey =
context.settingsDataStore.data
.first()[stringPreferencesKey("gemini_api_key")]
?.takeIf { it.isNotBlank() }
?: BuildConfig.GEMINI_API_KEY.takeIf { it.isNotBlank() }
if (apiKey.isNullOrBlank()) {
throw IllegalStateException(
"Gemini API key is not configured. Please set gemini_api_key in settings."
)
}

val requestJson =
JSONObject().apply {
put(
"contents",
JSONArray().apply {
put(
JSONObject().apply {
put(
"parts",
JSONArray().apply {
put(JSONObject().apply { put("text", prompt) })
}
)
}
)
}
)
put(
"generationConfig",
JSONObject().apply {
put("responseModalities", JSONArray().apply { put("IMAGE") })
if (!aspectRatio.isNullOrBlank()) {
put(
"imageConfig",
JSONObject().apply {
put("aspectRatio", aspectRatio)
}
)
}
}
)
}
Comment thread
calebareeveso marked this conversation as resolved.

val endpointUrl =
"https://generativelanguage.googleapis.com/v1beta/models/gemini-3.1-flash-image-preview:generateContent?key=$apiKey"
val url = URL(endpointUrl)
val connection =
(url.openConnection() as HttpURLConnection).apply {
requestMethod = "POST"
setRequestProperty("Content-Type", "application/json")
doOutput = true
connectTimeout = 30000
readTimeout = 60000
}

try {
connection.outputStream.use { os ->
os.write(requestJson.toString().toByteArray(Charsets.UTF_8))
}

val responseCode = connection.responseCode
if (responseCode != HttpURLConnection.HTTP_OK) {
val errorBody =
connection.errorStream?.bufferedReader()?.use { it.readText() }
?: "HTTP $responseCode"
throw IllegalStateException(
"Image generation failed ($responseCode): $errorBody"
)
}

val responseText = connection.inputStream.bufferedReader().use { it.readText() }
val responseJson = JSONObject(responseText)
val candidates = responseJson.optJSONArray("candidates")
if (candidates == null || candidates.length() == 0) {
throw IllegalStateException(
"No candidates returned from Gemini image generation API"
)
}

val parts =
candidates
.getJSONObject(0)
.optJSONObject("content")
?.optJSONArray("parts")
if (parts == null || parts.length() == 0) {
throw IllegalStateException(
"No parts returned in candidate content. Gemini response: $responseText"
)
}

var base64Data: String? = null
var mimeType = "image/png"
for (i in 0 until parts.length()) {
val part = parts.getJSONObject(i)
val inlineData =
part.optJSONObject("inlineData")
?: part.optJSONObject("inline_data")
if (inlineData != null) {
base64Data = inlineData.optString("data")
val returnedMime =
inlineData
.optString("mimeType")
.takeIf { it.isNotBlank() }
?: inlineData.optString("mime_type")
if (returnedMime.isNotBlank()) {
mimeType = returnedMime
}
break
}
}

if (base64Data.isNullOrBlank()) {
throw IllegalStateException(
"No inlineData image found in response parts. Gemini response: $responseText"
)
}

val imageBytes = Base64.decode(base64Data, Base64.DEFAULT)
val extension =
if (mimeType.contains("jpeg") || mimeType.contains("jpg")) "jpg" else "png"
val cachedFile =
File(
context.cacheDir,
"generated_${UUID.randomUUID()}.$extension"
)
cachedFile.writeBytes(imageBytes)

val authority = "${context.packageName}.fileprovider"
val contentUri = FileProvider.getUriForFile(context, authority, cachedFile)
GeneratedImageResult(
imageUri = contentUri.toString(),
mimeType = mimeType,
prompt = prompt
)
} finally {
connection.disconnect()
}
}

/** Represents the result of an image generation request. */
@AppFunctionSerializable
data class GeneratedImageResult(
/** The remote URI or URL of the generated image. */
val imageUri: String,
/** The MIME type of the generated image. */
val mimeType: String,
/** The original prompt used to generate the image. */
val prompt: String
)
}
Loading