Video clipper

AI Video Clipper/README.md

@@ -0,0 +1,63 @@
+# Gemini Video Editing App
+
+---
+An Android application demonstrating automated video highlight generation and intelligent cinematic editing suggestion using Vertex AI for Firebase (Gemini 2.5 Pro) and Firebase Cloud Storage.
+Users can select multiple videos, specify editing goals in natural language (e.g., "create a fast-paced 15-second action reel"), upload them to Cloud Storage, and let Gemini analyze the content to recommend and preview video trims and edits.
+
+## Prerequisites & Project Setup
+This project uses **Firebase Cloud Storage** to host video files and **Vertex AI for Firebase** (Gemini 2.5 Pro) for advanced media analysis and editing recommendations. Follow the setup steps below to configure your Firebase and Google Cloud project.
+### 1. Firebase Project Configuration
+1. Go to the [Firebase Console](https://console.firebase.google.com/).
+2. Click **Add project** and create a new project (or select an existing one).
+3. Vertex AI for Firebase requires the project to be on the pay-as-you-go **Blaze plan**. In the Firebase Console, upgrade your project by clicking **Upgrade** in the bottom-left corner.
+4. Register your Android App:
+   - Click the Android icon to add a new app.
+   - Enter the Android package name: `com.example.videoediting`.
+   - Click **Register app**.
+5. Download the `google-services.json` configuration file and place it in the `app/` directory of this project:
+   ```path
+   /app/google-services.json
+   ```
+---
+### 2. Enable Firebase Storage
+1. In the Firebase Console sidebar, go to **Build** > **Storage**.
+2. Click **Get Started**.
+3. Select a starting rule configuration (e.g., test mode) and choose your Storage location (e.g., `us-central1`).
+4. Once created, note your bucket URI (formatted as `gs://YOUR_PROJECT_ID.firebasestorage.app` or `gs://YOUR_PROJECT_ID.appspot.com`).
+5. **Create a `videos` folder**:
+   - Within the Storage console dashboard, click the **New folder** icon.
+   - Name the folder `videos`.
+   - *Note: While the app programmatically creates this folder upon upload, creating it manually ensures it is initialized for your project.*
+6. Set the **Storage Rules** to allow read/write access for uploads. For example, during development:
+   ```javascript
+   rules_version = '2';
+   service firebase.storage {
+     match /b/{bucket}/o {
+       match /videos/{allPaths=**} {
+         allow read, write: if true;
+       }
+     }
+   }
+   ```
+---
+### 3. Enable Vertex AI for Firebase (AI Logic)
+1. In the Firebase Console sidebar, go to **Build** > **Vertex AI** (or **Build with Gemini**).
+2. Click **Get Started** and follow the prompts.
+3. This activates the necessary Vertex AI APIs on the underlying Google Cloud project and sets up billing integration.
+---
+### 4. Configure Google Cloud Storage Permissions for Vertex AI
+Since Gemini processes video files directly from Google Cloud Storage, the Google Cloud Vertex AI service agent requires permission to read objects from your Storage bucket.
+1. Find your **Google Cloud Project Number**:
+   - In the Firebase Console, click the Gear icon next to **Project Overview** and select **Project Settings**.
+   - Copy the **Project number** (e.g., `123456789012`).
+2. Open the [Google Cloud Console IAM Page](https://console.cloud.google.com/iam-admin/iam).
+3. Ensure you are in the correct project.
+4. Check the **Include Google-provided role grants** box in the top-right corner of the IAM principal list.
+5. Search for the **Vertex AI Service Agent** service account:
+   ```text
+   service-PROJECT_NUMBER@gcp-sa-aiplatform.iam.gserviceaccount.com
+   ```
+   *(Replace `PROJECT_NUMBER` with your actual project number).*
+6. Click the edit icon (pencil) next to this service account principal to modify its roles.
+7. Click **Add another role** and select **Storage Object Viewer** (`roles/storage.objectViewer`).
+8. Save the permissions.

AI Video Clipper/app/.gitignore

@@ -0,0 +1 @@
+/build

AI Video Clipper/app/build.gradle.kts

@@ -0,0 +1,66 @@
+plugins {
+  alias(libs.plugins.android.application)
+  alias(libs.plugins.kotlin.compose)
+  alias(libs.plugins.kotlin.serialization)
+  id("com.google.gms.google-services")
+}
+
+android {
+  namespace = "com.example.videoediting"
+  compileSdk = 37
+
+  defaultConfig {
+    applicationId = "com.example.videoediting"
+    minSdk = 30
+    targetSdk = 37
+    versionCode = 1
+    versionName = "1.0"
+
+    testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner"
+  }
+
+  buildTypes {
+    release {
+      isMinifyEnabled = false
+      proguardFiles(getDefaultProguardFile("proguard-android-optimize.txt"), "proguard-rules.pro")
+    }
+  }
+  compileOptions {
+    sourceCompatibility = JavaVersion.VERSION_11
+    targetCompatibility = JavaVersion.VERSION_11
+  }
+  buildFeatures {
+    compose = true
+  }
+}
+
+dependencies {
+  implementation(platform(libs.androidx.compose.bom))
+  implementation(libs.androidx.activity.compose)
+  implementation(platform(libs.firebase.bom))
+  implementation(libs.firebase.ai)
+  implementation(libs.firebase.common)
+  implementation(libs.firebase.storage)
+  implementation(libs.androidx.compose.material3)
+  implementation(libs.androidx.compose.material.icons.extended)
+  implementation(libs.androidx.media3.common)
+  implementation(libs.androidx.media3.effect)
+  implementation(libs.androidx.media3.transformer)
+  implementation(libs.androidx.media3.ui.compose)
+  implementation(libs.androidx.media3.inspector)
+  implementation(libs.kotlinx.serialization.json)
+  implementation(libs.kotlinx.coroutines.guava)
+  implementation(libs.androidx.compose.ui)
+  implementation(libs.androidx.compose.ui.graphics)
+  implementation(libs.androidx.compose.ui.tooling.preview)
+  implementation(libs.androidx.core.ktx)
+  implementation(libs.androidx.lifecycle.runtime.ktx)
+  implementation(libs.androidx.lifecycle.viewmodel.compose)
+  testImplementation(libs.junit)
+  androidTestImplementation(platform(libs.androidx.compose.bom))
+  androidTestImplementation(libs.androidx.compose.ui.test.junit4)
+  androidTestImplementation(libs.androidx.espresso.core)
+  androidTestImplementation(libs.androidx.junit)
+  debugImplementation(libs.androidx.compose.ui.test.manifest)
+  debugImplementation(libs.androidx.compose.ui.tooling)
+}

AI Video Clipper/app/proguard-rules.pro

@@ -0,0 +1,21 @@
+# Add project specific ProGuard rules here.
+# You can control the set of applied configuration files using the
+# proguardFiles setting in build.gradle.
+#
+# For more details, see
+#   http://developer.android.com/guide/developing/tools/proguard.html
+
+# If your project uses WebView with JS, uncomment the following
+# and specify the fully qualified class name to the JavaScript interface
+# class:
+#-keepclassmembers class fqcn.of.javascript.interface.for.webview {
+#   public *;
+#}
+
+# Uncomment this to preserve the line number information for
+# debugging stack traces.
+#-keepattributes SourceFile,LineNumberTable
+
+# If you keep the line number information, uncomment this to
+# hide the original source file name.
+#-renamesourcefileattribute SourceFile

AI Video Clipper/app/src/androidTest/java/com/example/videoediting/ExampleInstrumentedTest.kt

@@ -0,0 +1,24 @@
+package com.example.videoediting
+
+import androidx.test.platform.app.InstrumentationRegistry
+import androidx.test.ext.junit.runners.AndroidJUnit4
+
+import org.junit.Test
+import org.junit.runner.RunWith
+
+import org.junit.Assert.*
+
+/**
+ * Instrumented test, which will execute on an Android device.
+ *
+ * See [testing documentation](http://d.android.com/tools/testing).
+ */
+@RunWith(AndroidJUnit4::class)
+class ExampleInstrumentedTest {
+  @Test
+  fun useAppContext() {
+    // Context of the app under test.
+    val appContext = InstrumentationRegistry.getInstrumentation().targetContext
+    assertEquals("com.example.videoediting", appContext.packageName)
+  }
+}

AI Video Clipper/app/src/main/AndroidManifest.xml

@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+    xmlns:tools="http://schemas.android.com/tools">
+
+    <uses-permission android:name="android.permission.INTERNET" />
+    <uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
+
+  <application
+      android:allowBackup="true"
+      android:dataExtractionRules="@xml/data_extraction_rules"
+      android:fullBackupContent="@xml/backup_rules"
+      android:icon="@mipmap/ic_launcher"
+      android:label="@string/app_name"
+      android:roundIcon="@mipmap/ic_launcher_round"
+      android:supportsRtl="true"
+      android:theme="@style/Theme.GeminiVideoEditing">
+    <activity
+        android:name=".MainActivity"
+        android:exported="true"
+        android:label="@string/app_name"
+        android:theme="@style/Theme.GeminiVideoEditing">
+      <intent-filter>
+        <action android:name="android.intent.action.MAIN" />
+
+        <category android:name="android.intent.category.LAUNCHER" />
+      </intent-filter>
+    </activity>
+  </application>
+
+</manifest>

AI Video Clipper/app/src/main/java/com/example/videoediting/AIResponseJsonSchema.kt

@@ -0,0 +1,18 @@
+package com.example.videoediting
+
+import com.google.firebase.ai.type.Schema
+
+val jsonSchema = Schema.obj(mapOf(
+  "videos" to Schema.array(
+    Schema.obj(mapOf(
+      "uri" to Schema.string(),
+      "mostEngagingSegment" to Schema.obj(
+        mapOf(
+          "startMs" to Schema.long("A start ms value for clipping"),
+          "endMs" to Schema.long("End ms value for clipping"),
+          "reasoning" to Schema.string("reasoning for not finding a clipping range")
+        )
+      ),
+    ))
+  ))
+)

AI Video Clipper/app/src/main/java/com/example/videoediting/FirebaseStorageRepository.kt

@@ -0,0 +1,27 @@
+package com.example.videoediting
+
+import android.net.Uri
+import android.util.Log
+import com.google.firebase.storage.FirebaseStorage
+import kotlinx.coroutines.tasks.await
+import java.util.UUID
+
+class FirebaseStorageRepository(private val storage: FirebaseStorage) {
+
+    suspend fun uploadFile(uri: Uri): String? {
+        return try {
+            val storageRef = storage.reference
+            val videoRef = storageRef.child("videos/${UUID.randomUUID()}.mp4")
+            val uploadTask = videoRef.putFile(uri)
+            uploadTask.await() // Wait for the upload to complete
+            val path = videoRef.path
+            val bucket = videoRef.bucket
+            val storageUrl = "gs://$bucket$path"
+            Log.d("FirebaseStorageRepo", "File uploaded successfully: $storageUrl")
+            storageUrl
+        } catch (e: Exception) {
+            Log.e("FirebaseStorageRepo", "Error uploading file", e)
+            null
+        }
+    }
+}

AI Video Clipper/app/src/main/java/com/example/videoediting/GeminiRepository.kt

@@ -0,0 +1,54 @@
+package com.example.videoediting
+
+import android.net.Uri
+import android.util.Log
+import com.google.firebase.Firebase
+import com.google.firebase.ai.ai
+import com.google.firebase.ai.type.GenerativeBackend
+import com.google.firebase.ai.type.content
+import com.google.firebase.ai.type.generationConfig
+import com.google.firebase.ai.type.thinkingConfig
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.withContext
+
+class GeminiRepository {
+
+    private val aiModel = Firebase.ai(backend = GenerativeBackend.vertexAI())
+        .generativeModel(
+            modelName = "gemini-2.5-pro",
+            generationConfig = generationConfig {
+                temperature = 0f
+                responseMimeType = "application/json"
+                responseSchema = jsonSchema
+                thinkingConfig = thinkingConfig {
+                    thinkingBudget = 1024
+                    includeThoughts = true
+                }
+            }
+        )
+
+    suspend fun generateEditResponse(promptData: String, uriPairs: List<Pair<String, Uri>>): GeminiResponse = withContext(Dispatchers.IO) {
+        val gsUris = uriPairs.map { it.first }
+        val requestContent = content {
+            gsUris.forEach { gsUri ->
+                fileData(gsUri, "video/mp4")
+            }
+
+            text(promptData)
+        }
+        val response = aiModel.generateContent(requestContent)
+
+        val aiThoughts = if (response.thoughtSummary.isNullOrEmpty()) null else response.thoughtSummary
+        val aiResponseText = if (response.text.isNullOrEmpty()) null else response.text
+        Log.d("GeminiRepository", "response $aiResponseText")
+        val aiTokensUsed = response.usageMetadata?.thoughtsTokenCount ?: -1
+
+        GeminiResponse(aiThoughts, aiResponseText, aiTokensUsed)
+    }
+}
+
+data class GeminiResponse(
+    val thoughts: String?,
+    val responseText: String?,
+    val tokensUsed: Int
+)

AI Video Clipper/app/src/main/java/com/example/videoediting/GenAIParser.kt

@@ -0,0 +1,67 @@
+package com.example.videoediting
+
+import android.util.Log
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.json.Json
+
+object GenAIParser {
+  private const val TAG = "GenAIParser"
+
+  // 1. JSON DTOs matching the exact schema returned by Gemini
+  @Serializable
+  private data class MostEngagingSegmentDto(
+    val startMs: Long? = null,
+    val endMs: Long? = null,
+    val reasoning: String? = null
+  )
+
+  @Serializable
+  private data class VideoSegmentDto(
+    val uri: String,
+    val mostEngagingSegment: MostEngagingSegmentDto? = null
+  )
+
+  @Serializable
+  private data class AIResponseDto(
+    val videos: List<VideoSegmentDto> = emptyList()
+  )
+
+  // 2. Public Domain Models (Unchanged to prevent breaking callers)
+  data class SegmentTimes(val startMs: Long, val endMs: Long)
+  data class VideoSegment(val uri: String, val segmentTimes: SegmentTimes?, val reasoning: String?)
+
+  private val jsonConfig = Json {
+    ignoreUnknownKeys = true // Resilient to extra fields returned by the AI
+  }
+
+  /**
+   * Parses a JSON string conforming to AIResponseJsonSchema and returns a list of VideoSegment objects.
+   *
+   * @param jsonString The JSON string response from the Gemini agent.
+   * @return A [List] of [VideoSegment] objects. Returns an empty list on failure.
+   */
+  fun parseVideoSegments(jsonString: String): List<VideoSegment> {
+    return try {
+      val responseDto = jsonConfig.decodeFromString<AIResponseDto>(jsonString)
+      responseDto.videos.map { dto ->
+        val segmentDto = dto.mostEngagingSegment
+        var segmentTimes: SegmentTimes? = null
+        var reasoning: String? = null
+
+        if (segmentDto != null) {
+          val startMs = segmentDto.startMs ?: -1L
+          val endMs = segmentDto.endMs ?: -1L
+          reasoning = segmentDto.reasoning
+
+          if (startMs >= 0 && endMs > startMs) {
+            segmentTimes = SegmentTimes(startMs, endMs)
+          }
+        }
+        VideoSegment(dto.uri, segmentTimes, reasoning)
+      }
+    } catch (e: Exception) {
+      Log.e(TAG, "Failed to parse video segments JSON with kotlinx-serialization", e)
+      emptyList()
+    }
+  }
+}