From d0614b8c2869542f09cf08b0ec067b6c21d4bec3 Mon Sep 17 00:00:00 2001 From: fmasalha Date: Fri, 8 May 2026 11:18:12 -0400 Subject: [PATCH] Bug 2036474 - Updated s2s telemetry --- .../components/concept/llm/LlmProvider.kt | 4 +- .../components/lib/llm/mlpa/MlpaLlm.kt | 20 ++++---- .../lib/llm/mlpa/MlpaLlmProvider.kt | 11 ++++- .../components/lib/llm/mlpa/MlpaLlmTest.kt | 4 ++ mobile/android/fenix/app/metrics.yaml | 19 ++++++- .../SummarizationTelemetryMiddleware.kt | 19 ++++--- .../SummarizationTelemetryMiddlewareTest.kt | 49 ++++++++++++++----- 7 files changed, 92 insertions(+), 34 deletions(-) diff --git a/mobile/android/android-components/components/concept/llm/src/main/java/mozilla/components/concept/llm/LlmProvider.kt b/mobile/android/android-components/components/concept/llm/src/main/java/mozilla/components/concept/llm/LlmProvider.kt index 4d81968917576..77b10fd44adeb 100644 --- a/mobile/android/android-components/components/concept/llm/src/main/java/mozilla/components/concept/llm/LlmProvider.kt +++ b/mobile/android/android-components/components/concept/llm/src/main/java/mozilla/components/concept/llm/LlmProvider.kt @@ -21,8 +21,10 @@ sealed interface LlmProvider { * * @property nameRes A string resource ID representing the display name of the provider. * @property iconRes A drawable resource ID representing the icon of the provider if present. + * @property model The identifier of the model that this provider serves (e.g. + * "moz-summarization"). Used for telemetry and logging. */ - data class Info(val nameRes: Int, val iconRes: Int? = null) + data class Info(val nameRes: Int, val iconRes: Int? = null, val model: String? = null) /** * Metadata about this provider, including its display name. diff --git a/mobile/android/android-components/components/lib/llm-mlpa/src/main/java/mozilla/components/lib/llm/mlpa/MlpaLlm.kt b/mobile/android/android-components/components/lib/llm-mlpa/src/main/java/mozilla/components/lib/llm/mlpa/MlpaLlm.kt index 2af2df2f75a87..7127856dbceca 100644 --- a/mobile/android/android-components/components/lib/llm-mlpa/src/main/java/mozilla/components/lib/llm/mlpa/MlpaLlm.kt +++ b/mobile/android/android-components/components/lib/llm-mlpa/src/main/java/mozilla/components/lib/llm/mlpa/MlpaLlm.kt @@ -16,19 +16,19 @@ import mozilla.components.lib.llm.mlpa.service.ChatService.Request.ModelID internal class MlpaLlm( val chatService: ChatService, val authorizationToken: AuthorizationToken, + val model: ModelID, ) : Llm { override suspend fun prompt(prompt: Prompt): Flow = chatService.completion( authorizationToken, - request = prompt.asRequest, + request = prompt.toRequest(model), ) } -internal val Prompt.asRequest - get() = Request( - model = ModelID.mozSummarization, - messages = buildList { - systemPrompt?.let { add(Message.system(it)) } - add(Message.user(userPrompt)) - }, - stream = true, - ) +internal fun Prompt.toRequest(model: ModelID) = Request( + model = model, + messages = buildList { + systemPrompt?.let { add(Message.system(it)) } + add(Message.user(userPrompt)) + }, + stream = true, +) diff --git a/mobile/android/android-components/components/lib/llm-mlpa/src/main/java/mozilla/components/lib/llm/mlpa/MlpaLlmProvider.kt b/mobile/android/android-components/components/lib/llm-mlpa/src/main/java/mozilla/components/lib/llm/mlpa/MlpaLlmProvider.kt index 2d6e2c675eb9f..25f701c0fe836 100644 --- a/mobile/android/android-components/components/lib/llm-mlpa/src/main/java/mozilla/components/lib/llm/mlpa/MlpaLlmProvider.kt +++ b/mobile/android/android-components/components/lib/llm-mlpa/src/main/java/mozilla/components/lib/llm/mlpa/MlpaLlmProvider.kt @@ -13,6 +13,7 @@ import mozilla.components.concept.llm.ErrorCode import mozilla.components.concept.llm.Llm import mozilla.components.concept.llm.LlmProvider import mozilla.components.lib.llm.mlpa.service.ChatService +import mozilla.components.lib.llm.mlpa.service.ChatService.Request.ModelID import mozilla.components.lib.llm.mlpa.service.ChatServiceError import mozilla.components.lib.llm.mlpa.service.MlpaService @@ -37,7 +38,13 @@ class MlpaLlmProvider( val storage: MlpaTokenStorage, val mlpaService: MlpaService, ) : CloudLlmProvider { - override val info = LlmProvider.Info(nameRes = R.string.mlpa_llm_provider_name, iconRes = R.drawable.firefox_icon) + private val model = ModelID.mozSummarization + + override val info = LlmProvider.Info( + nameRes = R.string.mlpa_llm_provider_name, + iconRes = R.drawable.firefox_icon, + model = model.value, + ) private val _state = MutableStateFlow(State.Available) /** @@ -55,7 +62,7 @@ class MlpaLlmProvider( */ override suspend fun prepare() { tokenProvider.fetchToken() - .onSuccess { _state.value = State.Ready(MlpaLlm(chatService, it)) } + .onSuccess { _state.value = State.Ready(MlpaLlm(chatService, it, model)) } .onFailure { _state.value = State.Unavailable( it as? Llm.Exception diff --git a/mobile/android/android-components/components/lib/llm-mlpa/src/test/java/mozilla/components/lib/llm/mlpa/MlpaLlmTest.kt b/mobile/android/android-components/components/lib/llm-mlpa/src/test/java/mozilla/components/lib/llm/mlpa/MlpaLlmTest.kt index c01ff3ceb0216..2a72a2f65908c 100644 --- a/mobile/android/android-components/components/lib/llm-mlpa/src/test/java/mozilla/components/lib/llm/mlpa/MlpaLlmTest.kt +++ b/mobile/android/android-components/components/lib/llm-mlpa/src/test/java/mozilla/components/lib/llm/mlpa/MlpaLlmTest.kt @@ -33,6 +33,7 @@ class MlpaLlmTest { successChatService.completion(token, request) }, authorizationToken = AuthorizationToken.Integrity("my-test-token"), + model = ChatService.Request.ModelID.mozSummarization, ) val actual = llm.prompt(Prompt("This is my prompt")).toList() @@ -49,6 +50,7 @@ class MlpaLlmTest { val llm = MlpaLlm( chatService = failureChatService, authorizationToken = AuthorizationToken.Integrity("my-test-token"), + model = ChatService.Request.ModelID.mozSummarization, ) llm.prompt(Prompt("This is my prompt")) @@ -67,6 +69,7 @@ class MlpaLlmTest { successChatService.completion(token, request) }, authorizationToken = AuthorizationToken.Integrity("my-test-token"), + model = ChatService.Request.ModelID.mozSummarization, ) llm.prompt(Prompt("user prompt", "system prompt")).toList() @@ -88,6 +91,7 @@ class MlpaLlmTest { successChatService.completion(token, request) }, authorizationToken = AuthorizationToken.Integrity("my-test-token"), + model = ChatService.Request.ModelID.mozSummarization, ) llm.prompt(Prompt("user prompt", null)).toList() diff --git a/mobile/android/fenix/app/metrics.yaml b/mobile/android/fenix/app/metrics.yaml index 7c1ec4fd99830..d91eb107557b3 100644 --- a/mobile/android/fenix/app/metrics.yaml +++ b/mobile/android/fenix/app/metrics.yaml @@ -15398,10 +15398,17 @@ ai_summarize: type: string error_type: description: | - If an error occurred, the resulting error code. Examples include: + If an error occurred, a human-readable name for the failure + (typically the exception class name, e.g. "RateLimited", + "RequestTooLarge"). + type: string + error_code: + description: | + If an error occurred, the integer error code associated with the + failure. Examples include: 1005 (meaning content was too large to summarize) 1007 (meaning LLM service has rate-limited the user) - type: string + type: quantity summarize_duration_ms: description: | Time in milliseconds between request and completion. @@ -15413,6 +15420,7 @@ ai_summarize: type: string bugs: - https://bugzilla.mozilla.org/show_bug.cgi?id=2025523 + - https://bugzilla.mozilla.org/show_bug.cgi?id=2036474 data_reviews: - https://phabricator.services.mozilla.com/D289882 data_sensitivity: @@ -15443,8 +15451,15 @@ ai_summarize: description: | Records when the user initiates summarization. (e.g. taps menu button, toolbar icon or shakes the device). + extra_keys: + trigger: + description: | + How the user initiated summarization. Possible values: + "shake", "menu". + type: string bugs: - https://bugzilla.mozilla.org/show_bug.cgi?id=2025523 + - https://bugzilla.mozilla.org/show_bug.cgi?id=2036474 data_reviews: - https://phabricator.services.mozilla.com/D289882 data_sensitivity: diff --git a/mobile/android/fenix/app/src/main/java/org/mozilla/fenix/summarization/SummarizationTelemetryMiddleware.kt b/mobile/android/fenix/app/src/main/java/org/mozilla/fenix/summarization/SummarizationTelemetryMiddleware.kt index d4d1fd4b72014..fc2bbce28c70f 100644 --- a/mobile/android/fenix/app/src/main/java/org/mozilla/fenix/summarization/SummarizationTelemetryMiddleware.kt +++ b/mobile/android/fenix/app/src/main/java/org/mozilla/fenix/summarization/SummarizationTelemetryMiddleware.kt @@ -76,11 +76,11 @@ class SummarizationTelemetryMiddleware( when (action) { ViewAppeared -> handleViewAppeared(stateBefore) is SummarizationRequested -> { - sessionTelemetry = sessionTelemetry.copy(model = action.info.nameRes.toString()) + sessionTelemetry = sessionTelemetry.copy(model = action.info.model) } is ContentExtracted -> handleExtractedContent(action.content) is SummarizationCompleted -> recordSummarizationCompleted() - is SummarizationFailed -> recordSummarizationCompleted(success = false, action.throwable.errorType) + is SummarizationFailed -> recordSummarizationCompleted(success = false, action.throwable) ViewDismissed -> { AiSummarize.closed.record( AiSummarize.ClosedExtra( @@ -119,8 +119,6 @@ class SummarizationTelemetryMiddleware( } private fun handleViewAppeared(stateBefore: SummarizationState) { - AiSummarize.requested.record() - timerId = AiSummarize.duration.start() if (stateBefore is SummarizationState.Inert) { val trigger = if (stateBefore.initializedWithShake) { SummarizationTrigger.SHAKE @@ -129,6 +127,10 @@ class SummarizationTelemetryMiddleware( } sessionTelemetry = sessionTelemetry.copy(trigger = trigger) } + AiSummarize.requested.record( + AiSummarize.RequestedExtra(trigger = sessionTelemetry.trigger?.toString()), + ) + timerId = AiSummarize.duration.start() } private fun handleExtractedContent(content: Content) { @@ -151,7 +153,7 @@ class SummarizationTelemetryMiddleware( ) } - private fun recordSummarizationCompleted(success: Boolean = true, errorType: String? = null) { + private fun recordSummarizationCompleted(success: Boolean = true, error: Throwable? = null) { timerId?.let { AiSummarize.duration.stopAndAccumulate(it) timerId = null @@ -161,7 +163,8 @@ class SummarizationTelemetryMiddleware( AiSummarize.CompletedExtra( connectionType = connectionType.toString(), contentType = sessionTelemetry.contentMetrics?.contentType, - errorType = errorType, + errorType = error?.errorType, + errorCode = error?.errorCode, language = sessionTelemetry.contentMetrics?.language, lengthChars = sessionTelemetry.contentMetrics?.charCount, lengthWords = sessionTelemetry.contentMetrics?.wordCount, @@ -173,4 +176,6 @@ class SummarizationTelemetryMiddleware( } } -private val Throwable.errorType get() = (this as? Llm.Exception)?.errorCode?.value?.toString() +private val Throwable.errorType get() = this::class.simpleName + +private val Throwable.errorCode get() = (this as? Llm.Exception)?.errorCode?.value diff --git a/mobile/android/fenix/app/src/test/java/org/mozilla/fenix/summarization/SummarizationTelemetryMiddlewareTest.kt b/mobile/android/fenix/app/src/test/java/org/mozilla/fenix/summarization/SummarizationTelemetryMiddlewareTest.kt index 076541f779a73..5c1e0d3fcf5b9 100644 --- a/mobile/android/fenix/app/src/test/java/org/mozilla/fenix/summarization/SummarizationTelemetryMiddlewareTest.kt +++ b/mobile/android/fenix/app/src/test/java/org/mozilla/fenix/summarization/SummarizationTelemetryMiddlewareTest.kt @@ -66,8 +66,11 @@ class SummarizationTelemetryMiddlewareTest { invokeMiddleware(ViewAppeared) invokeMiddleware(createContentExtractedAction()) - val extras = AiSummarize.started.testGetValue()!!.first().extra!! - assertEquals("SHAKE", extras["trigger"]) + val startedExtras = AiSummarize.started.testGetValue()!!.first().extra!! + assertEquals("SHAKE", startedExtras["trigger"]) + + val requestedExtras = AiSummarize.requested.testGetValue()!!.first().extra!! + assertEquals("SHAKE", requestedExtras["trigger"]) } @Test @@ -77,8 +80,11 @@ class SummarizationTelemetryMiddlewareTest { invokeMiddleware(ViewAppeared) invokeMiddleware(createContentExtractedAction()) - val extras = AiSummarize.started.testGetValue()!!.first().extra!! - assertEquals("MENU", extras["trigger"]) + val startedExtras = AiSummarize.started.testGetValue()!!.first().extra!! + assertEquals("MENU", startedExtras["trigger"]) + + val requestedExtras = AiSummarize.requested.testGetValue()!!.first().extra!! + assertEquals("MENU", requestedExtras["trigger"]) } @Test @@ -88,7 +94,7 @@ class SummarizationTelemetryMiddlewareTest { every { store.state } returns SummarizationState.Inert(initializedWithShake = false) invokeMiddleware(ViewAppeared) invokeMiddleware( - SummarizationRequested(LlmProvider.Info(nameRes = 42)), + SummarizationRequested(LlmProvider.Info(nameRes = 42, model = TEST_MODEL)), ) invokeMiddleware( createContentExtractedAction( @@ -106,7 +112,7 @@ class SummarizationTelemetryMiddlewareTest { val extras = snapshot.first().extra!! assertEquals("MENU", extras["trigger"]) - assertEquals("42", extras["model"]) + assertEquals(TEST_MODEL, extras["model"]) assertEquals("120", extras["length_words"]) assertEquals("15", extras["length_chars"]) assertEquals("[recipe]", extras["content_type"]) @@ -125,13 +131,14 @@ class SummarizationTelemetryMiddlewareTest { val extras = snapshot.first().extra!! assertEquals("true", extras["success"]) assertEquals("WIFI", extras["connection_type"]) - assertEquals("42", extras["model"]) + assertEquals(TEST_MODEL, extras["model"]) assertNull(extras["error_type"]) + assertNull(extras["error_code"]) assertNotNull(extras["summarize_duration_ms"]) } @Test - fun `WHEN SummarizationFailed is received THEN summarization_completed is recorded with success false`() { + fun `WHEN SummarizationFailed with Llm Exception THEN error_type is exception name and error_code is the int code`() { assertNull(AiSummarize.completed.testGetValue()) setupFullSession() @@ -143,7 +150,21 @@ class SummarizationTelemetryMiddlewareTest { val extras = snapshot.first().extra!! assertEquals("false", extras["success"]) - assertEquals(exception.errorCode.value.toString(), extras["error_type"]) + assertEquals("Exception", extras["error_type"]) + assertEquals("1001", extras["error_code"]) + } + + @Test + fun `WHEN SummarizationFailed with non-Llm throwable THEN error_type is class name and error_code is absent`() { + assertNull(AiSummarize.completed.testGetValue()) + + setupFullSession() + invokeMiddleware(SummarizationFailed(IllegalStateException("oops"))) + + val extras = AiSummarize.completed.testGetValue()!!.first().extra!! + assertEquals("false", extras["success"]) + assertEquals("IllegalStateException", extras["error_type"]) + assertNull(extras["error_code"]) } @Test @@ -160,12 +181,12 @@ class SummarizationTelemetryMiddlewareTest { every { store.state } returns SummarizationState.Inert(initializedWithShake = false) invokeMiddleware(ViewAppeared) invokeMiddleware( - SummarizationRequested(LlmProvider.Info(nameRes = 99)), + SummarizationRequested(LlmProvider.Info(nameRes = 99, model = "another-model")), ) invokeMiddleware(ViewDismissed) val extras = AiSummarize.closed.testGetValue()!!.first().extra!! - assertEquals("99", extras["model"]) + assertEquals("another-model", extras["model"]) } @Test @@ -234,7 +255,7 @@ class SummarizationTelemetryMiddlewareTest { every { store.state } returns SummarizationState.Inert(initializedWithShake = false) invokeMiddleware(ViewAppeared) invokeMiddleware( - SummarizationRequested(LlmProvider.Info(nameRes = 42)), + SummarizationRequested(LlmProvider.Info(nameRes = 42, model = TEST_MODEL)), ) invokeMiddleware(createContentExtractedAction()) } @@ -251,4 +272,8 @@ class SummarizationTelemetryMiddlewareTest { action = action, ) } + + private companion object { + const val TEST_MODEL = "moz-summarization" + } }