diff --git a/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/MainActivity.kt b/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/MainActivity.kt index 8cbafed1f6..3cb54e631e 100644 --- a/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/MainActivity.kt +++ b/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/MainActivity.kt @@ -26,16 +26,17 @@ import androidx.navigation.NavDestination import androidx.navigation.compose.NavHost import androidx.navigation.compose.composable import androidx.navigation.compose.rememberNavController -import com.google.firebase.quickstart.ai.feature.live.BidiViewModel import com.google.firebase.quickstart.ai.feature.hybrid.HybridInferenceViewModel +import com.google.firebase.quickstart.ai.feature.live.StreamAudioViewModel +import com.google.firebase.quickstart.ai.feature.live.StreamVideoViewModel import com.google.firebase.quickstart.ai.feature.text.ChatViewModel import com.google.firebase.quickstart.ai.feature.text.ServerPromptTemplateViewModel import com.google.firebase.quickstart.ai.feature.text.SvgViewModel import com.google.firebase.quickstart.ai.ui.ChatScreen +import com.google.firebase.quickstart.ai.ui.HybridInferenceScreen import com.google.firebase.quickstart.ai.ui.ServerPromptScreen import com.google.firebase.quickstart.ai.ui.StreamRealtimeScreen import com.google.firebase.quickstart.ai.ui.StreamRealtimeVideoScreen -import com.google.firebase.quickstart.ai.ui.HybridInferenceScreen import com.google.firebase.quickstart.ai.ui.SvgScreen import com.google.firebase.quickstart.ai.ui.navigation.FIREBASE_AI_SAMPLES import com.google.firebase.quickstart.ai.ui.navigation.MainMenuScreen @@ -107,14 +108,14 @@ class MainActivity : ComponentActivity() { } ScreenType.BIDI -> { - (vm as? BidiViewModel)?.let { + (vm as? StreamAudioViewModel)?.let { @SuppressLint("MissingPermission") StreamRealtimeScreen(it) } } ScreenType.BIDI_VIDEO -> { - (vm as? BidiViewModel)?.let { + (vm as? StreamVideoViewModel)?.let { @SuppressLint("MissingPermission") StreamRealtimeVideoScreen(it) } diff --git a/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/feature/live/BidiViewModel.kt b/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/feature/live/BidiViewModel.kt deleted file mode 100644 index 547931580c..0000000000 --- a/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/feature/live/BidiViewModel.kt +++ /dev/null @@ -1,45 +0,0 @@ -package com.google.firebase.quickstart.ai.feature.live - -import android.annotation.SuppressLint -import android.graphics.Bitmap -import androidx.lifecycle.ViewModel -import androidx.lifecycle.viewModelScope -import com.google.firebase.ai.type.FunctionCallPart -import com.google.firebase.ai.type.FunctionResponsePart -import com.google.firebase.ai.type.InlineData -import com.google.firebase.ai.type.LiveSession -import com.google.firebase.ai.type.PublicPreviewAPI -import kotlinx.coroutines.launch -import kotlinx.serialization.json.JsonObject -import java.io.ByteArrayOutputStream - - -@OptIn(PublicPreviewAPI::class) -abstract class BidiViewModel : ViewModel() { - protected lateinit var liveSession: LiveSession - - open fun handler(functionCall: FunctionCallPart): FunctionResponsePart { - return FunctionResponsePart(functionCall.name, JsonObject(emptyMap()), functionCall.id) - } - - // The permission check is handled by the view that calls this function. - @SuppressLint("MissingPermission") - suspend fun startConversation() { - liveSession.startAudioConversation(::handler) - } - - fun endConversation() { - liveSession.stopAudioConversation() - } - - fun sendVideoFrame(frame: Bitmap) { - viewModelScope.launch { - // Directly compress the Bitmap to a ByteArray - val byteArrayOutputStream = ByteArrayOutputStream() - frame.compress(Bitmap.CompressFormat.JPEG, 80, byteArrayOutputStream) - val jpegBytes = byteArrayOutputStream.toByteArray() - - liveSession.sendVideoRealtime(InlineData(jpegBytes, "image/jpeg")) - } - } -} diff --git a/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/feature/live/StreamAudioViewModel.kt b/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/feature/live/StreamAudioViewModel.kt index edea6ddf16..ed3462a773 100644 --- a/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/feature/live/StreamAudioViewModel.kt +++ b/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/feature/live/StreamAudioViewModel.kt @@ -1,19 +1,31 @@ package com.google.firebase.quickstart.ai.feature.live +import android.annotation.SuppressLint +import androidx.lifecycle.ViewModel +import androidx.lifecycle.viewModelScope import com.google.firebase.Firebase import com.google.firebase.ai.ai +import com.google.firebase.ai.type.AudioTranscriptionConfig import com.google.firebase.ai.type.FunctionCallPart import com.google.firebase.ai.type.FunctionDeclaration import com.google.firebase.ai.type.FunctionResponsePart import com.google.firebase.ai.type.GenerativeBackend +import com.google.firebase.ai.type.LiveSession import com.google.firebase.ai.type.PublicPreviewAPI import com.google.firebase.ai.type.ResponseModality import com.google.firebase.ai.type.Schema import com.google.firebase.ai.type.SpeechConfig import com.google.firebase.ai.type.Tool +import com.google.firebase.ai.type.Transcription import com.google.firebase.ai.type.Voice import com.google.firebase.ai.type.liveGenerationConfig import com.google.firebase.quickstart.ai.feature.text.functioncalling.WeatherRepository.Companion.fetchWeather +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.update +import kotlinx.coroutines.launch import kotlinx.coroutines.runBlocking import kotlinx.serialization.Serializable import kotlinx.serialization.json.JsonObject @@ -22,12 +34,29 @@ import kotlinx.serialization.json.jsonPrimitive @Serializable object StreamRealtimeAudioRoute +enum class TranscriptionSpeaker { + USER, + MODEL +} + +data class TranscriptionItem( + val speaker: TranscriptionSpeaker, + val text: String +) + @OptIn(PublicPreviewAPI::class) -class StreamAudioViewModel : BidiViewModel() { +class StreamAudioViewModel : ViewModel() { + private var liveSession: LiveSession + + private val _transcriptions = MutableStateFlow>(emptyList()) + val transcriptions: StateFlow> = _transcriptions.asStateFlow() + init { val liveGenerationConfig = liveGenerationConfig { speechConfig = SpeechConfig(voice = Voice("CHARON")) responseModality = ResponseModality.AUDIO + inputAudioTranscription = AudioTranscriptionConfig() + outputAudioTranscription = AudioTranscriptionConfig() } val liveModel = @@ -60,7 +89,7 @@ class StreamAudioViewModel : BidiViewModel() { runBlocking { liveSession = liveModel.connect() } } - override fun handler(functionCall: FunctionCallPart): FunctionResponsePart { + fun handleFunctionCall(functionCall: FunctionCallPart): FunctionResponsePart { val response: JsonObject if (functionCall.name == "fetchWeather") { val city = functionCall.args["city"]?.jsonPrimitive?.content @@ -79,4 +108,47 @@ class StreamAudioViewModel : BidiViewModel() { } return FunctionResponsePart(functionCall.name, response, functionCall.id) } + + private fun handleTranscription(input: Transcription?, output: Transcription?) { + input?.text?.let { text -> + if (text.isNotEmpty()) { + _transcriptions.update { current -> + val last = current.lastOrNull() + if (last != null && last.speaker == TranscriptionSpeaker.USER) { + current.dropLast(1) + last.copy(text = last.text + text) + } else { + current + TranscriptionItem(speaker = TranscriptionSpeaker.USER, text = text) + } + } + } + } + output?.text?.let { text -> + if (text.isNotEmpty()) { + _transcriptions.update { current -> + val last = current.lastOrNull() + if (last != null && last.speaker == TranscriptionSpeaker.MODEL) { + current.dropLast(1) + last.copy(text = last.text + text) + } else { + current + TranscriptionItem(speaker = TranscriptionSpeaker.MODEL, text = text) + } + } + } + } + } + + // The permission check is handled by the view that calls this function. + @SuppressLint("MissingPermission") + fun startConversation() { + viewModelScope.launch(Dispatchers.IO) { + liveSession.startAudioConversation(::handleFunctionCall, ::handleTranscription) + } + } + + fun endConversation() { + liveSession.stopAudioConversation() + } + + fun clearTranscriptions() { + _transcriptions.value = emptyList() + } } diff --git a/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/feature/live/StreamVideoViewModel.kt b/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/feature/live/StreamVideoViewModel.kt index 8e715785cb..89440f902d 100644 --- a/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/feature/live/StreamVideoViewModel.kt +++ b/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/feature/live/StreamVideoViewModel.kt @@ -1,25 +1,47 @@ package com.google.firebase.quickstart.ai.feature.live +import android.annotation.SuppressLint +import android.graphics.Bitmap +import androidx.lifecycle.ViewModel +import androidx.lifecycle.viewModelScope import com.google.firebase.Firebase import com.google.firebase.ai.ai +import com.google.firebase.ai.type.AudioTranscriptionConfig import com.google.firebase.ai.type.GenerativeBackend +import com.google.firebase.ai.type.InlineData +import com.google.firebase.ai.type.LiveSession import com.google.firebase.ai.type.PublicPreviewAPI import com.google.firebase.ai.type.ResponseModality import com.google.firebase.ai.type.SpeechConfig +import com.google.firebase.ai.type.Transcription import com.google.firebase.ai.type.Voice import com.google.firebase.ai.type.liveGenerationConfig +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.update +import kotlinx.coroutines.launch import kotlinx.coroutines.runBlocking import kotlinx.serialization.Serializable +import java.io.ByteArrayOutputStream @Serializable object StreamRealtimeVideoRoute @OptIn(PublicPreviewAPI::class) -class StreamVideoViewModel : BidiViewModel() { +class StreamVideoViewModel : ViewModel() { + private var liveSession: LiveSession + + private val _transcriptions = MutableStateFlow>(emptyList()) + val transcriptions: StateFlow> = _transcriptions.asStateFlow() + init { val liveGenerationConfig = liveGenerationConfig { speechConfig = SpeechConfig(voice = Voice("CHARON")) responseModality = ResponseModality.AUDIO + inputAudioTranscription = AudioTranscriptionConfig() + outputAudioTranscription = AudioTranscriptionConfig() } // Note that each backend supports a different set of models. @@ -33,4 +55,58 @@ class StreamVideoViewModel : BidiViewModel() { ) runBlocking { liveSession = liveModel.connect() } } + + private fun handleTranscription(input: Transcription?, output: Transcription?) { + input?.text?.let { text -> + if (text.isNotEmpty()) { + _transcriptions.update { current -> + val last = current.lastOrNull() + if (last != null && last.speaker == TranscriptionSpeaker.USER) { + current.dropLast(1) + last.copy(text = last.text + text) + } else { + current + TranscriptionItem(speaker = TranscriptionSpeaker.USER, text = text) + } + } + } + } + output?.text?.let { text -> + if (text.isNotEmpty()) { + _transcriptions.update { current -> + val last = current.lastOrNull() + if (last != null && last.speaker == TranscriptionSpeaker.MODEL) { + current.dropLast(1) + last.copy(text = last.text + text) + } else { + current + TranscriptionItem(speaker = TranscriptionSpeaker.MODEL, text = text) + } + } + } + } + } + + // The permission check is handled by the view that calls this function. + @SuppressLint("MissingPermission") + fun startConversation() { + viewModelScope.launch(Dispatchers.IO) { + liveSession.startAudioConversation(null, ::handleTranscription) + } + } + + fun endConversation() { + liveSession.stopAudioConversation() + } + + fun clearTranscriptions() { + _transcriptions.value = emptyList() + } + + fun sendVideoFrame(frame: Bitmap) { + viewModelScope.launch { + // Directly compress the Bitmap to a ByteArray + val byteArrayOutputStream = ByteArrayOutputStream() + frame.compress(Bitmap.CompressFormat.JPEG, 80, byteArrayOutputStream) + val jpegBytes = byteArrayOutputStream.toByteArray() + + liveSession.sendVideoRealtime(InlineData(jpegBytes, "image/jpeg")) + } + } } diff --git a/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/ui/StreamRealtimeScreen.kt b/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/ui/StreamRealtimeScreen.kt index 01df3111d9..7b4f97c174 100644 --- a/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/ui/StreamRealtimeScreen.kt +++ b/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/ui/StreamRealtimeScreen.kt @@ -4,16 +4,25 @@ import android.Manifest import androidx.annotation.RequiresPermission import androidx.compose.animation.animateContentSize import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.Spacer import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.fillMaxWidth import androidx.compose.foundation.layout.height import androidx.compose.foundation.layout.padding import androidx.compose.foundation.layout.size +import androidx.compose.foundation.layout.widthIn +import androidx.compose.foundation.lazy.LazyColumn +import androidx.compose.foundation.lazy.items +import androidx.compose.foundation.lazy.rememberLazyListState import androidx.compose.foundation.shape.CircleShape +import androidx.compose.foundation.shape.RoundedCornerShape import androidx.compose.material.icons.Icons import androidx.compose.material.icons.filled.CallEnd import androidx.compose.material.icons.filled.Mic +import androidx.compose.material3.Card +import androidx.compose.material3.CardDefaults import androidx.compose.material3.Icon import androidx.compose.material3.IconButton import androidx.compose.material3.IconButtonDefaults @@ -21,6 +30,11 @@ import androidx.compose.material3.MaterialTheme import androidx.compose.material3.Surface import androidx.compose.material3.Text import androidx.compose.runtime.Composable +import androidx.compose.runtime.DisposableEffect +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.draw.clip @@ -28,22 +42,31 @@ import androidx.compose.ui.graphics.Color import androidx.compose.ui.text.font.FontWeight import androidx.compose.ui.unit.dp import androidx.compose.ui.unit.sp -import androidx.compose.runtime.remember -import androidx.compose.runtime.mutableStateOf - -import androidx.lifecycle.viewmodel.compose.viewModel -import com.google.firebase.quickstart.ai.feature.live.BidiViewModel -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.launch - +import androidx.lifecycle.compose.collectAsStateWithLifecycle +import com.google.firebase.quickstart.ai.feature.live.StreamAudioViewModel +import com.google.firebase.quickstart.ai.feature.live.TranscriptionItem +import com.google.firebase.quickstart.ai.feature.live.TranscriptionSpeaker @RequiresPermission(Manifest.permission.RECORD_AUDIO) @Composable -fun StreamRealtimeScreen(bidiView: BidiViewModel) { +fun StreamRealtimeScreen(viewModel: StreamAudioViewModel) { val isConversationActive = remember { mutableStateOf(false) } - val backgroundColor = - MaterialTheme.colorScheme.background + val transcriptions by viewModel.transcriptions.collectAsStateWithLifecycle() + val listState = rememberLazyListState() + + DisposableEffect(Unit) { + onDispose { + viewModel.endConversation() + } + } + + LaunchedEffect(transcriptions.size, transcriptions.lastOrNull()?.text) { + if (transcriptions.isNotEmpty()) { + listState.animateScrollToItem(transcriptions.size - 1) + } + } + + val backgroundColor = MaterialTheme.colorScheme.background Surface( modifier = Modifier.fillMaxSize(), color = backgroundColor @@ -52,78 +75,107 @@ fun StreamRealtimeScreen(bidiView: BidiViewModel) { modifier = Modifier .fillMaxSize() .padding(16.dp), - horizontalAlignment = Alignment.CenterHorizontally, - verticalArrangement = Arrangement.Center + horizontalAlignment = Alignment.CenterHorizontally ) { - // The content will animate its size when it changes + // Header with status description Column( horizontalAlignment = Alignment.CenterHorizontally, - modifier = Modifier.animateContentSize() + modifier = Modifier + .padding(vertical = 8.dp) + .animateContentSize() ) { if (isConversationActive.value) { - // Active state UI Text( text = "Conversation Active", - fontSize = 22.sp, + fontSize = 20.sp, fontWeight = FontWeight.Bold, color = MaterialTheme.colorScheme.onSurface ) - Spacer(modifier = Modifier.height(8.dp)) + Spacer(modifier = Modifier.height(4.dp)) Text( - text = "Tap the end button to stop", - fontSize = 18.sp, + text = "Listening and speaking with Gemini...", + fontSize = 15.sp, color = MaterialTheme.colorScheme.onSurfaceVariant ) } else { - // Idle state UI Text( text = "Start Conversation", - fontSize = 22.sp, + fontSize = 20.sp, fontWeight = FontWeight.Bold, color = MaterialTheme.colorScheme.onSurface ) - Spacer(modifier = Modifier.height(8.dp)) + Spacer(modifier = Modifier.height(4.dp)) Text( text = "Tap the microphone to begin", - fontSize = 18.sp, + fontSize = 15.sp, color = MaterialTheme.colorScheme.onSurfaceVariant ) } } - Spacer(modifier = Modifier.height(80.dp)) + // Transcriptions area + Box( + modifier = Modifier + .weight(1f) + .fillMaxWidth() + .padding(vertical = 8.dp), + contentAlignment = Alignment.Center + ) { + if (transcriptions.isEmpty()) { + Text( + text = if (isConversationActive.value) { + "Transcriptions will appear here..." + } else { + "No active transcription" + }, + color = MaterialTheme.colorScheme.outline, + style = MaterialTheme.typography.bodyMedium + ) + } else { + LazyColumn( + state = listState, + modifier = Modifier.fillMaxSize(), + verticalArrangement = Arrangement.spacedBy(8.dp) + ) { + items(transcriptions) { item -> + TranscriptionBubble(item) + } + } + } + } - // The main button with pulsing animation + // Control button if (isConversationActive.value) { - // Button to end the conversation IconButton( onClick = { - bidiView.endConversation() - isConversationActive.value = false }, + viewModel.endConversation() + isConversationActive.value = false + }, modifier = Modifier - .size(90.dp) + .padding(bottom = 16.dp) + .size(80.dp) .clip(CircleShape), colors = IconButtonDefaults.iconButtonColors( - containerColor = Color(0xFFE63946), // A nice red color + containerColor = Color(0xFFE63946), contentColor = Color.White ) ) { Icon( imageVector = Icons.Default.CallEnd, contentDescription = "End Conversation", - modifier = Modifier.size(48.dp) + modifier = Modifier.size(40.dp) ) } } else { - // Button to start the conversation IconButton( onClick = { - CoroutineScope(Dispatchers.IO).launch { - bidiView.startConversation() - } - isConversationActive.value = true }, + viewModel.clearTranscriptions() + viewModel.startConversation() + isConversationActive.value = true + }, modifier = Modifier - .size(90.dp) + .padding(bottom = 16.dp) + .size(80.dp) .clip(CircleShape), colors = IconButtonDefaults.iconButtonColors( containerColor = MaterialTheme.colorScheme.primary, @@ -133,10 +185,57 @@ fun StreamRealtimeScreen(bidiView: BidiViewModel) { Icon( imageVector = Icons.Default.Mic, contentDescription = "Start Conversation", - modifier = Modifier.size(48.dp) + modifier = Modifier.size(40.dp) ) } } } } } + +@Composable +private fun TranscriptionBubble(item: TranscriptionItem) { + val isModel = item.speaker == TranscriptionSpeaker.MODEL + val backgroundColor = if (isModel) { + MaterialTheme.colorScheme.secondaryContainer + } else { + MaterialTheme.colorScheme.primaryContainer + } + val textColor = if (isModel) { + MaterialTheme.colorScheme.onSecondaryContainer + } else { + MaterialTheme.colorScheme.onPrimaryContainer + } + val bubbleShape = if (isModel) { + RoundedCornerShape(4.dp, 16.dp, 16.dp, 16.dp) + } else { + RoundedCornerShape(16.dp, 4.dp, 16.dp, 16.dp) + } + val horizontalAlignment = if (isModel) Alignment.Start else Alignment.End + + Column( + horizontalAlignment = horizontalAlignment, + modifier = Modifier + .fillMaxWidth() + .padding(horizontal = 4.dp, vertical = 2.dp) + ) { + Text( + text = if (isModel) "Gemini" else "You", + style = MaterialTheme.typography.labelSmall, + color = MaterialTheme.colorScheme.outline, + modifier = Modifier.padding(bottom = 2.dp, start = 4.dp, end = 4.dp) + ) + Card( + colors = CardDefaults.cardColors(containerColor = backgroundColor), + shape = bubbleShape, + modifier = Modifier.widthIn(min = 40.dp, max = 320.dp) + ) { + Text( + text = item.text, + color = textColor, + style = MaterialTheme.typography.bodyMedium, + modifier = Modifier.padding(12.dp) + ) + } + } +} diff --git a/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/ui/StreamRealtimeVideoScreen.kt b/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/ui/StreamRealtimeVideoScreen.kt index 6b42a8754b..b197c5ffa0 100644 --- a/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/ui/StreamRealtimeVideoScreen.kt +++ b/firebase-ai/app/src/main/java/com/google/firebase/quickstart/ai/ui/StreamRealtimeVideoScreen.kt @@ -5,10 +5,20 @@ import android.content.pm.PackageManager import androidx.activity.compose.rememberLauncherForActivityResult import androidx.activity.result.contract.ActivityResultContracts import androidx.annotation.RequiresPermission +import androidx.compose.foundation.layout.Arrangement import androidx.compose.foundation.layout.Box import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.fillMaxHeight import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.widthIn +import androidx.compose.foundation.lazy.LazyColumn +import androidx.compose.foundation.lazy.items +import androidx.compose.foundation.lazy.rememberLazyListState +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.material3.Card +import androidx.compose.material3.CardDefaults import androidx.compose.material3.MaterialTheme import androidx.compose.material3.Surface import androidx.compose.material3.Text @@ -20,20 +30,23 @@ import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.remember import androidx.compose.runtime.rememberCoroutineScope import androidx.compose.runtime.setValue +import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.unit.dp import androidx.core.content.ContextCompat -import androidx.lifecycle.viewmodel.compose.viewModel -import com.google.firebase.quickstart.ai.feature.live.BidiViewModel +import androidx.lifecycle.compose.collectAsStateWithLifecycle +import com.google.firebase.quickstart.ai.feature.live.StreamVideoViewModel +import com.google.firebase.quickstart.ai.feature.live.TranscriptionItem +import com.google.firebase.quickstart.ai.feature.live.TranscriptionSpeaker import kotlinx.coroutines.launch - @RequiresPermission(allOf = [Manifest.permission.RECORD_AUDIO, Manifest.permission.CAMERA]) @Composable -fun StreamRealtimeVideoScreen(bidiView: BidiViewModel) { +fun StreamRealtimeVideoScreen(viewModel: StreamVideoViewModel) { val backgroundColor = MaterialTheme.colorScheme.background - - val scope = rememberCoroutineScope() + val transcriptions by viewModel.transcriptions.collectAsStateWithLifecycle() + val listState = rememberLazyListState() val context = LocalContext.current var hasPermissions by remember { @@ -57,25 +70,115 @@ fun StreamRealtimeVideoScreen(bidiView: BidiViewModel) { } } + LaunchedEffect(transcriptions.size, transcriptions.lastOrNull()?.text) { + if (transcriptions.isNotEmpty()) { + listState.animateScrollToItem(transcriptions.size - 1) + } + } + DisposableEffect(hasPermissions) { if (hasPermissions) { - scope.launch { bidiView.startConversation() } + viewModel.clearTranscriptions() + viewModel.startConversation() } - onDispose { bidiView.endConversation() } + onDispose { viewModel.endConversation() } } Surface(modifier = Modifier.fillMaxSize(), color = backgroundColor) { Column(modifier = Modifier.fillMaxSize()) { if (hasPermissions) { - Box(modifier = Modifier.fillMaxSize()) { - CameraView( - modifier = Modifier.fillMaxHeight(0.5f), - onFrameCaptured = { bitmap -> bidiView.sendVideoFrame(bitmap) }, - ) + CameraView( + modifier = Modifier + .fillMaxWidth() + .fillMaxHeight(0.5f), + onFrameCaptured = { bitmap -> viewModel.sendVideoFrame(bitmap) }, + ) + Box( + modifier = Modifier + .fillMaxWidth() + .weight(1f) + .padding(8.dp), + contentAlignment = Alignment.Center + ) { + if (transcriptions.isEmpty()) { + Text( + text = "Live transcriptions will appear here...", + color = MaterialTheme.colorScheme.outline, + style = MaterialTheme.typography.bodyMedium + ) + } else { + LazyColumn( + state = listState, + modifier = Modifier.fillMaxSize(), + verticalArrangement = Arrangement.spacedBy(8.dp) + ) { + items(transcriptions) { item -> + VideoTranscriptionBubble(item) + } + } + } } } else { - Text("Camera and audio permissions are required to use this feature.") + Box( + modifier = Modifier + .fillMaxSize() + .padding(16.dp), + contentAlignment = Alignment.Center + ) { + Text( + text = "Camera and audio permissions are required to use this feature.", + color = MaterialTheme.colorScheme.error, + style = MaterialTheme.typography.bodyLarge + ) + } } } } } + +@Composable +private fun VideoTranscriptionBubble(item: TranscriptionItem) { + val isModel = item.speaker == TranscriptionSpeaker.MODEL + val backgroundColor = if (isModel) { + MaterialTheme.colorScheme.secondaryContainer + } else { + MaterialTheme.colorScheme.primaryContainer + } + val textColor = if (isModel) { + MaterialTheme.colorScheme.onSecondaryContainer + } else { + MaterialTheme.colorScheme.onPrimaryContainer + } + val bubbleShape = if (isModel) { + RoundedCornerShape(4.dp, 16.dp, 16.dp, 16.dp) + } else { + RoundedCornerShape(16.dp, 4.dp, 16.dp, 16.dp) + } + val horizontalAlignment = if (isModel) Alignment.Start else Alignment.End + + Column( + horizontalAlignment = horizontalAlignment, + modifier = Modifier + .fillMaxWidth() + .padding(horizontal = 4.dp, vertical = 2.dp) + ) { + Text( + text = if (isModel) "Gemini" else "You", + style = MaterialTheme.typography.labelSmall, + color = MaterialTheme.colorScheme.outline, + modifier = Modifier.padding(bottom = 2.dp, start = 4.dp, end = 4.dp) + ) + Card( + colors = CardDefaults.cardColors(containerColor = backgroundColor), + shape = bubbleShape, + modifier = Modifier.widthIn(min = 40.dp, max = 320.dp) + ) { + Text( + text = item.text, + color = textColor, + style = MaterialTheme.typography.bodyMedium, + modifier = Modifier.padding(12.dp) + ) + } + } +}