Voice Agent

RunAnywhere.loadLLMModel("smollm2-360m-instruct-q8_0")
RunAnywhere.loadSTTModel("sherpa-onnx-whisper-tiny.en")
RunAnywhere.loadTTSVoice("vits-piper-en_US-lessac-medium")

// Verify readiness
val ready = RunAnywhere.isVoiceAgentReady()

import com.runanywhere.sdk.public.extensions.VoiceAgent.VoiceSessionConfig

val config = VoiceSessionConfig(
    silenceDuration = 1.5,       // Seconds of silence before processing
    speechThreshold = 0.1f,      // Audio level threshold for speech detection
    autoPlayTTS = false,         // If true, SDK handles TTS playback
    continuousMode = true        // Auto-resume listening after each turn
)

import com.runanywhere.sdk.public.extensions.VoiceAgent.VoiceSessionEvent
import com.runanywhere.sdk.public.extensions.streamVoiceSession

lifecycleScope.launch {
    RunAnywhere.streamVoiceSession(audioFlow, config)
        .collect { event ->
            when (event) {
                is VoiceSessionEvent.Started -> {
                    updateUI("Session started")
                }
                is VoiceSessionEvent.Listening -> {
                    updateAudioLevel(event.audioLevel)
                }
                is VoiceSessionEvent.SpeechStarted -> {
                    updateUI("Speech detected...")
                }
                is VoiceSessionEvent.Processing -> {
                    updateUI("Processing...")
                }
                is VoiceSessionEvent.Transcribed -> {
                    updateUI("You: ${event.text}")
                }
                is VoiceSessionEvent.Responded -> {
                    updateUI("AI: ${event.text}")
                }
                is VoiceSessionEvent.Speaking -> {
                    updateUI("Speaking...")
                }
                is VoiceSessionEvent.TurnCompleted -> {
                    event.audio?.let { playAudio(it) }
                }
                is VoiceSessionEvent.Stopped -> {
                    updateUI("Session ended")
                }
                is VoiceSessionEvent.Error -> {
                    showError(event.message)
                }
            }
        }
}

sealed class VoiceSessionEvent {
    object Started : VoiceSessionEvent()
    data class Listening(val audioLevel: Float) : VoiceSessionEvent()
    object SpeechStarted : VoiceSessionEvent()
    object Processing : VoiceSessionEvent()
    data class Transcribed(val text: String) : VoiceSessionEvent()
    data class Responded(val text: String) : VoiceSessionEvent()
    object Speaking : VoiceSessionEvent()
    data class TurnCompleted(val audio: ByteArray?) : VoiceSessionEvent()
    object Stopped : VoiceSessionEvent()
    data class Error(val message: String) : VoiceSessionEvent()
}

import kotlinx.coroutines.channels.awaitClose
import kotlinx.coroutines.flow.callbackFlow

val audioFlow = callbackFlow<ByteArray> {
    val bufferSize = AudioRecord.getMinBufferSize(16000, AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT)
    val recorder = AudioRecord(MediaRecorder.AudioSource.MIC, 16000, AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT, bufferSize)
    recorder.startRecording()

    val buffer = ByteArray(1600) // 100ms at 16kHz
    while (isActive) {
        val read = recorder.read(buffer, 0, buffer.size)
        if (read > 0) trySend(buffer.copyOf(read))
    }

    awaitClose { recorder.stop(); recorder.release() }
}

class VoiceAssistantActivity : AppCompatActivity() {

    override fun onCreate(savedInstanceState: Bundle?) {
        super.onCreate(savedInstanceState)
        setContentView(R.layout.activity_voice_assistant)

        lifecycleScope.launch {
            // Configure voice agent
            RunAnywhere.configureVoiceAgent(VoiceAgentConfiguration(
                sttModelId = "whisper-tiny",
                llmModelId = "qwen-0.5b",
                ttsVoiceId = "en-us-default",
                systemPrompt = """
                    You are a helpful voice assistant.
                    Keep responses under 2 sentences.
                    Be friendly and conversational.
                """.trimIndent()
            ))

            // Check if ready
            if (!RunAnywhere.isVoiceAgentReady()) {
                showMessage("Loading models...")
                RunAnywhere.initializeVoiceAgentWithLoadedModels()
            }
        }

        startButton.setOnClickListener {
            startVoiceSession()
        }

        stopButton.setOnClickListener {
            lifecycleScope.launch {
                RunAnywhere.stopVoiceSession()
            }
        }
    }

    private fun startVoiceSession() {
        lifecycleScope.launch {
            RunAnywhere.startVoiceSession()
                .collect { event ->
                    runOnUiThread {
                        when (event) {
                            is VoiceSessionEvent.Listening -> {
                                statusText.text = "🎤 Listening..."
                                micButton.setColorFilter(Color.RED)
                            }
                            is VoiceSessionEvent.Transcribed -> {
                                transcriptionText.text = event.text
                            }
                            is VoiceSessionEvent.Thinking -> {
                                statusText.text = "🤔 Thinking..."
                            }
                            is VoiceSessionEvent.Responded -> {
                                responseText.text = event.text
                            }
                            is VoiceSessionEvent.Speaking -> {
                                statusText.text = "🔊 Speaking..."
                            }
                            is VoiceSessionEvent.Idle -> {
                                statusText.text = "Ready"
                                micButton.clearColorFilter()
                            }
                            is VoiceSessionEvent.Error -> {
                                Toast.makeText(this@VoiceAssistantActivity,
                                    event.message, Toast.LENGTH_SHORT).show()
                            }
                        }
                    }
                }
        }
    }
}

val states = RunAnywhere.voiceAgentComponentStates()
println("STT ready: ${states.sttReady}")
println("LLM ready: ${states.llmReady}")
println("TTS ready: ${states.ttsReady}")
println("VAD ready: ${states.vadReady}")