Vision Language Models

val result = RunAnywhere.vlm.generate(
    image = ImageInput.bitmap(bitmap),
    prompt = "What is in this photo?"
)

println(result.text)

suspend fun generate(image: ImageInput, prompt: String, options: LlmOptions? = null): GenerationResult
fun generateStream(image: ImageInput, prompt: String, options: LlmOptions? = null): Flow<GenerationEvent>

ImageInput.file(path: String)
ImageInput.bytes(data: ByteArray)
ImageInput.rawRgb(data: ByteArray, width: Int, height: Int)
ImageInput.rawRgba(data: ByteArray, width: Int, height: Int)
ImageInput.bitmap(bitmap: Bitmap)

val launcher = rememberLauncherForActivityResult(
    ActivityResultContracts.PickVisualMedia()
) { uri ->
    uri ?: return@rememberLauncherForActivityResult
    viewModel.describe(uri)
}

fun describe(uri: Uri) {
    viewModelScope.launch {
        val bitmap = context.contentResolver.openInputStream(uri).use {
            BitmapFactory.decodeStream(it)
        } ?: return@launch

        val result = RunAnywhere.vlm.generate(
            image = ImageInput.bitmap(bitmap),
            prompt = "Describe this image in one sentence."
        )
        _caption.value = result.text
    }
}

imageAnalysis.setAnalyzer(executor) { proxy ->
    val bitmap = proxy.toBitmap()
    proxy.close()

    if (inFlight.compareAndSet(false, true)) {
        viewModelScope.launch {
            try {
                val result = RunAnywhere.vlm.generate(
                    image = ImageInput.bitmap(bitmap),
                    prompt = "What do you see?"
                )
                _caption.value = result.text
            } finally {
                inFlight.set(false)
            }
        }
    }
}

RunAnywhere.vlm.generateStream(ImageInput.bitmap(bitmap), "Describe this in detail.")
    .collect { event ->
        if (event is GenerationEvent.Token) caption += event.text
    }