Vision Language Models

import { RunAnywhere, ImageInputs } from '@runanywhere/core'

const result = await RunAnywhere.vlm.generate(ImageInputs.file(path), 'What is in this photo?')

console.log(result.text)

generate(image: ImageInput, prompt: string, options?: LlmOptions): Promise<GenerationResult>
generateStream(image: ImageInput, prompt: string, options?: LlmOptions): AsyncIterable<GenerationEvent>

ImageInputs.file(path: string)
ImageInputs.bytes(data: Uint8Array)
ImageInputs.base64(value: string)
ImageInputs.rawRgb(data: Uint8Array, width: number, height: number)
ImageInputs.rawRgba(data: Uint8Array, width: number, height: number)

import * as ImagePicker from 'expo-image-picker'

async function describe(): Promise<void> {
  const result = await ImagePicker.launchImageLibraryAsync({ base64: true })
  if (result.canceled) return

  const asset = result.assets[0]
  const image = asset.base64 ? ImageInputs.base64(asset.base64) : ImageInputs.file(asset.uri)

  const answer = await RunAnywhere.vlm.generate(image, 'Describe this image in one sentence.')

  setCaption(answer.text)
}

const inFlight = useRef(false)

const onFrame = useCallback(async (uri: string) => {
  if (inFlight.current) return
  inFlight.current = true

  try {
    const result = await RunAnywhere.vlm.generate(ImageInputs.file(uri), 'What do you see?')
    setCaption(result.text)
  } finally {
    inFlight.current = false
  }
}, [])

for await (const event of RunAnywhere.vlm.generateStream(image, 'Describe this in detail.')) {
  if (event.type === 'textDelta') setCaption((c) => c + event.text)
}