Vision Language Models

const result = await RunAnywhere.vlm.generate(
  await ImageInput.element(img),
  'What is in this photo?'
)

console.log(result.text)

generate(image: ImageInput, prompt: string, options?: LlmOptions): Promise<GenerationResult>
generateStream(image: ImageInput, prompt: string, options?: LlmOptions): AsyncIterable<GenerationEvent>

ImageInput.file(path: string)
ImageInput.bytes(data: Uint8Array, mimeType = 'image/jpeg')
ImageInput.base64(value: string, mediaType = 'image/jpeg')
ImageInput.rawRgb(data: Uint8Array, width: number, height: number)
ImageInput.rawRgba(data: Uint8Array, width: number, height: number)
ImageInput.element(source: HTMLImageElement | HTMLCanvasElement | ImageBitmap): Promise<ImageInput>

const input = document.querySelector<HTMLInputElement>('#photo')!

input.addEventListener('change', async () => {
  const file = input.files?.[0]
  if (!file) return

  const bitmap = await createImageBitmap(file)
  const result = await RunAnywhere.vlm.generate(
    await ImageInput.element(bitmap),
    'Describe this image in one sentence.'
  )

  caption.textContent = result.text
  bitmap.close()
})

import { VideoCapture } from '@runanywhere/web/browser'

const capture = new VideoCapture()
await capture.start()

let inFlight = false

setInterval(async () => {
  if (inFlight) return
  inFlight = true

  try {
    const frame = await capture.grab()
    const result = await RunAnywhere.vlm.generate(
      await ImageInput.element(frame.bitmap),
      'What do you see?'
    )
    caption.textContent = result.text
  } finally {
    inFlight = false
  }
}, 2000)

for await (const event of RunAnywhere.vlm.generateStream(image, 'Describe this in detail.')) {
  if (event.type === 'textDelta') caption.textContent += event.text
}