Vision Language Models

const result = await RunAnywhere.vlm.generate(
  RunAnywhere.image.bytes(data),
  'What is in this screenshot?'
)

console.log(result.text)

generate(image: ImageInput, prompt: string, options?: LlmOptions): Promise<GenerationResult>
generateStream(image: ImageInput, prompt: string, options?: LlmOptions): AsyncIterableIterator<GenerationEvent>

RunAnywhere.image.file(path)
RunAnywhere.image.bytes(data)
RunAnywhere.image.rawRgb(data, width, height)

import { dialog } from 'electron'

const { filePaths } = await dialog.showOpenDialog({
  filters: [{ name: 'Images', extensions: ['png', 'jpg', 'jpeg'] }],
})

if (filePaths[0]) {
  const result = await RunAnywhere.vlm.generate(
    RunAnywhere.image.file(filePaths[0]),
    'Describe this image in one sentence.'
  )
}

import { desktopCapturer } from 'electron'

const sources = await desktopCapturer.getSources({
  types: ['window'],
  thumbnailSize: { width: 1280, height: 720 },
})

const image = RunAnywhere.image.bytes(sources[0].thumbnail.toPNG())
const result = await RunAnywhere.vlm.generate(image, 'What is this window showing?')

for await (const event of RunAnywhere.vlm.generateStream(image, 'Describe this in detail.')) {
  if (event.type === 'textDelta') caption += event.text
}