Vision Language Models

import RunAnywhereLlamaCPP
LlamaCPP.register()

let result = try await RunAnywhere.vlm.generate(
    image: ImageInput.file(path),
    prompt: "What is in this photo?"
)

print(result.text)

func generate(image: ImageInput, prompt: String, options: LlmOptions? = nil) async throws
    -> GenerationResult
func generateStream(image: ImageInput, prompt: String, options: LlmOptions? = nil) async throws
    -> AsyncThrowingStream<GenerationEvent, Error>

ImageInput.file(_ path: String)
ImageInput.bytes(_ data: Data)
ImageInput.rawRgb(_ data: Data, width: Int, height: Int)
ImageInput.cgImage(_ image: CGImage) throws
ImageInput.uiImage(_ image: UIImage) throws        // iOS
ImageInput.nsImage(_ image: NSImage) throws        // macOS
ImageInput.pixelBuffer(_ buffer: CVPixelBuffer) throws

import PhotosUI

func describe(_ item: PhotosPickerItem) async throws -> String {
    guard let data = try await item.loadTransferable(type: Data.self),
          let image = UIImage(data: data) else { return "" }

    let result = try await RunAnywhere.vlm.generate(
        image: try ImageInput.uiImage(image),
        prompt: "Describe this image in one sentence."
    )
    return result.text
}

func captureOutput(_ output: AVCaptureOutput,
                   didOutput sampleBuffer: CMSampleBuffer,
                   from connection: AVCaptureConnection) {
    guard let buffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }

    Task {
        let result = try await RunAnywhere.vlm.generate(
            image: try ImageInput.pixelBuffer(buffer),
            prompt: "What do you see?"
        )
        await MainActor.run { self.caption = result.text }
    }
}

let stream = try await RunAnywhere.vlm.generateStream(
    image: try ImageInput.uiImage(photo),
    prompt: "Describe this in detail."
)

for try await event in stream {
    if case .token(let token) = event { caption += token.text }
}