Multimodal AI Complete Guide - Vision, Audio, Video LLMs

from openai import OpenAI
 
client = OpenAI()
 
# Basic transcription
def transcribe_audio(file_path: str) -> str:
    with open(file_path, "rb") as audio_file:
        transcript = client.audio.transcriptions.create(
            model="whisper-1",
            file=audio_file,
            response_format="text"
        )
    return transcript
 
# Transcription with timestamps
def transcribe_with_timestamps(file_path: str) -> dict:
    with open(file_path, "rb") as audio_file:
        transcript = client.audio.transcriptions.create(
            model="whisper-1",
            file=audio_file,
            response_format="verbose_json",
            timestamp_granularities=["word", "segment"]
        )
    return transcript
 
# Translation (any language to English)
def translate_audio(file_path: str) -> str:
    with open(file_path, "rb") as audio_file:
        translation = client.audio.translations.create(
            model="whisper-1",
            file=audio_file,
            response_format="text"
        )
    return translation

from faster_whisper import WhisperModel
 
model = WhisperModel("large-v3", device="cuda", compute_type="float16")
 
segments, info = model.transcribe(
    "audio.wav",
    beam_size=5,
    vad_filter=True,
    vad_parameters=dict(min_silence_duration_ms=500)
)
 
print(f"Detected language: {info.language} (prob: {info.language_probability:.2f})")
for segment in segments:
    print(f"[{segment.start:.2f}s -> {segment.end:.2f}s] {segment.text}")

from openai import OpenAI
from pathlib import Path
 
client = OpenAI()
 
def text_to_speech(text: str, output_path: str, voice: str = "alloy"):
    """Available voices: alloy, echo, fable, onyx, nova, shimmer"""
    response = client.audio.speech.create(
        model="tts-1-hd",
        voice=voice,
        input=text,
        response_format="mp3"
    )
    response.stream_to_file(Path(output_path))

def analyze_meeting_video(video_path: str) -> dict:
    """Analyze a meeting recording for key insights."""
    audio = extract_audio(video_path)
    transcript = transcribe_with_timestamps(audio)
    frames = extract_keyframes(video_path, method="scene_change")
    visual_content = [analyze_image(frame) for frame in frames]
    
    return {
        "summary": generate_meeting_summary(transcript, visual_content),
        "action_items": extract_action_items(transcript),
        "key_decisions": extract_decisions(transcript),
        "slides_content": visual_content
    }

import base64
from openai import OpenAI
 
client = OpenAI()
 
def encode_image_to_base64(image_path: str) -> str:
    with open(image_path, "rb") as f:
        return base64.b64encode(f.read()).decode("utf-8")
 
def analyze_image(image_path: str, question: str = "Describe this image in detail.") -> str:
    base64_image = encode_image_to_base64(image_path)
    response = client.chat.completions.create(
        model="gpt-4o",
        messages=[{
            "role": "user",
            "content": [
                {"type": "text", "text": question},
                {
                    "type": "image_url",
                    "image_url": {
                        "url": f"data:image/png;base64,{base64_image}",
                        "detail": "high"
                    }
                }
            ]
        }],
        max_tokens=1024
    )
    return response.choices[0].message.content
 
def compare_images(image_paths: list[str], question: str) -> str:
    content = [{"type": "text", "text": question}]
    for path in image_paths:
        content.append({
            "type": "image_url",
            "image_url": {
                "url": f"data:image/png;base64,{encode_image_to_base64(path)}",
                "detail": "high"
            }
        })
    response = client.chat.completions.create(
        model="gpt-4o",
        messages=[{"role": "user", "content": content}],
        max_tokens=1024
    )
    return response.choices[0].message.content

import anthropic
import base64
 
client = anthropic.Anthropic()
 
def analyze_image_claude(image_path: str, question: str) -> str:
    with open(image_path, "rb") as f:
        image_data = base64.standard_b64encode(f.read()).decode("utf-8")
    
    ext = image_path.rsplit(".", 1)[-1].lower()
    media_types = {"jpg": "image/jpeg", "jpeg": "image/jpeg",
                   "png": "image/png", "gif": "image/gif", "webp": "image/webp"}
    
    message = client.messages.create(
        model="claude-sonnet-4-20250514",
        max_tokens=1024,
        messages=[{
            "role": "user",
            "content": [
                {"type": "image", "source": {
                    "type": "base64",
                    "media_type": media_types.get(ext, "image/png"),
                    "data": image_data
                }},
                {"type": "text", "text": question}
            ]
        }]
    )
    return message.content[0].text
 
def extract_document_text(document_image_path: str) -> str:
    """Extract structured text from a document image."""
    return analyze_image_claude(
        document_image_path,
        "Extract all text from this document. Preserve headers, tables (as markdown), "
        "and lists. Return in clean markdown format."
    )
 
def interpret_chart(chart_image_path: str) -> str:
    """Interpret a chart or graph from an image."""
    return analyze_image_claude(
        chart_image_path,
        "Analyze this chart: 1) Chart type 2) Title/axis labels 3) Key data points "
        "4) Trends observed 5) Notable outliers. Be precise with values."
    )

import requests
import base64
 
OLLAMA_BASE_URL = "http://localhost:11434"
 
def analyze_image_ollama(image_path: str, prompt: str, model: str = "llava:13b") -> str:
    with open(image_path, "rb") as f:
        image_base64 = base64.b64encode(f.read()).decode("utf-8")
    
    response = requests.post(
        f"{OLLAMA_BASE_URL}/api/generate",
        json={
            "model": model,
            "prompt": prompt,
            "images": [image_base64],
            "stream": False,
            "options": {"temperature": 0.1, "num_predict": 1024}
        }
    )
    response.raise_for_status()
    return response.json()["response"]
 
def batch_analyze_images(image_dir: str, prompt: str, model: str = "llava:13b") -> list:
    import os
    results = []
    for filename in sorted(os.listdir(image_dir)):
        if os.path.splitext(filename)[1].lower() in {".jpg", ".jpeg", ".png", ".webp"}:
            filepath = os.path.join(image_dir, filename)
            results.append({"filename": filename,
                            "analysis": analyze_image_ollama(filepath, prompt, model)})
    return results
 
# Available Ollama vision models
# llava:7b   - Fast, basic vision (8GB VRAM)
# llava:13b  - Better quality (16GB VRAM)
# llava:34b  - High quality (24GB+ VRAM)
# moondream  - Tiny (1.8B), very fast

import chromadb
from openai import OpenAI
 
client = OpenAI()
 
def generate_image_caption(image_path: str) -> str:
    import base64
    with open(image_path, "rb") as f:
        img_b64 = base64.b64encode(f.read()).decode()
    
    response = client.chat.completions.create(
        model="gpt-4o",
        messages=[{"role": "user", "content": [
            {"type": "text", "text": "Describe this image in detail for search indexing."},
            {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{img_b64}"}}
        ]}],
        max_tokens=500
    )
    return response.choices[0].message.content
 
def index_image(collection, image_path: str, doc_id: str, metadata: dict):
    caption = generate_image_caption(image_path)
    embedding = client.embeddings.create(
        model="text-embedding-3-small", input=caption
    ).data[0].embedding
    collection.add(ids=[doc_id], embeddings=[embedding],
                   metadatas=[{**metadata, "caption": caption}], documents=[caption])

from sentence_transformers import SentenceTransformer
from PIL import Image
import numpy as np
 
clip_model = SentenceTransformer("clip-ViT-L-14")
 
def embed_image_clip(image_path: str) -> np.ndarray:
    return clip_model.encode(Image.open(image_path))
 
def embed_text_clip(text: str) -> np.ndarray:
    return clip_model.encode(text)
 
def search_images(query: str, image_embeddings: dict, top_k: int = 5):
    query_emb = embed_text_clip(query)
    similarities = {}
    for img_id, img_emb in image_embeddings.items():
        sim = np.dot(query_emb, img_emb) / (np.linalg.norm(query_emb) * np.linalg.norm(img_emb))
        similarities[img_id] = float(sim)
    return sorted(similarities.items(), key=lambda x: x[1], reverse=True)[:top_k]

import fitz  # PyMuPDF
from pathlib import Path
 
def process_pdf_multimodal(pdf_path: str, output_dir: str) -> dict:
    doc = fitz.open(pdf_path)
    output = Path(output_dir)
    output.mkdir(parents=True, exist_ok=True)
    results = {"text_chunks": [], "images": [], "pages": []}
    
    for page_num, page in enumerate(doc):
        text = page.get_text("text")
        if text.strip():
            results["text_chunks"].append({"page": page_num + 1, "content": text.strip()})
        
        for img_idx, img in enumerate(page.get_images(full=True)):
            pix = fitz.Pixmap(doc, img[0])
            if pix.n - pix.alpha > 3:
                pix = fitz.Pixmap(fitz.csRGB, pix)
            img_path = str(output / f"page{page_num+1}_img{img_idx+1}.png")
            pix.save(img_path)
            results["images"].append({"page": page_num + 1, "path": img_path})
        
        page_pix = page.get_pixmap(dpi=200)
        page_img_path = str(output / f"page_{page_num+1}.png")
        page_pix.save(page_img_path)
        results["pages"].append(page_img_path)
    
    return results

# Conceptual ColPali pipeline
from colpali_engine.models import ColPali, ColPaliProcessor
 
model = ColPali.from_pretrained("vidore/colpali-v1.2")
processor = ColPaliProcessor.from_pretrained("vidore/colpali-v1.2")
 
def index_document_pages(pdf_path: str):
    doc = fitz.open(pdf_path)
    page_embeddings = []
    for page in doc:
        pix = page.get_pixmap(dpi=144)
        img = Image.frombytes("RGB", [pix.width, pix.height], pix.samples)
        inputs = processor(images=[img])
        with torch.no_grad():
            embeddings = model(**inputs)  # (1, num_patches, embed_dim)
        page_embeddings.append({"page_num": page.number + 1, "embeddings": embeddings})
    return page_embeddings
 
def query_documents(query: str, page_embeddings: list):
    query_inputs = processor(text=[query])
    with torch.no_grad():
        query_emb = model(**query_inputs)
    scores = []
    for page in page_embeddings:
        sim = torch.matmul(query_emb, page["embeddings"].transpose(-1, -2))
        scores.append((page["page_num"], sim.max(dim=-1).values.sum().item()))
    return sorted(scores, key=lambda x: x[1], reverse=True)

def process_invoice(image_path: str) -> dict:
    import json
    result = analyze_image_claude(image_path, """Extract from this invoice as JSON:
    {"vendor_name": "", "invoice_number": "", "invoice_date": "", "due_date": "",
     "line_items": [{"description": "", "quantity": 0, "unit_price": 0.0, "total": 0.0}],
     "subtotal": 0.0, "tax_amount": 0.0, "total_amount": 0.0}
    Return ONLY the JSON object.""")
    return json.loads(result)