CLIP Image-Text Similarity Search

from transformers import CLIPProcessor, CLIPModel
from PIL import Image
import torch
import requests
from io import BytesIO

model     = CLIPModel.from_pretrained('openai/clip-vit-base-patch32')
processor = CLIPProcessor.from_pretrained('openai/clip-vit-base-patch32')

# Dummy images (replace with real images)
images = [Image.new('RGB', (224,224), color=c) for c in [(255,0,0),(0,255,0),(0,0,255)]]
image_labels = ['red image', 'green image', 'blue image']
texts  = ['a red coloured picture', 'a green coloured picture', 'something blue']

inputs = processor(text=texts, images=images, return_tensors='pt', padding=True)
with torch.no_grad():
    outputs     = model(**inputs)
    logits      = outputs.logits_per_image  # shape: [n_images, n_texts]
    similarities = logits.softmax(dim=1)

for i, label in enumerate(image_labels):
    best_text = texts[similarities[i].argmax()]
    print(f'{label} -> best match: {best_text!r} ({similarities[i].max():.2%})')