基于CLIP和DINOv2实现图像相似性方面的比较

概述

在人工智能领域，CLIP和DINOv2是计算机视觉领域的两大巨头。CLIP彻底改变了图像理解，而DINOv2为自监督学习带来了新的方法。

在本文中，我们将踏上一段旅程，揭示定义CLIP和DINOv2的优势和微妙之处。我们的目标是发现这些模型中哪一个在图像相似性任务的世界中真正表现出色。让我们见证巨头的碰撞，看看哪个模型会脱颖而出。

1 使用CLIP计算图像相似性

使用CLIP计算两张图像之间的相似性是一个简单的过程，只需两步：首先，提取两张图像的特征，然后计算它们的余弦相似度。

首先，确保安装了必要的软件包。建议设置并使用虚拟环境：

# 首先设置虚拟环境
virtualenv venv-similarity
source venv-similarity/bin/activate
# 安装所需软件包
pip install transformers Pillow torch

接下来，计算图像相似性：

import torch
from PIL import Image
from transformers import AutoProcessor, CLIPModel
import torch.nn as nn

device = torch.device('cuda' if torch.cuda.is_available() else "cpu")
processor = AutoProcessor.from_pretrained("openai/clip-vit-base-patch32")
model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32").to(device)

# 从image1中提取特征
image1 = Image.open('img1.jpg')
with torch.no_grad():
    inputs1 = processor(images=image1, return_tensors="pt").to(device)
    image_features1 = model.get_image_features(**inputs1)

# 从image2中提取特征
image2 = Image.open('img2.jpg')
with torch.no_grad():
    inputs2 = processor(images=image2, return_tensors="pt").to(device)
    image_features2 = model.get_image_features(**inputs2)

# 计算它们的余弦相似度并将其转换为0到1之间的分数
cos = nn.CosineSimilarity(dim=0)
sim = cos(image_features1[0],image_features2[0]).item()
sim = (sim + 1) / 2
print('Similarity:', sim)

使用提供的两张相似图像的示例，获得的相似度分数令人印象深刻，达到了96.4%。

2 使用DINOv2计算图像相似性

使用DINOv2计算两张图像之间的相似性过程与CLIP类似。用DINOv2需要与前面提到的相同的软件包集，无需额外安装：

from transformers import AutoImageProcessor, AutoModel
from PIL import Image
import torch.nn as nn

device = torch.device('cuda' if torch.cuda.is_available() else "cpu")
processor = AutoImageProcessor.from_pretrained('facebook/dinov2-base')
model = AutoModel.from_pretrained('facebook/dinov2-base').to(device)

image1 = Image.open('img1.jpg')
with torch.no_grad():
    inputs1 = processor(images=image1, return_tensors="pt").to(device)
    outputs1 = model(**inputs1)
    image_features1 = outputs1.last_hidden_state
    image_features1 = image_features1.mean(dim=1)

image2 = Image.open('img2.jpg')
with torch.no_grad():
    inputs2 = processor(images=image2, return_tensors="pt").to(device)
    outputs2 = model(**inputs2)
    image_features2 = outputs2.last_hidden_state
    image_features2 = image_features2.mean(dim=1)

cos = nn.CosineSimilarity(dim=0)
sim = cos(image_features1[0],image_features2[0]).item()
sim = (sim + 1) / 2
print('Similarity:', sim)

使用与CLIP示例中相同的一对图像，DINOv2获得的相似度分数为96.4%:
在这里插入图片描述

3. 使用COCO数据集进行测试

在深入评估它们的性能之前，使用COCO数据集中的图像比较CLIP和DINOv2产生的结果。

实现流程：

遍历数据集以提取所有图像的特征。
将嵌入存储在FAISS索引中。
提取输入图像的特征。
检索最相似的三张图像。

3.1 特征提取和创建

import torch
from PIL import Image
from transformers import AutoProcessor, CLIPModel, AutoImageProcessor, AutoModel
import faiss
import os
import numpy as np

device = torch.device('cuda' if torch.cuda.is_available() else "cpu")

# 加载CLIP模型和处理器
processor_clip = AutoProcessor.from_pretrained("openai/clip-vit-base-patch32")
model_clip = CLIPModel.from_pretrained("openai/clip-vit-base-patch32").to(device)

# 加载DINOv2模型和处理器
processor_dino = AutoImageProcessor.from_pretrained('facebook/dinov2-base')
model_dino = AutoModel.from_pretrained('facebook/dinov2-base').to(device)

# 获取所有文件名
images = []
for root, dirs, files in os.walk('./val2017/'):
    for file in files:
        if file.endswith('jpg'):
            images.append(root + '/' + file)

# 定义一个函数，用于归一化嵌入并将其添加到索引中
def add_vector_to_index(embedding, index):
    # 将嵌入转换为numpy数组
    vector = embedding.detach().cpu().numpy()
    # 转换为float32 numpy数组
    vector = np.float32(vector)
    # 归一化向量：在搜索时避免错误结果很重要
    faiss.normalize_L2(vector)
    # 添加到索引中
    index.add(vector)

def extract_features_clip(image):
    with torch.no_grad():
        inputs = processor_clip(images=image, return_tensors="pt").to(device)
        image_features = model_clip.get_image_features(**inputs)
        return image_features

def extract_features_dino(image):
    with torch.no_grad():
        inputs = processor_dino(images=image, return_tensors="pt").to(device)
        outputs = model_dino(**inputs)
        image_features = outputs.last_hidden_state
        return image_features.mean(dim=1)

# 创建两个索引
index_clip = faiss.IndexFlatL2(512)
index_dino = faiss.IndexFlatL2(768)

# 遍历数据集以提取特征X2并将特征存储在索引中
for image_path in images:
    img = Image.open(image_path).convert('RGB')
    clip_features = extract_features_clip(img)
    add_vector_to_index(clip_features, index_clip)
    dino_features = extract_features_dino(img)
    add_vector_to_index(dino_features, index_dino)

# 将索引本地存储
faiss.write_index(index_clip, "clip.index")
faiss.write_index(index_dino, "dino.index")

3.2 图像相似性搜索

import faiss
import numpy as np
import torch
from transformers import AutoImageProcessor, AutoModel, AutoProcessor, CLIPModel
from PIL import Image
import os

# 输入图像
source = 'laptop.jpg'
image = Image.open(source)
device = torch.device('cuda' if torch.cuda.is_available() else "cpu")

# 加载DINOv2和CLIP的模型和处理器
processor_clip = AutoProcessor.from_pretrained("openai/clip-vit-base-patch32")
model_clip = CLIPModel.from_pretrained("openai/clip-vit-base-patch32").to(device)
processor_dino = AutoImageProcessor.from_pretrained('facebook/dinov2-base')
model_dino = AutoModel.from_pretrained('facebook/dinov2-base').to(device)

# 为CLIP提取特征
with torch.no_grad():
    inputs_clip = processor_clip(images=image, return_tensors="pt").to(device)
    image_features_clip = model_clip.get_image_features(**inputs_clip)

# 为DINOv2提取特征
with torch.no_grad():
    inputs_dino = processor_dino(images=image, return_tensors="pt").to(device)
    outputs_dino = model_dino(**inputs_dino)
    image_features_dino = outputs_dino.last_hidden_state
    image_features_dino = image_features_dino.mean(dim=1)

def normalizeL2(embeddings):
    vector = embeddings.detach().cpu().numpy()
    vector = np.float32(vector)
    faiss.normalize_L2(vector)
    return vector

image_features_dino = normalizeL2(image_features_dino)
image_features_clip = normalizeL2(image_features_clip)

# 搜索最相似的5张图像
index_clip = faiss.read_index("clip.index")
index_dino = faiss.read_index("dino.index")

# 获取图像的距离和相关索引
d_dino, i_dino = index_dino.search(image_features_dino, 5)
d_clip, i_clip = index_clip.search(image_features_clip, 5)