t-SNE可視化圖像特徵
t-SNE(t-Distributed Stochastic Neighbor Embedding)是一種非線性降維技術,主要用途爲對高維數據進行可視化。
easy sample(使用sklearn包):
from matplotlib.offsetbox import OffsetImage, AnnotationBbox
from sklearn.manifold import TSNE
import cv2
import numpy as np
import matplotlib.pyplot as plt
# 傳入圖像的embedding特徵和對應的圖像路徑
def draw_tsne(features, imgs):
"""
Args:
feature: [n_samples embed_dim], full data embedding of test samples.
imgs: list [n_samples], list of datapaths corresponding to <feature>
"""
#print(imgs)
# 初始化一個TSNE模型,這裏的參數設置可以查看SKlearn的官網
tsne = TSNE(n_components=2, init='pca', perplexity=30)
# Y是降成兩維後的數據
Y = tsne.fit_transform(features)
fig, ax = plt.subplots()
# 設置圖像大小
# fig.set_size_inches(21.6, 14.4)
plt.axis('off')
imscatter(Y[:, 0], Y[:, 1], imgs, zoom=0.1, ax=ax)
plt.savefig(fname='figure.jpg', format='jpg')
plt.show()
def imscatter(x, y, images, zoom, ax=None):
artists = []
for x0, y0, image in zip(x, y, images):
print(image)
im = cv2.imread(image)
im = cv2.resize(im, (224, 224))
im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
im_f = OffsetImage(im, zoom=zoom)
ab = AnnotationBbox(im_f, (x0, y0), xycoords='data', frameon=False)
artists.append(ax.add_artist(ab))
ax.update_datalim(np.column_stack([x, y]))
ax.autoscale()
return artists
x = np.array([[0, 0, 0], [0, 1, 1], [1, 0, 1], [1, 1, 1]])
y = ['D:/000001.jpg',
'D:/000002.jpg',
'D:/000003.jpg',
'D:/000004.jpg']
draw_tsne(x, y)
效果圖:
不過在使用相似代碼對cub200數據集進行處理時,發現圖片相互之間的遮擋情況十分嚴重,例如:
後來發現了一個python包:Embeddings2Image
瞬間被它的描述吸引:create "Karpathy's style" 2d images out of your image embeddings
,但在win10環境下我一直沒有配置成功,只在服務器上配置好了環境。
然鵝,我用主頁上給出的樣例代碼跑出來的圖片是一片黑。。。,最後根據它的源碼仿寫了幾個函數,也發現了一些問題,笑哭.jpg
Scatter圖
# scatter圖 這個的處理是直接把跟之前的圖片有遮擋部分的圖片取消顯示了 (。。。逃)
def scatter(projection_vectors, image_list):
image_num = len(image_list)
output_img_size = 4000 # 輸出圖片的大小
each_img_size = 50
tmp_vectors = projection_vectors * output_img_size
image = np.ones((output_img_size + each_img_size, output_img_size + each_img_size, 3))
for i in tqdm(range(image_num)):
img_path = image_list[i]
x0, y0 = map(int, tmp_vectors[i])
small_img, x1, y1, dx, dy = get_image(img_path, each_img_size)
if small_img is None:
continue
# test if there is an image there already
if np.mean(image[y0 + dy:y0 + dy + y1, x0 + dx:x0 + dx + x1]) != 1:
continue
image[y0 + dy:y0 + dy + y1, x0 + dx:x0 + dx + x1] = small_img
return image
輔助函數:
"""
img_tools.py
"""
import numpy as np
import cv2
from math import ceil
def standard_resize(image, max_side):
if image is None:
return None, None, None
original_h, original_w, _ = image.shape
# if all(side < max_side for side in [original_h, original_w]):
# return image, original_h, original_w
aspect_ratio = float(np.amax((original_w, original_h)) / float(np.amin((original_h, original_w))))
if original_w >= original_h:
new_w = max_side
new_h = max_side / aspect_ratio
else:
new_h = max_side
new_w = max_side / aspect_ratio
new_h = int(new_h)
new_w = int(new_w)
resized_image = cv2.resize(image, (new_w, new_h))
return resized_image, new_w, new_h
def get_image(img_path, image_new_size):
np_img = cv2.imread(img_path)
# np_img = np.array(img)
if np_img is None:
return None, None, None, None, None
small_image, x1, y1 = standard_resize(np_img, image_new_size)
if small_image is None:
return None, None, None, None, None
dx = int(ceil((image_new_size - x1) / 2))
dy = int(ceil((image_new_size - y1) / 2))
return small_image, x1, y1, dx, dy
部分效果:
Grid圖
寫到這裏,其實我最開始的想法是做一個grid
圖,圖與圖之間如果相連則剛好緊緊相連,不相連的話也會空出來相應的空間,類似於這樣:
驚奇地發現也有類似的grid
函數,仿寫的函數如下(這部分我修改的還挺多的 主要是感覺它那個包裏面的代碼不太對勁 。。。 emmm 感覺錯誤蠻多的):
def grid(projection_vectors, image_list):
#print(projection_vectors)
output_img_size = 4000
each_img_size = 50
ratio = int(output_img_size / each_img_size)
tsne_norm = projection_vectors * output_img_size
print(tsne_norm)
used_imgs = np.equal(projection_vectors[:, 0], None)
image = np.ones((output_img_size + each_img_size, output_img_size + each_img_size, 3))
for x in tqdm(range(ratio)):
x0 = x * each_img_size
x05 = (x + 0.5) * each_img_size
y = 0
while y < ratio:
y0 = y * each_img_size
y05 = (y + 0.5) * each_img_size
tmp_tsne = tsne_norm - [x05, y05]
#tmp_tsne[used_imgs] = 99999 # don't use the same img twice
tsne_dist = np.hypot(tmp_tsne[:, 0], tmp_tsne[:, 1])
min_index = np.argmin(tsne_dist)
y += 1
if used_imgs[min_index] == True:
continue
used_imgs[min_index] = True
img_path = image_list[min_index]
small_img, x1, y1, dx, dy = get_image(img_path, each_img_size)
if small_img is None:
continue
image[y0 + dy:y0 + dy + y1, x0 + dx:x0 + dx + x1] = small_img
return image
部分效果:
雖然跟想象中圖片緊密挨着的情形差別還是蠻大的,但也先作爲初步結果吧 hhhhh~
Example
from sklearn.manifold import TSNE
import cv2
import numpy as np
from tqdm import tqdm
from math import ceil
from img_tools import get_image
# 傳入圖像的embedding特徵和對應的圖像路徑
def draw_tsne(features, imgs):
"""
Args:
feature: [n_samples embed_dim], full data embedding of test samples.
imgs: list [n_samples], list of datapaths corresponding to <feature>
"""
# 初始化一個TSNE模型,這裏的參數設置可以查看SKlearn的官網
tsne = TSNE(n_components=2, init='pca', perplexity=30)
Y = tsne.fit_transform(features)
#print(Y)
# 歸一化處理
Y -= Y.min(axis=0)
Y /= Y.max(axis=0)
constructed_image = grid(Y, imgs)
scatter_image = scatter(Y, imgs)
cv2.imwrite('1.jpg', constructed_image)
cv2.imwrite('2.jpg', scatter_image)
# print(imgs)
# scatter圖 這個的處理是直接把跟之前的圖片有遮擋部分的圖片取消顯示了 (。。。逃)
def scatter(projection_vectors, image_list):
image_num = len(image_list)
output_img_size = 2500 # 輸出圖片的大小
each_img_size = 50
tmp_vectors = projection_vectors * output_img_size
image = np.ones((output_img_size + each_img_size, output_img_size + each_img_size, 3))
for i in tqdm(range(image_num)):
img_path = image_list[i]
x0, y0 = map(int, tmp_vectors[i])
small_img, x1, y1, dx, dy = get_image(img_path, each_img_size)
if small_img is None:
continue
# test if there is an image there already
if np.mean(image[y0 + dy:y0 + dy + y1, x0 + dx:x0 + dx + x1]) != 1:
continue
image[y0 + dy:y0 + dy + y1, x0 + dx:x0 + dx + x1] = small_img
return image
def grid(projection_vectors, image_list):
#print(projection_vectors)
output_img_size = 2500
each_img_size = 50
ratio = int(output_img_size / each_img_size)
tsne_norm = projection_vectors * output_img_size
print(tsne_norm)
used_imgs = np.equal(projection_vectors[:, 0], None)
image = np.ones((output_img_size + each_img_size, output_img_size + each_img_size, 3))
for x in tqdm(range(ratio)):
x0 = x * each_img_size
x05 = (x + 0.5) * each_img_size
y = 0
while y < ratio:
y0 = y * each_img_size
y05 = (y + 0.5) * each_img_size
tmp_tsne = tsne_norm - [x05, y05]
#tmp_tsne[used_imgs] = 99999 # don't use the same img twice
tsne_dist = np.hypot(tmp_tsne[:, 0], tmp_tsne[:, 1])
min_index = np.argmin(tsne_dist)
y += 1
if used_imgs[min_index] == True:
continue
used_imgs[min_index] = True
img_path = image_list[min_index]
small_img, x1, y1, dx, dy = get_image(img_path, each_img_size)
if small_img is None:
continue
image[y0 + dy:y0 + dy + y1, x0 + dx:x0 + dx + x1] = small_img
return image