TensorFlow實戰（5）：Word2Vec

論文地址：Efficient Estimation of Word Representations in Vector Space
時間：2013.1

word2Vec簡介

在NLP領域中，爲了能表示人類的語言符號，一般會把這些符號轉成一種數學向量形式以方便處理，我們把語言單詞嵌入到向量空間中就叫詞嵌入（word embedding）。以往對文本的處理採用One-Hot Encoder，對特徵編碼往往是隨機的，不考慮詞之間存在的關係。使用向量表達則可以有效地解決這個問題，向量空間模型將字詞轉爲連續值的向量表達，並將其中意思相近的詞映射到向量空間相近的位置。
向量空間模型可以分爲兩類，一類是計數模型，統計在語料庫中，相鄰出現的詞的頻率，如Latent Semantic Analysis，一類是預測模型，根據一個詞周圍相鄰的詞推測出這個詞，如Neural Probabilistic Models。
Word2Vec爲預測模型，主要分爲CBOW和Skip-Gram模式，CBOW是從原始語句中推測目標字詞，Skip-Gram從目標字詞中推測出原始語句。

Skip-Gram原理

書中用的是“the quick brown fox jumped over the lazy dog”爲例，這裏我也用這句話進行講解。最終我們訓練要達到的效果是可以從給出的詞預測其他詞。比如說給出quick可以預測到the。
參數skip_window代表對每個單詞，前面skip_window個單詞和後面skip_window個單詞要能預測到該單詞，所以窗口span的大小爲2*skip_window+1。以skip_window=1爲例，對quick來講，quick要可以推測到the和brown（由目標詞語推測語境）

TensorFlow實現（Skip_Gram模式）

代碼詳解：

import tensorflow as tf
import zipfile
import collections
import numpy as np
import random
import math
import matplotlib.pyplot as plt
from sklearn.manifold import TSNE

filename = './text8.zip' #文件地址
def read_data(filename):
    with zipfile.ZipFile(filename) as f:
        data = tf.compat.as_str(f.read(f.namelist()[0])).split()
    return data

words = read_data(filename)
print(len(words))

vocabulary_size = 50000 #取頻數前50000

#創建vocabulary詞彙表
def build_dataset(words):
    count = [['UNK', -1]] #記錄除詞頻前50000的其他詞（記爲unknown）的個數
    # count爲一個二元數組，count[i][0]爲單詞，count[i][1]爲該單詞的個數
    count.extend(collections.Counter(words).most_common(vocabulary_size-1)) #取top50000的單詞作爲vocabulary
    dictionary = dict()
    #將count[i][0]作爲keys，count[i][1]作爲values創建dictionary
    for word, _ in count:
        dictionary[word] = len(dictionary)
    data = list() #該數組存放的是words中每個單詞的映射
    unk_count = 0 #記錄unknown單詞出現的次數
    for word in words:
        #遍歷words，取當前單詞的索引位置，所有的unknown單詞索引均爲0
        if word in dictionary:
            index = dictionary[word]
        else:
            #如果dictionary裏面沒有，即爲unknown word
            index = 0
            unk_count += 1
        data.append(index)
    count[0][1] = unk_count
    reverse_dictionary = dict(zip(dictionary.values(), dictionary.keys())) #將dictionary的鍵值索引互換
    return data, count, dictionary, reverse_dictionary

data, count, dictionaru, reverse_dictionary = build_dataset(words)

del words #刪除原始詞彙表
print("Most common words (+UNK)", count[:5])
print('Sample data', data[:10], [reverse_dictionary[i] for i in data[:10]])

data_index = 0 #全局變量

#生成一個batch的訓練樣本
#batch_size爲batch的大小，num_skips表示對每個單詞生成多少個樣本，skip_window指單詞最遠可聯繫的距離
def generate_batch(batch_size, num_skips, skip_window):
    global data_index
    assert batch_size % num_skips == 0 #batch_size爲num_skips的整數倍
    #將batch和labels初始化爲數組
    assert num_skips <= 2 * skip_window #num_skips不能超過skip_window的兩倍
    batch = np.ndarray(shape=(batch_size), dtype=np.int32)
    labels = np.ndarray(shape=(batch_size,1), dtype = np.int32)
    span = 2*skip_window + 1 #爲每個單詞創建樣本時會使用到的單詞數量
    buffer = collections.deque(maxlen=span) #容量爲span的雙向隊列deque
    #將span個單詞讀入buffer作爲初始值
    for _ in range(span):
        buffer.append(data[data_index])
        data_index = (data_index + 1) % len(data) #索引加一，防止超過容量，循環加一
    #batch_size // num_skips爲每個batch包含單詞個數
    for i in range(batch_size // num_skips):
        target = skip_window
        targets_to_avoid = [skip_window] #存放已經生成的樣本
        for j in range(num_skips):
            #判斷該樣本是否已經生成過，若生成過，則重新在0-span-1中選一個
            while target in targets_to_avoid:
                target = random.randint(0, span-1)
            targets_to_avoid.append(target)
            batch[i*num_skips+j] = buffer[skip_window]
            labels[i*num_skips+j, 0] = buffer[target]
        buffer.append(data[data_index]) #添加data[data_index]，同時buffer的第一個單詞會被擠掉
        data_index = (data_index + 1) % len(data)#索引加一，防止超過容量，循環加一
    return batch, labels

#檢驗generate_batch函數
'''
batch, labels = generate_batch(batch_size=8, num_skips=2, skip_window=1)
for i in range(8):
    print(batch[i], reverse_dictionary[batch[i]], '->', labels[i,0], reverse_dictionary[labels[i,0]])
'''

batch_size = 128
embedding_size = 128 #單詞轉爲稠密向量的維度
skip_window = 1
num_skips = 2

valid_size = 16 #驗證集單詞個數
valid_window = 100 #驗證集單詞的最高頻數
valid_examples = np.random.choice(valid_window, valid_size, replace = False) #選擇驗證集
num_sampled = 64 #訓練時用來做負樣本的噪聲單詞的數量

#定義網絡結構
graph = tf.Graph() #創建graph
with graph.as_default():
    train_inputs = tf.placeholder(tf.int32, shape=[batch_size])
    train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1])
    valid_dataset = tf.constant(valid_examples, dtype=tf.int32)

    with tf.device('/cpu:0'): #限定在CPU上進行
        embeddings = tf.Variable(tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0))
        embed = tf.nn.embedding_lookup(embeddings, train_inputs)

        nce_weights = tf.Variable(tf.truncated_normal([vocabulary_size, embedding_size],
                                                     stddev = 1.0/ math.sqrt(embedding_size))
                                  )
        nce_biases = tf.Variable(tf.zeros([vocabulary_size]))

    #計算Noise Contrastive Estimation Loss並求平均值
    loss = tf.reduce_mean(tf.nn.nce_loss(weights = nce_weights,
                                         biases = nce_biases,
                                         labels = train_labels,
                                         inputs = embed,
                                         num_sampled = num_sampled,
                                         num_classes = vocabulary_size))

    optimizer = tf.train.GradientDescentOptimizer(1.0).minimize(loss)

    norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True))
    normalized_embeddings = embeddings / norm
    valid_embeddings = tf.nn.embedding_lookup(normalized_embeddings, valid_dataset)
    similarity = tf.matmul(valid_embeddings, normalized_embeddings, transpose_b=True)

    init = tf.global_variables_initializer()

num_steps = 100001

with tf.Session(graph=graph) as session:
    init.run()
    print("Initialized")

    average_loss = 0
    for step in range(num_steps):
        batch_inputs, batch_labels = generate_batch(batch_size, num_skips, skip_window)
        feed_dict = {train_inputs : batch_inputs, train_labels : batch_labels}

        _, loss_val = session.run([optimizer, loss], feed_dict=feed_dict)
        average_loss += loss_val
        # 每循環2000次，計算loss
        if step % 2000 == 0:
            if step > 0:
                average_loss /= 2000
            print("Average loss at step ", step, " :", average_loss)
            average_loss = 0
        #每循環10000次，計算驗證單詞與全部單詞的相似度，並將每個驗證單詞最相似的8個單詞展示出來
        if step % 10000 == 0:
            sim = similarity.eval()
            for i in range(valid_size):
                valid_word = reverse_dictionary[valid_examples[i]] #查詢驗證單詞對應的index
                top_k = 8 #最高頻數爲8
                nearset = (-sim[i, :]).argsort()[1:top_k+1]
                log_str = "Nearest to %s:" % (valid_word)
                for k in range(top_k):
                    close_word = reverse_dictionary[nearset[k]]
                    log_str = "%s %s, " % (log_str, close_word)
                print(log_str)
    final_embeddings = normalized_embeddings.eval()

#進行可視化處理
def plot_with_labels(low_dim_embs, labels, filename = 'tsne.png'):
    assert low_dim_embs.shape[0] >= len(labels), "More labels than embeddings"
    plt.figure(figsize=(18,18))
    for i, label in enumerate(labels):
        x, y = low_dim_embs[i, :]
        plt.scatter(x, y)
        plt.annotate(labels, xy = (x,y), xytext = (5,2), textcoords = 'offset points', ha = 'right', va = 'bottom')
    plt.show()
    plt.savefig(filename)

tsne = TSNE(perplexity=30, n_components=2, init = 'pca', n_iter=5000) #將原始的額128維嵌入向量降到2維
plot_only = 100 #只顯示頻數最高的100個單詞
low_dim_embs = tsne.fit_transform(final_embeddings[:plot_only, :])
labels = [reverse_dictionary[i] for i in range(plot_only)]
plot_with_labels(low_dim_embs, labels)

運行結果：

參考資料

Skip-Gram模型
Skip-Gram訓練篇