TensorFlow实战（5）：Word2Vec

论文地址：Efficient Estimation of Word Representations in Vector Space
时间：2013.1

word2Vec简介

在NLP领域中，为了能表示人类的语言符号，一般会把这些符号转成一种数学向量形式以方便处理，我们把语言单词嵌入到向量空间中就叫词嵌入（word embedding）。以往对文本的处理采用One-Hot Encoder，对特征编码往往是随机的，不考虑词之间存在的关系。使用向量表达则可以有效地解决这个问题，向量空间模型将字词转为连续值的向量表达，并将其中意思相近的词映射到向量空间相近的位置。
向量空间模型可以分为两类，一类是计数模型，统计在语料库中，相邻出现的词的频率，如Latent Semantic Analysis，一类是预测模型，根据一个词周围相邻的词推测出这个词，如Neural Probabilistic Models。
Word2Vec为预测模型，主要分为CBOW和Skip-Gram模式，CBOW是从原始语句中推测目标字词，Skip-Gram从目标字词中推测出原始语句。

Skip-Gram原理

书中用的是“the quick brown fox jumped over the lazy dog”为例，这里我也用这句话进行讲解。最终我们训练要达到的效果是可以从给出的词预测其他词。比如说给出quick可以预测到the。
参数skip_window代表对每个单词，前面skip_window个单词和后面skip_window个单词要能预测到该单词，所以窗口span的大小为2*skip_window+1。以skip_window=1为例，对quick来讲，quick要可以推测到the和brown（由目标词语推测语境）

TensorFlow实现（Skip_Gram模式）

代码详解：

import tensorflow as tf
import zipfile
import collections
import numpy as np
import random
import math
import matplotlib.pyplot as plt
from sklearn.manifold import TSNE

filename = './text8.zip' #文件地址
def read_data(filename):
    with zipfile.ZipFile(filename) as f:
        data = tf.compat.as_str(f.read(f.namelist()[0])).split()
    return data

words = read_data(filename)
print(len(words))

vocabulary_size = 50000 #取频数前50000

#创建vocabulary词汇表
def build_dataset(words):
    count = [['UNK', -1]] #记录除词频前50000的其他词（记为unknown）的个数
    # count为一个二元数组，count[i][0]为单词，count[i][1]为该单词的个数
    count.extend(collections.Counter(words).most_common(vocabulary_size-1)) #取top50000的单词作为vocabulary
    dictionary = dict()
    #将count[i][0]作为keys，count[i][1]作为values创建dictionary
    for word, _ in count:
        dictionary[word] = len(dictionary)
    data = list() #该数组存放的是words中每个单词的映射
    unk_count = 0 #记录unknown单词出现的次数
    for word in words:
        #遍历words，取当前单词的索引位置，所有的unknown单词索引均为0
        if word in dictionary:
            index = dictionary[word]
        else:
            #如果dictionary里面没有，即为unknown word
            index = 0
            unk_count += 1
        data.append(index)
    count[0][1] = unk_count
    reverse_dictionary = dict(zip(dictionary.values(), dictionary.keys())) #将dictionary的键值索引互换
    return data, count, dictionary, reverse_dictionary

data, count, dictionaru, reverse_dictionary = build_dataset(words)

del words #删除原始词汇表
print("Most common words (+UNK)", count[:5])
print('Sample data', data[:10], [reverse_dictionary[i] for i in data[:10]])

data_index = 0 #全局变量

#生成一个batch的训练样本
#batch_size为batch的大小，num_skips表示对每个单词生成多少个样本，skip_window指单词最远可联系的距离
def generate_batch(batch_size, num_skips, skip_window):
    global data_index
    assert batch_size % num_skips == 0 #batch_size为num_skips的整数倍
    #将batch和labels初始化为数组
    assert num_skips <= 2 * skip_window #num_skips不能超过skip_window的两倍
    batch = np.ndarray(shape=(batch_size), dtype=np.int32)
    labels = np.ndarray(shape=(batch_size,1), dtype = np.int32)
    span = 2*skip_window + 1 #为每个单词创建样本时会使用到的单词数量
    buffer = collections.deque(maxlen=span) #容量为span的双向队列deque
    #将span个单词读入buffer作为初始值
    for _ in range(span):
        buffer.append(data[data_index])
        data_index = (data_index + 1) % len(data) #索引加一，防止超过容量，循环加一
    #batch_size // num_skips为每个batch包含单词个数
    for i in range(batch_size // num_skips):
        target = skip_window
        targets_to_avoid = [skip_window] #存放已经生成的样本
        for j in range(num_skips):
            #判断该样本是否已经生成过，若生成过，则重新在0-span-1中选一个
            while target in targets_to_avoid:
                target = random.randint(0, span-1)
            targets_to_avoid.append(target)
            batch[i*num_skips+j] = buffer[skip_window]
            labels[i*num_skips+j, 0] = buffer[target]
        buffer.append(data[data_index]) #添加data[data_index]，同时buffer的第一个单词会被挤掉
        data_index = (data_index + 1) % len(data)#索引加一，防止超过容量，循环加一
    return batch, labels

#检验generate_batch函数
'''
batch, labels = generate_batch(batch_size=8, num_skips=2, skip_window=1)
for i in range(8):
    print(batch[i], reverse_dictionary[batch[i]], '->', labels[i,0], reverse_dictionary[labels[i,0]])
'''

batch_size = 128
embedding_size = 128 #单词转为稠密向量的维度
skip_window = 1
num_skips = 2

valid_size = 16 #验证集单词个数
valid_window = 100 #验证集单词的最高频数
valid_examples = np.random.choice(valid_window, valid_size, replace = False) #选择验证集
num_sampled = 64 #训练时用来做负样本的噪声单词的数量

#定义网络结构
graph = tf.Graph() #创建graph
with graph.as_default():
    train_inputs = tf.placeholder(tf.int32, shape=[batch_size])
    train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1])
    valid_dataset = tf.constant(valid_examples, dtype=tf.int32)

    with tf.device('/cpu:0'): #限定在CPU上进行
        embeddings = tf.Variable(tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0))
        embed = tf.nn.embedding_lookup(embeddings, train_inputs)

        nce_weights = tf.Variable(tf.truncated_normal([vocabulary_size, embedding_size],
                                                     stddev = 1.0/ math.sqrt(embedding_size))
                                  )
        nce_biases = tf.Variable(tf.zeros([vocabulary_size]))

    #计算Noise Contrastive Estimation Loss并求平均值
    loss = tf.reduce_mean(tf.nn.nce_loss(weights = nce_weights,
                                         biases = nce_biases,
                                         labels = train_labels,
                                         inputs = embed,
                                         num_sampled = num_sampled,
                                         num_classes = vocabulary_size))

    optimizer = tf.train.GradientDescentOptimizer(1.0).minimize(loss)

    norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True))
    normalized_embeddings = embeddings / norm
    valid_embeddings = tf.nn.embedding_lookup(normalized_embeddings, valid_dataset)
    similarity = tf.matmul(valid_embeddings, normalized_embeddings, transpose_b=True)

    init = tf.global_variables_initializer()

num_steps = 100001

with tf.Session(graph=graph) as session:
    init.run()
    print("Initialized")

    average_loss = 0
    for step in range(num_steps):
        batch_inputs, batch_labels = generate_batch(batch_size, num_skips, skip_window)
        feed_dict = {train_inputs : batch_inputs, train_labels : batch_labels}

        _, loss_val = session.run([optimizer, loss], feed_dict=feed_dict)
        average_loss += loss_val
        # 每循环2000次，计算loss
        if step % 2000 == 0:
            if step > 0:
                average_loss /= 2000
            print("Average loss at step ", step, " :", average_loss)
            average_loss = 0
        #每循环10000次，计算验证单词与全部单词的相似度，并将每个验证单词最相似的8个单词展示出来
        if step % 10000 == 0:
            sim = similarity.eval()
            for i in range(valid_size):
                valid_word = reverse_dictionary[valid_examples[i]] #查询验证单词对应的index
                top_k = 8 #最高频数为8
                nearset = (-sim[i, :]).argsort()[1:top_k+1]
                log_str = "Nearest to %s:" % (valid_word)
                for k in range(top_k):
                    close_word = reverse_dictionary[nearset[k]]
                    log_str = "%s %s, " % (log_str, close_word)
                print(log_str)
    final_embeddings = normalized_embeddings.eval()

#进行可视化处理
def plot_with_labels(low_dim_embs, labels, filename = 'tsne.png'):
    assert low_dim_embs.shape[0] >= len(labels), "More labels than embeddings"
    plt.figure(figsize=(18,18))
    for i, label in enumerate(labels):
        x, y = low_dim_embs[i, :]
        plt.scatter(x, y)
        plt.annotate(labels, xy = (x,y), xytext = (5,2), textcoords = 'offset points', ha = 'right', va = 'bottom')
    plt.show()
    plt.savefig(filename)

tsne = TSNE(perplexity=30, n_components=2, init = 'pca', n_iter=5000) #将原始的额128维嵌入向量降到2维
plot_only = 100 #只显示频数最高的100个单词
low_dim_embs = tsne.fit_transform(final_embeddings[:plot_only, :])
labels = [reverse_dictionary[i] for i in range(plot_only)]
plot_with_labels(low_dim_embs, labels)

运行结果：

参考资料

Skip-Gram模型
Skip-Gram训练篇