Python深度學習（5）：詞嵌入（GloVe模型）

詞嵌入簡介

深度學習模型只能處理數值型張量，因此需要將文本轉換爲數值張量，即文本向量化。將文本分解成標記token（單詞、字符或n-gram），將標記與向量關聯的方法常用的one-hot編碼和詞嵌入word embedding。
詞嵌入的作用是將人類語言映射到幾何空間，利用詞向量之間的幾何關係表示這些詞之間的語義關係。一個好的詞嵌入空間很大程度上取決於任務。

1.利用Embedding層學習詞嵌入

可以將一個embedding層理解爲字典，他是接受整數作爲輸入，返回相關聯的向量。（單詞索引->對應的詞向量）
在訓練過程中，embedding的權重最開始是隨機的。在訓練過程中，利用反向傳播逐漸調節這些詞向量。
網絡結構：

代碼詳解：

from keras.datasets import imdb
from keras import preprocessing
from keras.models import Sequential
from keras.layers import Flatten, Dense, Embedding

max_features = 10000 #作爲特徵的單詞個數
maxlen = 20 #在maxlen個但此後截斷文本

(x_train, y_train), (x_test, y_test) = imdb.load_data(num_words=max_features) #加載數據
#將整數列表轉換爲(samples,maxlen)的二維整數張量
x_train = preprocessing.sequence.pad_sequences(x_train, maxlen=maxlen)
x_test = preprocessing.sequence.pad_sequences(x_test, maxlen=maxlen)

model = Sequential()
model.add(Embedding(10000, 8, input_length=maxlen)) #標記的個數爲10000， 嵌入的維度爲64， 最大輸入長度爲maxlen

model.add(Flatten()) #將輸出三維張量展平爲二維張量
model.add(Dense(1, activation='sigmoid')) #添加分類器
model.compile(optimizer='rmsprop', loss = 'binary_crossentropy', metrics=['acc'])

print(model.summary())

history = model.fit(x_train, y_train, epochs = 10, batch_size=32, validation_split=0.2)

運行結果：

2.使用預訓練的Glove

Glove這種訓練詞向量的方法的核心思想是通過對’詞-詞’共現矩陣進行分解從而得到詞表示的方法。需要先到https://nlp.stanford.edu/projects/glove下載glove.6B.zip文件並解壓。
代碼詳解：

import os
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
import numpy as np
from keras.models import Sequential
from keras.layers import Flatten, Dense, Embedding
import matplotlib.pyplot as plt


imdb_dir = './aclImdb/aclImdb' #數據集
train_dir = imdb_dir + '/train' #訓練集
test_dir = imdb_dir + '/test' #測試集
glove_dir = './glove.6B' #預訓練的glove模型



labels = [] #存放標籤
texts = []
#將負面和正面分別用0,1表示
for label_type in ['neg', 'pos']:
    dir_name = train_dir + '/' + label_type
    for fname in os.listdir(dir_name):
        if fname[-4:] == '.txt':
            f = open(os.path.join(dir_name, fname),encoding='gb18030', errors='ignore')
            texts.append(f.read())
            f.close()
            if label_type == 'neg':
                labels.append(0)
            else:
                labels.append(1)

test_labels = [] #存放標籤
test_texts = []
#將負面和正面分別用0,1表示
for label_type in ['neg', 'pos']:
    dir_name = test_dir + '/' + label_type
    for fname in os.listdir(dir_name):
        if fname[-4:] == '.txt':
            f = open(os.path.join(dir_name, fname),encoding='gb18030', errors='ignore')
            test_texts.append(f.read())
            f.close()
            if label_type == 'neg':
                test_labels.append(0)
            else:
                test_labels.append(1)

maxlen = 100 #100個單詞後截斷評論
training_samples = 200 #訓練樣本數
validation_samples = 10000 #驗證樣本數
max_words = 10000 #詞頻數限制

tokenizer = Tokenizer(num_words=max_words) #創建分詞器
tokenizer.fit_on_texts(texts) #構建單詞索引
sequences = tokenizer.texts_to_sequences(texts) #將字符串轉爲整數索引組成的列表

word_index = tokenizer.word_index #找回單詞索引
print('Found %s unique tokens.'%len(word_index))

data = pad_sequences(sequences, maxlen=maxlen)

labels = np.asarray(labels)
print('Shape of data tensor:', data.shape)
print('Shape of label tensor:', labels.shape)

#打亂數據
indices = np.arange(data.shape[0])
np.random.shuffle(indices) #索引重組
data = data[indices]
labels = labels[indices]

x_train = data[:training_samples]
y_train = labels[:training_samples]
x_val = data[training_samples: training_samples + validation_samples]
y_val = labels[training_samples: training_samples + validation_samples]

sequences = tokenizer.texts_to_sequences(test_texts) #將字符串轉爲整數索引組成的列表
x_test = pad_sequences(sequences, maxlen=maxlen)
y_test = np.array(test_labels)

#對glove進行預處理，構建一個將單詞（字符串）映射爲其向量表示（數值向量）的索引
embeddings_index = {}

f = open(glove_dir+ '/glove.6B.100d.txt',encoding='gb18030', errors='ignore')
for line in f:
    values = line.split()
    word = values[0] #第一個字符爲單詞索引
    coefs = np.asarray(values[1:], dtype='float32') #其後的爲該單詞對應的embedding_dim向量
    embeddings_index[word] = coefs
f.close()

print('Found %s word vectors.' %len(embeddings_index))

#準備詞嵌入矩陣
embedding_dim = 100
embedding_matrix = np.zeros((max_words, embedding_dim))
for word, i in word_index.items():
    if i < max_words:
        embedding_vector = embeddings_index.get(word)
        if embedding_vector is not None:
            embedding_matrix[i] = embedding_vector

model = Sequential()
model.add(Embedding(max_words, embedding_dim, input_length=maxlen)) #添加embedding層
model.add(Flatten()) #張量展開
model.add(Dense(32, activation='relu'))
model.add(Dense(1, activation='sigmoid'))
print(model.summary())

model.layers[0].set_weights([embedding_matrix]) #將預訓練的詞嵌入加載到embedding層中
model.layers[0].trainable = False

model.compile(optimizer='rmsprop', loss = 'binary_crossentropy', metrics=['acc']) #編譯網絡
history = model.fit(x_train, y_train, epochs = 10, batch_size=32, validation_data=(x_val, y_val)) #訓練
model.save_weights('pre_trained_glove_model.h5') #保存


model.load_weights('./pre_trained_glove_model.h5')
test_loss , test_acc = model.evaluate(x_test, y_test)
print(test_acc)

#繪製acc和loss
'''
acc = history.history['acc']
val_acc = history.history['val_acc']
loss = history.history['loss']
val_loss = history.history['val_loss']

epochs = range(1,len(acc) + 1)

plt.plot(epochs, acc, 'bo', label = 'Training acc')
plt.plot(epochs, val_acc, 'b', label = 'Validation acc')
plt.title('Training and validation accuracy')
plt.legend()

plt.figure()
plt.plot(epochs, loss, 'bo', label = 'training loss')
plt.plot(epochs, val_loss, 'b', label = 'Validation loss')
plt.title('Training and Validation loss')
plt.legend()
plt.show()
'''

運行結果：


測試集準確度：

參考資料

Glove總結