sentiment_embedding_pytorch

1. 下載IMDB公開數據集，並處理

import torch
from torchtext import data

SEED = 1234

torch.manual_seed(SEED) #爲CPU設置隨機種子
torch.cuda.manual_seed(SEED)#爲GPU設置隨機種子

#在程序剛開始加這條語句可以提升一點訓練速度，沒什麼額外開銷。
torch.backends.cudnn.deterministic = True  

#用來定義字段的處理方法（文本字段，標籤字段）
TEXT = data.Field(tokenize='spacy')#torchtext.data.Field : 
LABEL = data.LabelField(dtype=torch.float)

from torchtext import datasets
train_data, test_data = datasets.IMDB.splits(TEXT, LABEL)
print(f'Number of training examples: {len(train_data)}')
print(f'Number of testing examples: {len(test_data)}')

2. 製作訓練集和驗證集¶

import random
train_data, valid_data = train_data.split(random_state=random.seed(SEED))

3. 讀入glove詞向量

TEXT.build_vocab(train_data, max_size=25000, vectors="glove.6B.100d",
                 unk_init=torch.Tensor.normal_)
LABEL.build_vocab(train_data)

BATCH_SIZE = 64

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

train_iterator, valid_iterator, test_iterator = data.BucketIterator.splits(
    (train_data, valid_data, test_data), 
    batch_size=BATCH_SIZE,
    device=device)

4. 建立模型

import torch.nn as nn
import torch.nn.functional as F

class WordAVGModel(nn.Module):
    def __init__(self, vocab_size, embedding_dim, output_dim, pad_idx):
        super().__init__()
        self.embedding = nn.Embedding(vocab_size, embedding_dim, padding_idx=pad_idx)
        self.fc = nn.Linear(embedding_dim, output_dim)
        
    def forward(self, text):
        embedded = self.embedding(text) # [sent_len, batch _size, emb_size]
        embedded = embedded.permute(1, 0, 2) # [batch size, sent len, emb dim]
        pooled = F.avg_pool2d(embedded, (embedded.shape[1], 1)).squeeze(1) # [batch size, embedding_dim]
        return self.fc(pooled)

INPUT_DIM = len(TEXT.vocab) #詞個數
EMBEDDING_DIM = 100 #詞嵌入維度
OUTPUT_DIM = 1 #輸出維度
PAD_IDX = TEXT.vocab.stoi[TEXT.pad_token] #pad索引

model = WordAVGModel(INPUT_DIM, EMBEDDING_DIM, OUTPUT_DIM, PAD_IDX)

def count_parameters(model):
    return sum(p.numel() for p in model.parameters() if p.requires_grad)

print(f'The model has {count_parameters(model):,} trainable parameters')

pretrained_embeddings = TEXT.vocab.vectors
model.embedding.weight.data.copy_(pretrained_embeddings)

UNK_IDX = TEXT.vocab.stoi[TEXT.unk_token]

model.embedding.weight.data[UNK_IDX] = torch.zeros(EMBEDDING_DIM)
model.embedding.weight.data[PAD_IDX] = torch.zeros(EMBEDDING_DIM)

import torch.optim as optim

optimizer = optim.Adam(model.parameters())
criterion = nn.BCEWithLogitsLoss()
model = model.to(device)
criterion = criterion.to(device)

def binary_accuracy(preds, y):
    """
    Returns accuracy per batch, i.e. if you get 8/10 right, this returns 0.8, NOT 8
    """

    #round predictions to the closest integer
    rounded_preds = torch.round(torch.sigmoid(preds))
    correct = (rounded_preds == y).float() #convert into float for division 
    acc = correct.sum()/len(correct)
    return acc

5. 訓練

def train(model, iterator, optimizer, criterion):
    
    
    epoch_loss = 0
    epoch_acc = 0
    total_len = 0
    model.train() #model.train()代表了訓練模式
    #這步一定要加，是爲了區分model訓練和測試的模式的。
    #有時候訓練時會用到dropout、歸一化等方法，但是測試的時候不能用dropout等方法。
    
    
    
    for batch in iterator: #iterator爲train_iterator
        optimizer.zero_grad() #加這步防止梯度疊加
        
        predictions = model(batch.text).squeeze(1)
        #batch.text 就是上面forward函數的參數text
        #壓縮維度，不然跟batch.label維度對不上
        
        loss = criterion(predictions, batch.label)
        acc = binary_accuracy(predictions, batch.label)
        
        
        loss.backward() #反向傳播
        optimizer.step() #梯度下降
        
        epoch_loss += loss.item() * len(batch.label)
        #loss.item()已經本身除以了len(batch.label)
        #所以得再乘一次，得到一個batch的損失，累加得到所有樣本損失。
        
        epoch_acc += acc.item() * len(batch.label)
        #（acc.item()：一個batch的正確率） *batch數 = 正確數
        #train_iterator所有batch的正確數累加。
        
        total_len += len(batch.label)
        #計算train_iterator所有樣本的數量，不出意外應該是17500
        
    return epoch_loss / total_len, epoch_acc / total_len
    #epoch_loss / total_len ：train_iterator所有batch的損失
    #epoch_acc / total_len ：train_iterator所有batch的正確率

def evaluate(model, iterator, criterion):
    
    epoch_loss = 0
    epoch_acc = 0
    model.eval()
    
    with torch.no_grad():
        for batch in iterator:
            predictions = model(batch.text).squeeze(1)
            loss = criterion(predictions, batch.label)
            acc = binary_accuracy(predictions, batch.label)
            epoch_loss += loss.item()
            epoch_acc += acc.item()
        
    return epoch_loss / len(iterator), epoch_acc / len(iterator)

#import time
N_EPOCHS = 10

best_valid_loss = float('inf') #無窮大

for epoch in range(N_EPOCHS):

    #start_time = time.time()
    
    train_loss, train_acc = train(model, train_iterator, optimizer, criterion)
    valid_loss, valid_acc = evaluate(model, valid_iterator, criterion)
    
    #end_time = time.time()

    #epoch_mins, epoch_secs = epoch_time(start_time, end_time)
    
    if valid_loss < best_valid_loss: #只要模型效果變好，就存模型
        best_valid_loss = valid_loss
        torch.save(model.state_dict(), 'wordavg-model.pt')
    
    #print(f'Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s')
    print(f'\tTrain Loss: {train_loss:.3f} | Train Acc: {train_acc*100:.2f}%')
    print(f'\t Val. Loss: {valid_loss:.3f} |  Val. Acc: {valid_acc*100:.2f}%')