深度卷積神經網絡（AlexNet）

LeNet: 在大的真實數據集上的表現並不盡如⼈意。
1.神經網絡計算複雜。
2.還沒有⼤量深⼊研究參數初始化和⾮凸優化算法等諸多領域。

機器學習的特徵提取:手工定義的特徵提取函數
神經網絡的特徵提取：通過學習得到數據的多級表徵，並逐級表⽰越來越抽象的概念或模式。

神經網絡發展的限制:數據、硬件

AlexNet

首次證明了學習到的特徵可以超越⼿⼯設計的特徵，從而⼀舉打破計算機視覺研究的前狀。
特徵：

8層變換，其中有5層卷積和2層全連接隱藏層，以及1個全連接輸出層。
將sigmoid激活函數改成了更加簡單的ReLU激活函數。
用Dropout來控制全連接層的模型複雜度。
引入數據增強，如翻轉、裁剪和顏色變化，從而進一步擴大數據集來緩解過擬合。

#目前GPU算力資源預計17日上線，在此之前本代碼只能使用CPU運行。
#考慮到本代碼中的模型過大，CPU訓練較慢，
#我們還將代碼上傳了一份到 https://www.kaggle.com/boyuai/boyu-d2l-modernconvolutionalnetwork
#如希望提前使用gpu運行請至kaggle。


import time
import torch
from torch import nn, optim
import torchvision
import numpy as np
import sys
sys.path.append("/home/kesci/input/") 
import d2lzh1981 as d2l
import os
import torch.nn.functional as F

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

class AlexNet(nn.Module):
    def __init__(self):
        super(AlexNet, self).__init__()
        self.conv = nn.Sequential(
            nn.Conv2d(1, 96, 11, 4), # in_channels, out_channels, kernel_size, stride, padding
            nn.ReLU(),
            nn.MaxPool2d(3, 2), # kernel_size, stride
            # 減小卷積窗口，使用填充爲2來使得輸入與輸出的高和寬一致，且增大輸出通道數
            nn.Conv2d(96, 256, 5, 1, 2),
            nn.ReLU(),
            nn.MaxPool2d(3, 2),
            # 連續3個卷積層，且使用更小的卷積窗口。除了最後的卷積層外，進一步增大了輸出通道數。
            # 前兩個卷積層後不使用池化層來減小輸入的高和寬
            nn.Conv2d(256, 384, 3, 1, 1),
            nn.ReLU(),
            nn.Conv2d(384, 384, 3, 1, 1),
            nn.ReLU(),
            nn.Conv2d(384, 256, 3, 1, 1),
            nn.ReLU(),
            nn.MaxPool2d(3, 2)
        )
         # 這裏全連接層的輸出個數比LeNet中的大數倍。使用丟棄層來緩解過擬合
        self.fc = nn.Sequential(
            nn.Linear(256*5*5, 4096),
            nn.ReLU(),
            nn.Dropout(0.5),
            #由於使用CPU鏡像，精簡網絡，若爲GPU鏡像可添加該層
            #nn.Linear(4096, 4096),
            #nn.ReLU(),
            #nn.Dropout(0.5),

            # 輸出層。由於這裏使用Fashion-MNIST，所以用類別數爲10，而非論文中的1000
            nn.Linear(4096, 10),
        )

    def forward(self, img):

        feature = self.conv(img)
        output = self.fc(feature.view(img.shape[0], -1))
        return output

net = AlexNet()
print(net)

AlexNet(
  (conv): Sequential(
    (0): Conv2d(1, 96, kernel_size=(11, 11), stride=(4, 4))
    (1): ReLU()
    (2): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)
    (3): Conv2d(96, 256, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2))
    (4): ReLU()
    (5): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)
    (6): Conv2d(256, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (7): ReLU()
    (8): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (9): ReLU()
    (10): Conv2d(384, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (11): ReLU()
    (12): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)
  )
  (fc): Sequential(
    (0): Linear(in_features=6400, out_features=4096, bias=True)
    (1): ReLU()
    (2): Dropout(p=0.5, inplace=False)
    (3): Linear(in_features=4096, out_features=10, bias=True)
  )
)

#載入數據集
# 本函數已保存在d2lzh_pytorch包中方便以後使用
def load_data_fashion_mnist(batch_size, resize=None, root='/home/kesci/input/FashionMNIST2065'):
    """Download the fashion mnist dataset and then load into memory."""
    trans = []
    if resize:
        trans.append(torchvision.transforms.Resize(size=resize))
    trans.append(torchvision.transforms.ToTensor())
    
    transform = torchvision.transforms.Compose(trans)
    mnist_train = torchvision.datasets.FashionMNIST(root=root, train=True, download=True, transform=transform)
    mnist_test = torchvision.datasets.FashionMNIST(root=root, train=False, download=True, transform=transform)

    train_iter = torch.utils.data.DataLoader(mnist_train, batch_size=batch_size, shuffle=True, num_workers=2)
    test_iter = torch.utils.data.DataLoader(mnist_test, batch_size=batch_size, shuffle=False, num_workers=2)

    return train_iter, test_iter

#batchsize=128
batch_size = 16
# 如出現“out of memory”的報錯信息，可減小batch_size或resize
train_iter, test_iter = load_data_fashion_mnist(batch_size,224)
for X, Y in train_iter:
    print('X =', X.shape,
        '\nY =', Y.type(torch.int32))
    break

#X = torch.Size([16, 1, 224, 224]) 
#Y = tensor([5, 2, 9, 3, 1, 8, 3, 3, 2, 6, 1, 6, 2, 4, 4, 8], dtype=torch.int32)

#訓練
lr, num_epochs = 0.001, 3
optimizer = torch.optim.Adam(net.parameters(), lr=lr)
d2l.train_ch5(net, train_iter, test_iter, batch_size, optimizer, device, num_epochs)

使用重複元素的網絡（VGG）

VGG：通過重複使⽤簡單的基礎塊來構建深度模型。
Block:數個相同的填充爲1、窗口形狀爲3×3的卷積層,接上一個步幅爲2、窗口形狀爲2×2的最大池化層。
卷積層保持輸入的高和寬不變，而池化層則對其減半。

VGG11的簡單實現

def vgg_block(num_convs, in_channels, out_channels): #卷積層個數，輸入通道數，輸出通道數
    blk = []
    for i in range(num_convs):
        if i == 0:
            blk.append(nn.Conv2d(in_channels, out_channels, kernel_size=3, padding=1))
        else:
            blk.append(nn.Conv2d(out_channels, out_channels, kernel_size=3, padding=1))
        blk.append(nn.ReLU())
    blk.append(nn.MaxPool2d(kernel_size=2, stride=2)) # 這裏會使寬高減半
    return nn.Sequential(*blk)
conv_arch = ((1, 1, 64), (1, 64, 128), (2, 128, 256), (2, 256, 512), (2, 512, 512))
# 經過5個vgg_block, 寬高會減半5次, 變成 224/32 = 7
fc_features = 512 * 7 * 7 # c * w * h
fc_hidden_units = 4096 # 任意
def vgg(conv_arch, fc_features, fc_hidden_units=4096):
    net = nn.Sequential()
    # 卷積層部分
    for i, (num_convs, in_channels, out_channels) in enumerate(conv_arch):
        # 每經過一個vgg_block都會使寬高減半
        net.add_module("vgg_block_" + str(i+1), vgg_block(num_convs, in_channels, out_channels))
    # 全連接層部分
    net.add_module("fc", nn.Sequential(d2l.FlattenLayer(),
                                 nn.Linear(fc_features, fc_hidden_units),
                                 nn.ReLU(),
                                 nn.Dropout(0.5),
                                 nn.Linear(fc_hidden_units, fc_hidden_units),
                                 nn.ReLU(),
                                 nn.Dropout(0.5),
                                 nn.Linear(fc_hidden_units, 10)
                                ))
    return net

net = vgg(conv_arch, fc_features, fc_hidden_units)
X = torch.rand(1, 1, 224, 224)

# named_children獲取一級子模塊及其名字(named_modules會返回所有子模塊,包括子模塊的子模塊)
for name, blk in net.named_children(): 
    X = blk(X)
    print(name, 'output shape: ', X.shape)
#output
vgg_block_1 output shape:  torch.Size([1, 64, 112, 112])
vgg_block_2 output shape:  torch.Size([1, 128, 56, 56])
vgg_block_3 output shape:  torch.Size([1, 256, 28, 28])
vgg_block_4 output shape:  torch.Size([1, 512, 14, 14])
vgg_block_5 output shape:  torch.Size([1, 512, 7, 7])
fc output shape:  torch.Size([1, 10])

ratio = 8
small_conv_arch = [(1, 1, 64//ratio), (1, 64//ratio, 128//ratio), (2, 128//ratio, 256//ratio), 
                   (2, 256//ratio, 512//ratio), (2, 512//ratio, 512//ratio)]
net = vgg(small_conv_arch, fc_features // ratio, fc_hidden_units // ratio)
print(net)

Sequential(
  (vgg_block_1): Sequential(
    (0): Conv2d(1, 8, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (1): ReLU()
    (2): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
  )
  (vgg_block_2): Sequential(
    (0): Conv2d(8, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (1): ReLU()
    (2): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
  )
  (vgg_block_3): Sequential(
    (0): Conv2d(16, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (1): ReLU()
    (2): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (3): ReLU()
    (4): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
  )
  (vgg_block_4): Sequential(
    (0): Conv2d(32, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (1): ReLU()
    (2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (3): ReLU()
    (4): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
  )
  (vgg_block_5): Sequential(
    (0): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (1): ReLU()
    (2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (3): ReLU()
    (4): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
  )
  (fc): Sequential(
    (0): FlattenLayer()
    (1): Linear(in_features=3136, out_features=512, bias=True)
    (2): ReLU()
    (3): Dropout(p=0.5, inplace=False)
    (4): Linear(in_features=512, out_features=512, bias=True)
    (5): ReLU()
    (6): Dropout(p=0.5, inplace=False)
    (7): Linear(in_features=512, out_features=10, bias=True)
  )
)


batchsize=16
#batch_size = 64
# 如出現“out of memory”的報錯信息，可減小batch_size或resize
# train_iter, test_iter = d2l.load_data_fashion_mnist(batch_size, resize=224)

lr, num_epochs = 0.001, 5
optimizer = torch.optim.Adam(net.parameters(), lr=lr)
d2l.train_ch5(net, train_iter, test_iter, batch_size, optimizer, device, num_epochs)

⽹絡中的⽹絡（NiN）

LeNet、AlexNet和VGG：先以由卷積層構成的模塊充分抽取空間特徵，再以由全連接層構成的模塊來輸出分類結果。
NiN：串聯多個由卷積層和“全連接”層構成的小⽹絡來構建⼀個深層⽹絡。
⽤了輸出通道數等於標籤類別數的NiN塊，然後使⽤全局平均池化層對每個通道中所有元素求平均並直接⽤於分類。

1×1卷積核作用
1.放縮通道數：通過控制卷積核的數量達到通道數的放縮。
2.增加非線性。1×1卷積核的卷積過程相當於全連接層的計算過程，並且還加入了非線性激活函數，從而可以增加網絡的非線性。
3.計算參數少

def nin_block(in_channels, out_channels, kernel_size, stride, padding):
    blk = nn.Sequential(nn.Conv2d(in_channels, out_channels, kernel_size, stride, padding),
                        nn.ReLU(),
                        nn.Conv2d(out_channels, out_channels, kernel_size=1),
                        nn.ReLU(),
                        nn.Conv2d(out_channels, out_channels, kernel_size=1),
                        nn.ReLU())
    return blk

# 已保存在d2lzh_pytorch
class GlobalAvgPool2d(nn.Module):
    # 全局平均池化層可通過將池化窗口形狀設置成輸入的高和寬實現
    def __init__(self):
        super(GlobalAvgPool2d, self).__init__()
    def forward(self, x):
        return F.avg_pool2d(x, kernel_size=x.size()[2:])

net = nn.Sequential(
    nin_block(1, 96, kernel_size=11, stride=4, padding=0),
    nn.MaxPool2d(kernel_size=3, stride=2),
    nin_block(96, 256, kernel_size=5, stride=1, padding=2),
    nn.MaxPool2d(kernel_size=3, stride=2),
    nin_block(256, 384, kernel_size=3, stride=1, padding=1),
    nn.MaxPool2d(kernel_size=3, stride=2), 
    nn.Dropout(0.5),
    # 標籤類別數是10
    nin_block(384, 10, kernel_size=3, stride=1, padding=1),
    GlobalAvgPool2d(), 
    # 將四維的輸出轉成二維的輸出，其形狀爲(批量大小, 10)
    d2l.FlattenLayer())

X = torch.rand(1, 1, 224, 224)
for name, blk in net.named_children(): 
    X = blk(X)
    print(name, 'output shape: ', X.shape)

#0 output shape:  torch.Size([1, 96, 54, 54])
#1 output shape:  torch.Size([1, 96, 26, 26])
#2 output shape:  torch.Size([1, 256, 26, 26])
#3 output shape:  torch.Size([1, 256, 12, 12])
#4 output shape:  torch.Size([1, 384, 12, 12])
#5 output shape:  torch.Size([1, 384, 5, 5])
#6 output shape:  torch.Size([1, 384, 5, 5])
#7 output shape:  torch.Size([1, 10, 5, 5])
#8 output shape:  torch.Size([1, 10, 1, 1])
#9 output shape:  torch.Size([1, 10])

batch_size = 128
# 如出現“out of memory”的報錯信息，可減小batch_size或resize
#train_iter, test_iter = d2l.load_data_fashion_mnist(batch_size, resize=224)

lr, num_epochs = 0.002, 5
optimizer = torch.optim.Adam(net.parameters(), lr=lr)
d2l.train_ch5(net, train_iter, test_iter, batch_size, optimizer, device, num_epochs)

NiN重複使⽤由卷積層和代替全連接層的1×1卷積層構成的NiN塊來構建深層⽹絡。
NiN去除了容易造成過擬合的全連接輸出層，而是將其替換成輸出通道數等於標籤類別數的NiN塊和全局平均池化層。
NiN的以上設計思想影響了後⾯⼀系列卷積神經⽹絡的設計。

GoogLeNet

由Inception基礎塊組成。
Inception塊相當於⼀個有4條線路的⼦⽹絡。它通過不同窗口形狀的卷積層和最⼤池化層來並⾏抽取信息，並使⽤1×1卷積層減少通道數從而降低模型複雜度。
可以⾃定義的超參數是每個層的輸出通道數，我們以此來控制模型複雜度。

class Inception(nn.Module):
    # c1 - c4爲每條線路里的層的輸出通道數
    def __init__(self, in_c, c1, c2, c3, c4):
        super(Inception, self).__init__()
        # 線路1，單1 x 1卷積層
        self.p1_1 = nn.Conv2d(in_c, c1, kernel_size=1)
        # 線路2，1 x 1卷積層後接3 x 3卷積層
        self.p2_1 = nn.Conv2d(in_c, c2[0], kernel_size=1)
        self.p2_2 = nn.Conv2d(c2[0], c2[1], kernel_size=3, padding=1)
        # 線路3，1 x 1卷積層後接5 x 5卷積層
        self.p3_1 = nn.Conv2d(in_c, c3[0], kernel_size=1)
        self.p3_2 = nn.Conv2d(c3[0], c3[1], kernel_size=5, padding=2)
        # 線路4，3 x 3最大池化層後接1 x 1卷積層
        self.p4_1 = nn.MaxPool2d(kernel_size=3, stride=1, padding=1)
        self.p4_2 = nn.Conv2d(in_c, c4, kernel_size=1)

    def forward(self, x):
        p1 = F.relu(self.p1_1(x))
        p2 = F.relu(self.p2_2(F.relu(self.p2_1(x))))
        p3 = F.relu(self.p3_2(F.relu(self.p3_1(x))))
        p4 = F.relu(self.p4_2(self.p4_1(x)))
        return torch.cat((p1, p2, p3, p4), dim=1)  # 在通道維上連結輸出

GoogLeNet模型

完整模型結構

b1 = nn.Sequential(nn.Conv2d(1, 64, kernel_size=7, stride=2, padding=3),
                   nn.ReLU(),
                   nn.MaxPool2d(kernel_size=3, stride=2, padding=1))

b2 = nn.Sequential(nn.Conv2d(64, 64, kernel_size=1),
                   nn.Conv2d(64, 192, kernel_size=3, padding=1),
                   nn.MaxPool2d(kernel_size=3, stride=2, padding=1))

b3 = nn.Sequential(Inception(192, 64, (96, 128), (16, 32), 32),
                   Inception(256, 128, (128, 192), (32, 96), 64),
                   nn.MaxPool2d(kernel_size=3, stride=2, padding=1))

b4 = nn.Sequential(Inception(480, 192, (96, 208), (16, 48), 64),
                   Inception(512, 160, (112, 224), (24, 64), 64),
                   Inception(512, 128, (128, 256), (24, 64), 64),
                   Inception(512, 112, (144, 288), (32, 64), 64),
                   Inception(528, 256, (160, 320), (32, 128), 128),
                   nn.MaxPool2d(kernel_size=3, stride=2, padding=1))

b5 = nn.Sequential(Inception(832, 256, (160, 320), (32, 128), 128),
                   Inception(832, 384, (192, 384), (48, 128), 128),
                   d2l.GlobalAvgPool2d())

net = nn.Sequential(b1, b2, b3, b4, b5, 
                    d2l.FlattenLayer(), nn.Linear(1024, 10))

net = nn.Sequential(b1, b2, b3, b4, b5, d2l.FlattenLayer(), nn.Linear(1024, 10))

X = torch.rand(1, 1, 96, 96)

for blk in net.children(): 
    X = blk(X)
    print('output shape: ', X.shape)

#batchsize=128
batch_size = 16
# 如出現“out of memory”的報錯信息，可減小batch_size或resize
#train_iter, test_iter = d2l.load_data_fashion_mnist(batch_size, resize=96)

lr, num_epochs = 0.001, 5
optimizer = torch.optim.Adam(net.parameters(), lr=lr)
d2l.train_ch5(net, train_iter, test_iter, batch_size, optimizer, device, num_epochs)

動手學深度學習-08 卷積神經網絡進階

深度卷積神經網絡（AlexNet）

AlexNet

使用重複元素的網絡（VGG）

VGG11的簡單實現

⽹絡中的⽹絡（NiN）

GoogLeNet

GoogLeNet模型

動手學深度學習-20 數據增強

動手學深度學習-19 優化算法進階

動手學深度學習-18 梯度下降

動手學深度學習-16 Transformer

動手學深度學習-15 注意力機制與Seq2seq模型

https://yachay.unat.edu.pe/blog/index.php?comment_area=format_blog&comment_component=blog&comment_co

linux以太網驅動總結