Pytorch入門與實踐——神經網絡工具箱

import torch as t
from torch import nn
from torch.autograd import Variable as V
from torch.nn import functional as F

from PIL import Image
from torchvision.transforms import ToTensor, ToPILImage
from matplotlib import pyplot as plt

#nn.Module
class Linear(nn.Module):    #繼承nn.Module。用nn.Module實現自己的全連接層
    def __init__(self, in_features, out_features):
        super(Linear, self).__init__()  #等價於nn.Module.__init__(self)
        self.w = nn.Parameter(t.randn(in_features, out_features))
        self.b = nn.Parameter(t.randn(out_features))

    def forward(self, x):
        x = x.mm(self.w)
        return x + self.b.expand_as(x)

# layer = Linear(4,  3)
# input = V(t.randn(2, 4))
# output = layer(input)   #等價於layers.__call__(input)，在__call__函數中，主要調用的是layer.forward(x)
# # print(output)
# for name, parameter in layer.named_parameters():    #Module中的可學習參數可通過named_parameters()或者parameters()返回迭代器
#     print(name, parameter)  #w and b

class Perceptron(nn.Module):    #多層感知機
    def __init__(self, in_features, hidden_features, out_features):
        nn.Module.__init__(self)
        self.layer1 = Linear(in_features, hidden_features)
        self.layer2 = Linear(hidden_features, out_features)
    def forward(self, x):
        x = self.layer1(x)
        x = t.sigmoid(x)
        return self.layer2(x)

# perceptron = Perceptron(3, 4, 1)
# for name, param in perceptron.named_parameters():
#     print(name, param.size())


##########################常用的神經網絡層#############################
#圖像相關層
to_tensor = ToTensor()  #img -> tensor
to_pil = ToPILImage()
cat = Image.open(r'C:\Users\45840\Pictures\Saved Pictures\cat.jpg')
# plt.imshow(cat)
# plt.show()
# print(cat)

# cat = cat.convert('L')  #轉換爲灰度圖像
# plt.imshow(cat)
# plt.show()
input = to_tensor(cat).unsqueeze(0) #將數據僞裝成batch=1的batch
# print(input.shape)
kernel = t.ones(3, 3)/-9.
kernel[1][1] = 1
conv = nn.Conv2d(1, 1, (3, 3), 1, bias=False)
conv.weight.data = kernel.view(1, 1, 3, 3)
# out = conv(V(input))      #卷積操作
# plt.imshow(to_pil(out.data.squeeze(0)))
# plt.show()

pool = nn.AvgPool2d(2, 2)   #池化層沒有可學習參數，weight是固定的
# print(list(pool.parameters()))
# out = pool(V(input))
# plt.imshow(to_pil(out.data.squeeze(0)))     #池化操作
# plt.show()

input = V(t.randn(2, 3))
linear = nn.Linear(3, 4)    #全連接層
h = linear(input)
# print(h)

bn = nn.BatchNorm1d(4)
bn.weight.data = t.ones(4) * 4
bn.bias.data = t.zeros(4)
bn_out = bn(h)
# print(bn_out)
# print(bn_out.mean(0), bn_out.var(0, unbiased=False))   #使用unbiased=False，分母不減1

dropout = nn.Dropout(0.5)
o = dropout(bn_out)
# print(o)      #o的值一部分爲0，一部分的值變大

#激活函數
relu = nn.ReLU(inplace=True)    #因爲inplace=True，所以input自身也會改變,會把輸出直接覆蓋到輸入中
input = V(t.randn(2, 3))
# print(input)
relu(input)     #input值被覆蓋
# print(input)

#Sequential的三種寫法
net1 = nn.Sequential()
net1.add_module('conv', nn.Conv2d(3, 3, 3))
net1.add_module('batchnorm', nn.BatchNorm2d(3))
net1.add_module('activation_layer', nn.ReLU())

net2 = nn.Sequential(
    nn.Conv2d(3, 3, 3),
    nn.BatchNorm2d(3),
    nn.ReLU()
)

from collections import OrderedDict
net3 = nn.Sequential(OrderedDict([
    ('conv1', nn.Conv2d(3, 3, 3)),
    ('bn', nn.BatchNorm2d(3)),
    ('relu1', nn.ReLU())
]))

# print('net1:', net1)
# print('net2:', net2)
# print('net3:', net3)
# print(net1.conv, net2[0], net3.conv1)     #可根據名字或序號取出子module

# input = V(t.rand(1, 3, 4, 4))
# output1 = net1(input)
# output2 = net2(input)
# output3 = net3(input)
# output4 = net3.relu1(net1.batchnorm(net1.conv(input)))
# print(output1, output2, output3, output4)


class MyModule(nn.Module):
    def __init__(self):
        super(MyModule, self).__init__()
        self.list = [nn.Linear(3, 4), nn.ReLU()]    #list中的子module不能被主module識別，而ModuleList中的子module可以
        self.module_list = nn.ModuleList([nn.Conv2d(3, 3, 3), nn.ReLU()])
    def forward(self):
        pass

model = MyModule()
# print(model)
# for name, param in model.named_parameters():
#     print(name, param.size())

#循環神經網絡
#PASS

#損失函數，交叉熵損失CrossEntropyloss爲例
# score = V(t.randn(3, 2))    #batch_size = 3
# label = V(t.Tensor([1, 0, 1])).long()   #label必須是LongTensor
# criterion = nn.CrossEntropyLoss()
# loss = criterion(score, label)
# print(loss)


##################優化器########################
#首先定義一個LeNet網絡
class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.features = nn.Sequential(
            nn.Conv2d(3, 6, 5),
            nn.ReLU(),
            nn.MaxPool2d(2, 2),
            nn.Conv2d(6, 16 ,5),
            nn.ReLU(),
            nn.MaxPool2d(2, 2)
        )
        self.classifier = nn.Sequential(
            nn.Linear(16*5*5, 120),
            nn.ReLU(),
            nn.Linear(120, 84),
            nn.ReLU(),
            nn.Linear(84, 10)
        )

    def forward(self, x):
        x = self.features(x)
        x = x.view(-1, 16*5*5)
        x = self.classifier(x)
        return x

net = Net()

from torch import optim     #常用優化方法全部封裝在torch.optim中

# optimizer = optim.SGD(params=net.parameters(), lr=1)
# optimizer.zero_grad()   #梯度清零，等價於net.zero_grad()
#
# input = V(t.randn(1, 3, 32, 32))
# output = net(input)
# # print(output)
# output.backward(output)     #fake backward
# optimizer.step()    #執行優化

#爲不同子網絡設置不同的學習率，在finetune中經常用到
# optimizer = optim.SGD([
#     {'params': net.features.parameters()},      #未指定學習率，使用默認學習率，爲1e-5
#     {'params': net.classifier.parameters(), 'lr': 1e-2}
# ], lr = 1e-5)

#只爲兩個全連接層設置較大的學習率，其餘層的學習率較小
# special_layers = nn.ModuleList([net.classifier[0], net.classifier[3]])
# special_layers_params = list(map(id, special_layers.parameters()))
# base_params = filter(lambda p: id(p) not in special_layers_params, net.parameters())
# optimizer = optim.SGD([
#     {'params': base_params},
#     {'params': special_layers.parameters(), 'lr': 0.01}
# ], lr = 0.001)

#調整學習率，新建一個optimizer
# old_lr = 0.1
# optimizer = optim.SGD([
#     {'params': net.features.parameters()},
#     {'params': net.classifier.parameters(), 'lr': old_lr*0.1}
# ], lr = 1e-5)


#################nn.functional###############
# input = V(t.randn(2, 3))
# model = nn.Linear(3, 4)
# output1 = model(input)
# output2 = nn.functional.linear(input, model.weight, model.bias)
# # print(output1 == output2)
#
# b = nn.functional.relu(input)
# b2 = nn.ReLU()(input)
# # print(b == b2)
#
# from torch.nn import functional as F
# class Net(nn.Module):
#     def __init__(self):
#         super(Net, self).__init__()
#         self.conv1 = nn.Conv2d(3, 6, 5)
#         self.conv2 = nn.Conv2d(6, 16, 5)
#         self.fc1 = nn.Linear(16*5*5, 120)
#         self.fc2 = nn.Linear(120, 84)
#         self.fc3 = nn.Linear(84, 10)
#
#     def forward(self, x):   #不具備可學習參數的層(激活層、池化層等)可以用函數代替，這樣可以不用放置在構造函數__init__中
#         x = F.pool(F.relu(self.conv1(x)), 2)
#         x = F.pool(F.relu(self.conv2(x)), 2)
#         x = x.view(-1, 16*5*5)
#         x = F.relu(self.fc1(x))
#         x = F.relu(self.fc2(x))
#         x = self.fc3(x)
#         return x


####################初始化策略######################
#PyTorch中的nn.init模塊專門爲初始化設計，實現了常用的初始化策略
#利用nn.init初始化
# from torch.nn import init
# linear = nn.Linear(3, 4)
# t.manual_seed(1)
# init.xavier_normal(linear.weight)   #等價於linear.weight.data.normal_(0, std)

#直接初始化
# import math
# linear = nn.Linear(4, 3)
# t.manual_seed(1)
# std = math.sqrt(2)/math.sqrt(7.)    #xavier初始化的計算公式
# print(linear.weight.data.normal_(0, std))

#對模型的所有參數進行初始化
# for name, params in net.named_parameters():
#     if name.find('linear') != -1:
#         #init linear
#         params[0]   #weight
#         params[1]   #bias
#     elif name.find('conv') != -1:
#         pass
#     elif name.find('norm') != -1:
#         pass

#保存模型
# t.save(net.state_dict(), 'net.pth')
# #加載已保存的模型
# net2 = Net()
# net2.load_state_dict(t.load('net.path'))


########################50行代碼搭建ResNet########################
class ResidualBlock(nn.Module):
    #實現子module:Residual Block
    def __init__(self, inchannel, outchannel, stride=1, shortcut=None):
        super(ResidualBlock, self).__init__()
        self.left = nn.Sequential(
            nn.Conv2d(inchannel, outchannel, 3, stride, 1, bias=False),
            nn.BatchNorm2d(outchannel),
            nn.ReLU(inplace=True),
            nn.Conv2d(outchannel, outchannel, 3, 1, 1, bias=False),
            nn.BatchNorm2d(outchannel)
        )
        self.right = shortcut

    def forward(self, x):
        out = self.left(x)
        residual = x if self.right is None else self.right(x)
        out += residual
        return F.relu(out)

class ResNet(nn.Module):
    def __init__(self, num_classes=1000):
        super(ResNet, self).__init__()
        #前幾層圖像轉換
        self.pre = nn.Sequential(
            nn.Conv2d(3, 64, 7, 2, 3, bias=False),
            nn.BatchNorm2d(64),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(3, 2, 1)
        )
        # 重複的layer,分別有3， 4， 6， 3個residual block
        self.layer1 = self._make_layer(64, 128, 3)
        self.layer2 = self._make_layer(128, 256, 4, stride=2)
        self.layer3 = self._make_layer(256, 512, 6, stride=2)
        self.layer4 = self._make_layer(512, 512, 3, stride=2)

        #分類用的全連接
        self.fc = nn.Linear(512, num_classes)

    def _make_layer(self, inchannel, outchannel, block_num, stride=1):
        #構建layer,包含多個residual block
        shortcut = nn.Sequential(
            nn.Conv2d(inchannel, outchannel, 1, stride, bias=False),
            nn.BatchNorm2d(outchannel)
        )

        layers = []
        layers.append(ResidualBlock(inchannel, outchannel, stride, shortcut))
        for i in range(1, block_num):
            layers.append(ResidualBlock(outchannel, outchannel))
        return nn.Sequential(*layers)

    def forward(self, x):
        x = self.pre(x)

        x = self.layer1(x)
        x = self.layer2(x)
        x = self.layer3(x)
        x = self.layer4(x)

        x = F.avg_pool2d(x, 7)
        x = x.view(x.size(0), -1)
        return self.fc(x)

model = ResNet()
input = V(t.randn(1, 3, 224, 224))
o = model(input)
print(o)