Pytorch專題實戰——激活函數(Activation Functions)

1.激活函數的兩種用法

1.1.softmax激活函數

import torch
import torch.nn as nn
import torch.nn.functional as F   

x = torch.tensor([-1.0, 1.0, 2.0, 3.0])
output = torch.softmax(x, dim=0)   #用torch調用softmax
print(output)

sm = nn.Softmax(dim=0)     #用nn.Softmax
output = sm(x)
print(output)

1.2.sigmoid激活函數

output = torch.sigmoid(x)   #用torch調用sigmoid
print(output)

s = nn.Sigmoid()     #用nn.Sigmoid
output = s(x)
print(output)

1.3.tanh激活函數

output = torch.tanh(x)      #用torch調用tanh
print(output)

t = nn.Tanh()        #用nn.Tanh
output = t(x)
print(output)

1.4.relu激活函數

output = torch.relu(x)       #用torch調用relu
print(output)

relu = nn.ReLU()      #用nn.RuLU
output = relu(x)
print(output)

1.5.leaky_relu激活函數

output = F.leaky_relu(x)     #用F調用leaky_relu
print(output)

lrelu = nn.LeakyReLU()      #用nn.LeakyReLU
output = lrelu(x)
print(output)

2.用激活函數的不同方法構造函數

2.1.nn.ReLU()法

class NeuralNet(nn.Module):
    def __init__(self, input_size, hidden_size):
        super(NeuralNet, self).__init__()
        self.Linear1 = nn.Linear(input_size, hidden_size)
        self.relu = nn.ReLU()      #在__init__用 nn.ReLU
        self.linear2 = nn.Linear(hidden_size, 1)
        self.sigmoid = nn.Sigmoid()
        
    def forward(self, x):
        out = self.linear1(x)
        out = self.relu(out)
        out = slef.linear2(out)
        out = self.sigmoid(out)
        return out

2.2.torch.relu()法

class NeuralNet(nn.Module):
    def __init__(self, input_size, hidden_size):
        super(NerualNet, self).__init__()
        self.Linear1 = nn.Linear(input_size, hidden_size)
        self.Linear2 = nn.Linear(hidden_size, 1)
    
    def forward(self, x):
        out = torch.relu(self.Linear1(x))     #在forward中用torch.relu
        out = torch.sigmoid(self.Linear2(out))
        return out

3.可視化不同激活函數

3.1.sigmoid

import numpy as np
import matplotlib.pyplot as plt

sigmoid = lambda x: 1 / (1 + np.exp(-x))
x = np.linspace(-10, 10, 10)
y = np.linspace(-10, 10, 10)
fig = plt.figure()
plt.plot(y, sigmoid(y))
plt.grid(linestyle='--')
plt.xlabel('X Axis')
plt.ylabel('Y Axis')
plt.title('Sogmoid Function')
plt.xticks([-4, -3, -2, -1, 0, 1, 2, 3, 4])
plt.yticks([-2, -1, 0, 1, 2])
plt.xlim(-4,4)
plt.ylim(-2,2)
plt.show()

3.2.tanh

tanh = lambda x: 2*sigmoid(2*x)-1
x = np.linspace(-10, 10, 10)
y = np.linspace(-10, 10, 10)
plt.plot(y, tanh(y), 'b', label='linspace(-10, 10, 100)')
plt.grid(linestyle='--')
plt.xlabel('X Axis')
plt.ylabel('Y Axis')
plt.title('Tanh Function')
plt.xticks([-4, -3, -2, -1, 0, 1, 2, 3, 4])
plt.yticks([-4, -3, -2, -1, 0, 1, 2, 3, 4])
plt.xlim(-4,4)
plt.ylim(-4,4)
plt.show()

3.3.relu

relu = lambda x: np.where(x>=0, x, 0)
x = np.linspace(-10, 10, 10)
y = np.linspace(-10, 10, 1000)
plt.plot(y, relu(y), 'b', label='linspace(-10, 10, 100)')
plt.grid(linestyle='--')
plt.xlabel('X Axis')
plt.ylabel('Y Axis')
plt.title('RuLU')
plt.xticks([-4, -3, -2, -1, 0, 1, 2, 3, 4])
plt.yticks([-4, -3, -2, -1, 0, 1, 2, 3, 4])
plt.xlim(-4,4)
plt.ylim(-4,4)
plt.show()

3.4.leaky_relu

leakyrelu = lambda x: np.where(x>=0, x, 0.1*x)
x = np.linspace(-10, 10, 10)
y = np.linspace(-10, 10, 1000)
plt.plot(y, leakyrelu(y), 'b', label='linspace(-10, 10, 100)')
plt.grid(linestyle='--')
plt.xlabel('X Axis')
plt.ylabel('Y Axis')
plt.title('Leaky ReLU')
plt.xticks([-4, -3, -2, -1, 0, 1, 2, 3, 4])
plt.yticks([-4, -3, -2, -1, 0, 1, 2, 3, 4])
plt.xlim(-4, 4)
plt.ylim(-4, 4)
plt.show()

3.5.階梯函數

bstep = lambda x: np.where(x>=0, 1, 0)
x = np.linspace(-10, 10, 10)
y = np.linspace(-10, 10, 1000)
plt.plot(y, bstep(y), 'b', label='linspace(-10, 10, 100)')
plt.grid(linestyle='--')
plt.xlabel('X Axis')
plt.ylabel('Y Axis')
plt.title('Step Function')
plt.xticks([-4, -3, -3, -1, 0, 1, 2, 3, 4])
plt.yticks([-2, -1, 0, 1, 2])
plt.xlim(-4, 4)
plt.ylim(-2, 2)
plt.show()