总说
记录一些比较有用的pytorch代码(有些是自己写的, 有些是从网上看到的)
本文和Pytorch学习(十五)------ 杂项知识汇总估计会经常更新, 记录一些琐碎的东西.
目录
- 提取网络特征(适用于sequential构建的网络)
- 修改复杂Pretrained的网络(如ResNet等)
- 预训练模型的简单更改(如ResNet等)
- Sequential网络中插入自定义层
- 高斯模糊
- 自定义操作放入Transform中
- 网络不同层用不同学习率的方法
网络不同层用不同学习率的方法
https://github.com/hytseng0509/CrossDomainFewShot/blob/master/methods/LFTNet.py
class LFTNet(nn.Module):
def __init__(self, params, tf_path=None, change_way=True):
super(LFTNet, self).__init__()
#
# Some code here
#
model_params, ft_params = self.split_model_parameters()
self.model_optim = torch.optim.Adam(model_params)
self.ft_optim = torch.optim.Adam(ft_params, weight_decay=1e-8, lr=1e-3)
# total epochs
self.total_epoch = params.stop_epoch
# split the parameters of feature-wise transforamtion layers and others
def split_model_parameters(self):
model_params = []
ft_params = []
for n, p in self.model.named_parameters():
n = n.split('.')
if n[-1] == 'gamma' or n[-1] == 'beta':
ft_params.append(p)
else:
model_params.append(p)
return model_params, ft_params
这种方法，就是直接在定义网络时，将参数进行分离，并用独立的优化器。另一种方法可以参考：
Pytorch学习（二十三）---- 不同layer用不同学习率（高级版本）
提取网络特征
对于内部是sequence构建的网络
class VGG16FeatureExtractor(nn.Module):
def __init__(self):
super().__init__()
vgg16 = models.vgg16(pretrained=True)
self.enc_1 = nn.Sequential(*vgg16.features[:5])
self.enc_2 = nn.Sequential(*vgg16.features[5:10])
self.enc_3 = nn.Sequential(*vgg16.features[10:17])
# fix the encoder
for i in range(3):
# 这种写法挺好的啊!!!!!!!!!!!
for param in getattr(self, 'enc_{:d}'.format(i + 1)).parameters():
param.requires_grad = False
def forward(self, image):
results = [image]
for i in range(3):
func = getattr(self, 'enc_{:d}'.format(i + 1))
results.append(func(results[-1]))
return results[1:]
对于内部不是单纯用sequential构建的网络
import torch
import torchvision.models as models
import torch.nn as nn
class ResNetFeat(nn.Module):
def __init__(self):
super(ResNetFeat, self).__init__()
resnet50_pretrained = models.resnet50(pretrained=True)
conv_modules = list(resnet50_pretrained.children())[:-2] # all layers expect Adaptive AvgPool2D and FC
pool_modules = [list(resnet50_pretrained.children())[-2]] # layers Adaptive AvgPool2D: there is only one.
fc_modules = [list(resnet50_pretrained.children())[-1]] # fc modules (1 only)
self.model_conv = nn.Sequential(*conv_modules)
# 可以看到这里用.children()获得所有的层， 做成list
# 再用sequential包起来
# 值得注意的是，这样的可以适用于任意网络的截取
self.pool = nn.Sequential(*pool_modules)
self.fc = nn.Sequential(*fc_modules)
def forward(self, image):
feat2d = self.model_conv(image)
print(feat2d.shape)
featedpooled = self.pool(feat2d)
# flatten the feature before feeding the fc.
featdense = featedpooled.view(featedpooled.size(0), -1)
pred = self.fc(featdense)
return featdense, pred
if __name__ == "__main__":
x_image = torch.randn(1, 3, 224, 224)
featurizer = ResNetFeat()
feat, pred = featurizer(x_image)
print(feat.shape, pred.shape)
修改复杂Pretrained的网络(如ResNet等)
当你要稍微改改已有的大网络时, 其实可以直接将github中torchvision的相关文件复制, 比如改进ResNet, 先复制 resnet.py ,
这种大的网络, 里面的网络有多个属性构建, 每个属性都是一个block
class ResNet(nn.Module):
def __init__(self, block, layers, num_classes=1000, zero_init_residual=False):
super(ResNet, self).__init__()
self.inplanes = 64
self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3,
bias=False)
self.bn1 = nn.BatchNorm2d(64)
self.relu = nn.ReLU(inplace=True)
self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
self.layer1 = self._make_layer(block, 64, layers[0])
# self.layer2 = self._make_layer(block, 128, layers[1], stride=2)
# self.layer3 = self._make_layer(block, 256, layers[2], stride=2)
# self.layer4 = self._make_layer(block, 512, layers[3], stride=2)
# self.avgpool = nn.AdaptiveAvgPool2d((1, 1))
# self.fc = nn.Linear(512 * block.expansion, num_classes)
...
def forward(self, x):
x = self.conv1(x)
x = self.bn1(x)
x = self.relu(x)
x = self.maxpool(x)
x = self.layer1(x)
# x = self.layer2(x)
# x = self.layer3(x)
# x = self.layer4(x)
# x = self.avgpool(x)
# x = x.view(x.size(0), -1)
# x = self.fc(x)
return x
def resnet101(pretrained=False, **kwargs):
"""Constructs a ResNet-101 model.
Args:
pretrained (bool): If True, returns a model pre-trained on ImageNet
"""
model = ResNet(Bottleneck, [3, 4, 23, 3], **kwargs)
if pretrained:
# 这里一定要`False`, 才能将预先训练的ResNet101的前10层权重复制进去.
model.load_state_dict(model_zoo.load_url(model_urls['resnet101']), False)
return model
比如, 拿到ResNet101前面10层的卷积, 这样改就行了. 然后在其他文件,
from .resnet import resnet101
...
class ResBase(nn.Module):
def __init__(self):
super(ResBase, self).__init__()
# front_end has been truncated till conv2_3
self.front_end = resnet101(pretrained=True)
self.max_pool = torch.nn.MaxPool2d(3, stride=2, padding=1, dilation=1, ceil_mode=False)
# Add some layers you want
self.back_end = torch.nn.Sequential(
torch.nn.Conv2d(256, 128, 3, dilation=1, padding=1),
torch.nn.ReLU(),
torch.nn.Conv2d(128, 64, 3, dilation=1, padding=1),
torch.nn.ReLU(),
torch.nn.Conv2d(64, 3, 3, dilation=1, padding=1)
)
def forward(self, x):
x = self.front_end(x)
x = self.max_pool(x)
return self.back_end(x)
# reinit front_end, especially for resnet.
def _initialize_weights(self):
self.front_end = resnet101(pretrained=True)
**稍微注意一下, 这样构建的网络, 基本是 frond_end用原始weights进行, 再后面的back_end用随机初始化. **
所以, 先整个网络进行随机初始化, 再调用resnet._initialize_weights()就行.
预训练模型的简单更改(如ResNet等)
上面的方法可以进行任意的更改, 毕竟你都直接复制文件了, 不过你如果只是简单的改改fc层, 比如之前是分类1000类的, 现在仍旧用ResNet改成分类100类.
model_conv = torchvision.models.resnet18(pretrained=True)
for param in model_conv.parameters():
param.requires_grad = False
# 知道FC层的输入是几维
num_ftrs = model_conv.fc.in_features
# 原本是从 num_ftrs到1000, 现在直接替换成100类.
model_conv.fc = nn.Linear(num_ftrs, 100)
model_conv = model_conv.to(device)
criterion = nn.CrossEntropyLoss()
# 注意: 这里只是把新的fc层参数输入optimizer中
optimizer_conv = optim.SGD(model_conv.fc.parameters(), lr=0.001, momentum=0.9)
# Decay LR by a factor of 0.1 every 7 epochs
exp_lr_scheduler = lr_scheduler.StepLR(optimizer_conv, step_size=7, gamma=0.1)
这个真的没啥毛病, 很厉害. 在外面直接用新建的fc层替换掉老fc.
另外一点, 你可能觉得奇怪, 就是前面的参数requires_grad都是False了, 都不会计算梯度, 那么放入optimizer不也行吗
Setting requires grad to False means that no gradients will be computed for this Tensor. And so your_tensor.grad will contain it’s previous value (None or a Tensor) and will never be updated.The thing is that when you have momentum (or l2 normalization), even having a gradient of 0 will make your weights change. So if you don’t want to optimize these weights, you want to exclude them from the optimizer to be sure (note that some optimizer will completely ignore tensors that have a grad field set to None and what you proposed will work. But this is not True for all optimizers and thus not safe).
Sequential网络中插入自定义层
Sequential的好处就是, 顺序执行里面包含的一层层网络.
class Mnist_CNN(nn.Module):
def __init__(self):
super().__init__()
self.conv1 = nn.Conv2d(1, 16, kernel_size=3, stride=2, padding=1)
self.conv2 = nn.Conv2d(16, 16, kernel_size=3, stride=2, padding=1)
self.conv3 = nn.Conv2d(16, 10, kernel_size=3, stride=2, padding=1)
def forward(self, xb):
xb = xb.view(-1, 1, 28, 28)
xb = F.relu(self.conv1(xb))
xb = F.relu(self.conv2(xb))
xb = F.relu(self.conv3(xb))
xb = F.avg_pool2d(xb, 4)
return xb.view(-1, xb.size(1))
这里你会说, 咦, 有时候我们想, 要不把view也作为一个网络的层把, 这样直接print(net)的时候, 就可以很清楚, 知道最后拉成了一个向量之类的了.
# 定义一个万能的Lambda层, 继承`nn.Module`类, 传入是任意函数
# 将函数作为一个类属性, 在forward中, 用该函数对输入进行处理.
class Lambda(nn.Module):
def __init__(self, func):
super().__init__()
self.func = func
def forward(self, x):
return self.func(x)
def preprocess(x):
return x.view(-1, 1, 28, 28)
model = nn.Sequential(
Lambda(preprocess),
nn.Conv2d(1, 16, kernel_size=3, stride=2, padding=1),
nn.ReLU(),
nn.Conv2d(16, 16, kernel_size=3, stride=2, padding=1),
nn.ReLU(),
nn.Conv2d(16, 10, kernel_size=3, stride=2, padding=1),
nn.ReLU(),
nn.AvgPool2d(4),
Lambda(lambda x: x.view(x.size(0), -1)),
)
opt = optim.SGD(model.parameters(), lr=lr, momentum=0.9)
高斯模糊
import math
import numbers
import torch
from torch import nn
from torch.nn import functional as F
class GaussianSmoothing(nn.Module):
"""
Apply gaussian smoothing on a
1d, 2d or 3d tensor. Filtering is performed seperately for each channel
in the input using a depthwise convolution.
Arguments:
channels (int, sequence): Number of channels of the input tensors. Output will
have this number of channels as well.
kernel_size (int, sequence): Size of the gaussian kernel.
sigma (float, sequence): Standard deviation of the gaussian kernel.
dim (int, optional): The number of dimensions of the data.
Default value is 2 (spatial).
"""
def __init__(self, channels, kernel_size, sigma, dim=2):
super(GaussianSmoothing, self).__init__()
if isinstance(kernel_size, numbers.Number):
kernel_size = [kernel_size] * dim
if isinstance(sigma, numbers.Number):
sigma = [sigma] * dim
# The gaussian kernel is the product of the
# gaussian function of each dimension.
kernel = 1
meshgrids = torch.meshgrid(
[
torch.arange(size, dtype=torch.float32)
for size in kernel_size
]
)
for size, std, mgrid in zip(kernel_size, sigma, meshgrids):
mean = (size - 1) / 2
kernel *= 1 / (std * math.sqrt(2 * math.pi)) * \
torch.exp(-((mgrid - mean) / std) ** 2 / 2)
# Make sure sum of values in gaussian kernel equals 1.
kernel = kernel / torch.sum(kernel)
# Reshape to depthwise convolutional weight
kernel = kernel.view(1, 1, *kernel.size())
kernel = kernel.repeat(channels, *[1] * (kernel.dim() - 1))
self.register_buffer('weight', kernel)
# change it to 1 if DO NOT want to be channel-wise.
self.groups = channels
if dim == 1:
self.conv = F.conv1d
elif dim == 2:
self.conv = F.conv2d
elif dim == 3:
self.conv = F.conv3d
else:
raise RuntimeError(
'Only 1, 2 and 3 dimensions are supported. Received {}.'.format(dim)
)
def forward(self, input):
"""
Apply gaussian filter to input.
Arguments:
input (torch.Tensor): Input to apply gaussian filter on.
Returns:
filtered (torch.Tensor): Filtered output.
"""
return self.conv(input, weight=self.weight, groups=self.groups)
smoothing = GaussianSmoothing(3, 5, 1)
input = torch.rand(1, 3, 100, 100)
input = F.pad(input, (2, 2, 2, 2), mode='reflect')
output = smoothing(input)
自定义操作放入Transform中
https://discuss.pytorch.org/t/using-a-special-function-as-a-transform/42760