HybridSN中添加SE模块

代码练习

HybridSN中添加SE模块

#! wget http://www.ehu.eus/ccwintco/uploads/6/67/Indian_pines_corrected.mat
#! wget http://www.ehu.eus/ccwintco/uploads/c/c4/Indian_pines_gt.mat
#! pip install spectral
import numpy as np
import matplotlib.pyplot as plt
import scipy.io as sio
from sklearn.decomposition import PCA
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix, accuracy_score, classification_report, cohen_kappa_score
import spectral
import torch
import torchvision
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim

class_num = 16

class SEBlock(nn.Module):
  def __init__(self,in_channels,r=16):
    super(SEBlock,self).__init__()
    self.globalAvgPool = nn.AdaptiveAvgPool2d((1,1))
    self.fc1 = nn.Linear(in_channels,round(in_channels/r))
    self.fc2 = nn.Linear(round(in_channels/r),in_channels)
  def forward(self,x):
    out = self.globalAvgPool(x)
    out = out.view(x.shape[0],-1)
    out = F.relu(self.fc1(out))
    out = F.sigmoid(self.fc2(out))
    out = out.view(x.shape[0],x.shape[1],1,1)
    out = x * out
    return out

class HybridSN(nn.Module):
  def __init__(self):
    super(HybridSN,self).__init__()
    self.conv3d1 = nn.Conv3d(1,8,kernel_size=(7,3,3),stride=1,padding=0)
    self.bn1 = nn.BatchNorm3d(8)
    self.conv3d2 = nn.Conv3d(8,16,kernel_size=(5,3,3),stride=1,padding=0)
    self.bn2 = nn.BatchNorm3d(16)
    self.conv3d3 = nn.Conv3d(16,32,kernel_size=(3,3,3),stride=1,padding=0)
    self.bn3 = nn.BatchNorm3d(32)
    self.conv2d4 = nn.Conv2d(576,64,kernel_size=(3,3),stride=1,padding=0)
    self.SElayer = SEBlock(64,16)
    self.bn4 = nn.BatchNorm2d(64)
    self.fc1 = nn.Linear(18496,256)
    self.fc2 = nn.Linear(256,128)
    self.fc3 = nn.Linear(128,16)
    self.dropout = nn.Dropout(0.4)

  def forward(self,x):
    out = F.relu(self.bn1(self.conv3d1(x)))
    out = F.relu(self.bn2(self.conv3d2(out)))
    out = F.relu(self.bn3(self.conv3d3(out)))
    out = F.relu(self.bn4(self.conv2d4(out.reshape(out.shape[0],-1,19,19))))
    out = self.SElayer(out)
    out = out.reshape(out.shape[0],-1)
    out = F.relu(self.dropout(self.fc1(out)))
    out = F.relu(self.dropout(self.fc2(out)))
    out = self.fc3(out)
    return out


def applyPCA(X, numComponents):
    newX = np.reshape(X, (-1, X.shape[2]))
    pca = PCA(n_components=numComponents, whiten=True)
    newX = pca.fit_transform(newX)
    newX = np.reshape(newX, (X.shape[0], X.shape[1], numComponents))
    return newX

# 对单个像素周围提取 patch 时，边缘像素就无法取了，因此，给这部分像素进行 padding 操作
def padWithZeros(X, margin=2):
    newX = np.zeros((X.shape[0] + 2 * margin, X.shape[1] + 2* margin, X.shape[2]))
    x_offset = margin
    y_offset = margin
    newX[x_offset:X.shape[0] + x_offset, y_offset:X.shape[1] + y_offset, :] = X
    return newX

# 在每个像素周围提取 patch ，然后创建成符合 keras 处理的格式
def createImageCubes(X, y, windowSize=5, removeZeroLabels = True):
    # 给 X 做 padding
    margin = int((windowSize - 1) / 2)
    zeroPaddedX = padWithZeros(X, margin=margin)
    # split patches
    patchesData = np.zeros((X.shape[0] * X.shape[1], windowSize, windowSize, X.shape[2]))
    patchesLabels = np.zeros((X.shape[0] * X.shape[1]))
    patchIndex = 0
    for r in range(margin, zeroPaddedX.shape[0] - margin):
        for c in range(margin, zeroPaddedX.shape[1] - margin):
            patch = zeroPaddedX[r - margin:r + margin + 1, c - margin:c + margin + 1]   
            patchesData[patchIndex, :, :, :] = patch
            patchesLabels[patchIndex] = y[r-margin, c-margin]
            patchIndex = patchIndex + 1
    if removeZeroLabels:
        patchesData = patchesData[patchesLabels>0,:,:,:]
        patchesLabels = patchesLabels[patchesLabels>0]
        patchesLabels -= 1
    return patchesData, patchesLabels

def splitTrainTestSet(X, y, testRatio, randomState=345):
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=testRatio, random_state=randomState, stratify=y)
    return X_train, X_test, y_train, y_test
# 地物类别
class_num = 16
X = sio.loadmat('Indian_pines_corrected.mat')['indian_pines_corrected']
y = sio.loadmat('Indian_pines_gt.mat')['indian_pines_gt']

# 用于测试样本的比例
test_ratio = 0.90
# 每个像素周围提取 patch 的尺寸
patch_size = 25
# 使用 PCA 降维，得到主成分的数量
pca_components = 30

print('Hyperspectral data shape: ', X.shape)
print('Label shape: ', y.shape)

print('\n... ... PCA tranformation ... ...')
X_pca = applyPCA(X, numComponents=pca_components)
print('Data shape after PCA: ', X_pca.shape)

print('\n... ... create data cubes ... ...')
X_pca, y = createImageCubes(X_pca, y, windowSize=patch_size)
print('Data cube X shape: ', X_pca.shape)
print('Data cube y shape: ', y.shape)

print('\n... ... create train & test data ... ...')
Xtrain, Xtest, ytrain, ytest = splitTrainTestSet(X_pca, y, test_ratio)
print('Xtrain shape: ', Xtrain.shape)
print('Xtest  shape: ', Xtest.shape)

# 改变 Xtrain, Ytrain 的形状，以符合 keras 的要求
Xtrain = Xtrain.reshape(-1, patch_size, patch_size, pca_components, 1)
Xtest  = Xtest.reshape(-1, patch_size, patch_size, pca_components, 1)
print('before transpose: Xtrain shape: ', Xtrain.shape) 
print('before transpose: Xtest  shape: ', Xtest.shape) 

# 为了适应 pytorch 结构，数据要做 transpose
Xtrain = Xtrain.transpose(0, 4, 3, 1, 2)
Xtest  = Xtest.transpose(0, 4, 3, 1, 2)
print('after transpose: Xtrain shape: ', Xtrain.shape) 
print('after transpose: Xtest  shape: ', Xtest.shape) 


""" Training dataset"""
class TrainDS(torch.utils.data.Dataset): 
    def __init__(self):
        self.len = Xtrain.shape[0]
        self.x_data = torch.FloatTensor(Xtrain)
        self.y_data = torch.LongTensor(ytrain)        
    def __getitem__(self, index):
        # 根据索引返回数据和对应的标签
        return self.x_data[index], self.y_data[index]
    def __len__(self): 
        # 返回文件数据的数目
        return self.len

""" Testing dataset"""
class TestDS(torch.utils.data.Dataset): 
    def __init__(self):
        self.len = Xtest.shape[0]
        self.x_data = torch.FloatTensor(Xtest)
        self.y_data = torch.LongTensor(ytest)
    def __getitem__(self, index):
        # 根据索引返回数据和对应的标签
        return self.x_data[index], self.y_data[index]
    def __len__(self): 
        # 返回文件数据的数目
        return self.len

# 创建 trainloader 和 testloader
trainset = TrainDS()
testset  = TestDS()
train_loader = torch.utils.data.DataLoader(dataset=trainset, batch_size=128, shuffle=True, num_workers=2)
test_loader  = torch.utils.data.DataLoader(dataset=testset,  batch_size=128, shuffle=False, num_workers=2)
# 使用GPU训练，可以在菜单 "代码执行工具" -> "更改运行时类型" 里进行设置
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

# 网络放到GPU上
net = HybridSN().to(device)

criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(net.parameters(), lr=0.001)
scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'min',verbose=True,factor=0.9,min_lr=1e-6)

# 开始训练
total_loss = 0
net.train()
for epoch in range(100):
    for i, (inputs, labels) in enumerate(train_loader):
        inputs = inputs.to(device)
        labels = labels.to(device)
        # 优化器梯度归零
        optimizer.zero_grad()
        # 正向传播 +　反向传播 + 优化 
        outputs = net(inputs)
        loss = criterion(outputs, labels)
        loss.backward()
        optimizer.step()
        scheduler.step(loss) 
        total_loss += loss.item()
        nn.ReLU()
    print('[Epoch: %d]   [loss avg: %.4f]   [current loss: %.4f]' %(epoch + 1, total_loss/(epoch+1), loss.item()))

print('Finished Training')
net.eval()
count = 0
for inputs,labels in test_loader:
  inputs = inputs.to(device)
  labels = labels.to(device)
  outputs = net(inputs)
  _,preds = torch.max(outputs,1)
  count += (preds == labels).sum().item()
print("Test ACC:{}".format(count/len(testset)))

测试结果对比

未加BN、SE模块时测试三次的结果：0.9805、0.9412、0.9624
添加BN层后测试三次的结果：0.9860、0.9789、0.9820
再添加学习率衰减后测试三次的结果：0.9895、0.9901、0.9897
再添加SE模块后测试三次的结果(测试在2D卷积层前后都添加SE模块，效果并不好）：0.9913、0.9895、0.9912

添加BN层后训练结果比较稳定，而且收敛的很快。
添加SE模块后，准确率还是略有提高的，但效果没有那么明显，可能是因为模型比较简单，已经采取提升性能的一些操作，所以提升并没有那么明显。

问题思考

训练网络，然后多测试几次，会发现每次分类的结果都不一样的原因

原因是 网络中添加了dropout层和BN层时，在测试时应当关闭这两个层。
在训练时应当指定当前是训练模式:model.train（）开启这两个层；在测试时应当指定是测试模式:model.eval（）关闭这两个层。
dropout层在训练过程中以指定概率p使神经元失活，让它在这次的传播过程的输出为0。当我们的模型训练好进行预测时，要使用所有神经元而且要乘以一个补偿系数。所以要指定当前是训练还是测试模式。
BN层在测试的时候采用的是固定的mean和var，这俩固定的参数是在训练时统计计算得到的。因为这俩参数是在前向传播过程中计算的，所以在测试模式的时候你如果没有指定model.eval（）,那么这俩参数还会根据你的测试数据更新，导致结果的参考价值不大。
综上，如果网络中添加了BN层和dropout层而不使用model.eval（）的话，每次测试的时候 模型并不是固定的，所以每次的分类结果可能并不一致。

SENet的提升分类性能的本质原理

Excitation 的输出的权重看做是进过特征选择后的每个特征通道的重要性，然后通过乘法逐通道加权到先前的特征上，完成在通道维度上的对原始特征的重标定，提升有用的特征并抑制对当前任务用处不大的特征。
猜测：有用的特征的乘的scale比较大，它的数值就比较大；用处不大的特征乘的scale较小，它的数值就比较小。考虑在最后一个全局平均池化层处，有用的特征经过池化后的输出也比较大，它对最终分类结果的影响就比较大，同理用处不大的特征对最终分类结果的影响就比较小，这样提升了分类的性能。