机器学习日志 猫狗分类 Dogs vs. Cats pytorch 神经网络

题目让判断一个图片是狗的概率

写了个神经网络score是2.94，1200多名，太差了，慢慢改

发现的几个问题，总结一下：

1.运行的时候老是提示内存不足，这时候改一下虚拟内存就行，具体方法百度，或者调小batch_size也行

2.python是逐行运行的，也就是说你后面的代码有错误它会先跑完前面才报错，所以训练的模型及时保存

点我展开代码import math
import torch
import torch.nn as nn
import pandas as pd
import numpy as np
from torch.utils import data
from myModel import WYJ_CNN
import os
from PIL import Image
import numpy as np
from torchvision import transforms as T
from torch.utils.data import DataLoader

transform = T.Compose([#HWC怎么变为CHW?
    T.Resize(224),  # 缩放图片(Image)，保持长宽比不变，最短边为224像素
    T.CenterCrop(224),  # 从图片中间切出224*224的图片
    T.ToTensor(),  # 将图片(Image)转成Tensor，归一化至[0, 1]
    T.Normalize(mean=[.5, .5, .5], std=[.5, .5, .5])  # 标准化至[-1, 1]，规定均值和标准差
])

class DogCat(data.Dataset):
    def __init__(self, root, transforms=None):
        imgs = os.listdir(root)
        # 所有图片的绝对路径
        # 这里不实际加载图片，只是指定路径，当调用__getitem__时才会真正读图片
        self.imgs = [os.path.join(root, img) for img in imgs]
        self.transforms = transforms

    def __getitem__(self, index):
        img_path = self.imgs[index]
        label = 1 if 'dog' in img_path.split('/')[-1] else 0
        data = Image.open(img_path)
        if self.transforms:
            data = self.transforms(data)
        return data, label

    def __len__(self):
        return len(self.imgs)


net = WYJ_CNN()

CalcLoss = nn.CrossEntropyLoss()#loss用交叉熵来算
optimizer = torch.optim.SGD(net.parameters(), lr=0.05)#lr是学习率

trainset = DogCat('./train', transforms=transform)
trainloader = DataLoader(trainset, batch_size=100, shuffle=True, num_workers=0, drop_last=False)

testset = DogCat('./test', transforms=transform)
testloader = DataLoader(testset, batch_size=10, shuffle=False, num_workers=0, drop_last=False)
for cnt in range(10):
    sum=0
    for batch_datas, batch_labels in trainloader:
        sum=sum+1
        outputs = net(batch_datas)
        loss = CalcLoss(outputs, batch_labels)
        optimizer.zero_grad()  # 注意清空优化器的梯度，防止累计
        loss.backward()
        optimizer.step()
        print("training",cnt," ",sum*100)
    if np.isnan(np.isnan(outputs.detach().numpy().any())):
        print("NAN error!")
        break;

torch.save(net, "my_cnn.nn")
net = torch.load("my_cnn.nn")

id = [];
label = [];

i=0
for batch_datas, batch_labels in testloader:
    print(batch_datas.shape)
    outputs = net(batch_datas)
    id.extend(np.arange(i*10+1,(i+1)*10+1))
    label.extend(outputs[:,1].detach().numpy())
    i = i + 1
    print("testing", i*10)
myAns = pd.DataFrame({'id': id, 'label': label})
myAns.to_csv("myAns.csv", index=False, sep=',')

放一个飞桨版本

改出来了，踩的坑不少，到时候另开一个总结一下

用的是resnet50，训练效果挺好的，排到了中间位置

点我展开代码import math
import paddle
import paddle.nn as nn
import pandas as pd
import numpy as np
from myModel import WYJ_CNN
import os
from PIL import Image
import numpy as np
import paddle.vision.transforms as T
from paddle.io import DataLoader
from IPython.display import display

transform = T.Compose([
    T.Resize(224),
    T.RandomResizedCrop(224,scale=(0.6,1.0),ratio=(0.8,1.0)),
    T.RandomHorizontalFlip(),
    T.ColorJitter(brightness=0.5, contrast=0, saturation=0, hue=0),
    T.ColorJitter(brightness=0, contrast=0.5, saturation=0, hue=0),
    T.ToTensor(),
    T.Normalize(mean=[.5, .5, .5], std=[.5, .5, .5])
])


class DogCat(paddle.io.Dataset):
    def __init__(self, root, transforms=None):
        imgs = os.listdir(root)
        # 所有图片的绝对路径
        # 这里不实际加载图片，只是指定路径，当调用__getitem__时才会真正读图片
        self.imgs = [os.path.join(root, img) for img in imgs]
        self.transforms = transforms

    def __getitem__(self, index):
        img_path = self.imgs[index]
        label = 1 if 'dog' in img_path.split('/')[-1] else 0
        data = Image.open(img_path)
        if self.transforms:
            data = self.transforms(data)
        return data, label ,img_path

    def __len__(self):
        return len(self.imgs)

paddle.device.set_device("gpu:0")

net = paddle.vision.models.resnet50(pretrained=True)
net.fc = nn.Sequential(nn.Linear(2048,2))

CalcLoss = nn.CrossEntropyLoss()#loss用交叉熵来算
optimizer = paddle.optimizer.SGD(learning_rate =0.01 , parameters=net.parameters() )#lr是学习率

net_state_dict=paddle.load('net.pdparams')
net.set_state_dict(net_state_dict)
optimizer_state_dict=paddle.load('optimizer.pdopt')
optimizer.set_state_dict(optimizer_state_dict)


trainset = DogCat('./train', transforms=transform)
a_list = paddle.io.random_split(trainset, [int(len(trainset)*0.8), len(trainset)-int(len(trainset)*0.8)])
trainloader0 = DataLoader(a_list[0], batch_size=32, shuffle=True, num_workers=0, drop_last=False)
trainloader1 = DataLoader(a_list[1], batch_size=32, shuffle=True, num_workers=0, drop_last=False)
testset = DogCat('./test', transforms=transform)
testloader = DataLoader(testset, batch_size=10, shuffle=False, num_workers=0, drop_last=False)


for cnt in range(3):
    sum=0
    for batch_datas, batch_labels ,name in trainloader0:
        sum=sum+1
        outputs = net(batch_datas)
        loss = CalcLoss(outputs, batch_labels)
        optimizer.clear_grad()  # 注意清空优化器的梯度，防止累计
        loss.backward()
        optimizer.step()

        #print(name[0],batch_labels[0])

        if sum%100==0:
            print("training",cnt," ",sum*32)
    if np.isnan(np.isnan(outputs.detach().numpy().any())):
        print("NAN error!")
        break;
    total_loss = 0  # 保存这次测试总的loss
    with paddle.no_grad():   # 下面不需要反向传播，所以不需要自动求导
        for batch_datas, batch_labels ,name in trainloader1:
            outputs = net(batch_datas)
            loss = CalcLoss(outputs, batch_labels)
            total_loss += loss  # 累计误差
    print("第{}次训练的Loss:{}".format(cnt + 1, total_loss))
    paddle.save(net.state_dict(),'net.pdparams')
    paddle.save(optimizer.state_dict(),'optimizer.pdopt')


id = [];
label = [];
ans_dict={}
i=0
for batch_datas, batch_labels ,name in testloader:
    #print(batch_datas.shape)
    outputs = net(batch_datas)
    m = paddle.nn.Softmax()
    outputs=m(outputs) 

    for j in range(len(outputs)):
        num = ''.join([x for x in name[j] if x.isdigit()])
        ans_dict[num]=outputs[j][1].item()

    i = i + 1
    if i%100==0:
        print("testing", i*10)

for i in range(len(ans_dict)):
    id.append(i+1)
    label.append(ans_dict.get(str(i+1)))
myAns = pd.DataFrame({'id': id, 'label': label})
myAns.to_csv("myAns.csv", index=False, sep=',')