机器学习日志 猫狗分类 Dogs vs. Cats pytorch 神经网络
题目让判断一个图片是狗的概率
写了个神经网络score是2.94，1200多名，太差了，慢慢改
发现的几个问题，总结一下：
1.运行的时候老是提示内存不足，这时候改一下虚拟内存就行，具体方法百度，或者调小batch_size也行
2.python是逐行运行的，也就是说你后面的代码有错误它会先跑完前面才报错，所以训练的模型及时保存
import math
import torch
import torch.nn as nn
import pandas as pd
import numpy as np
from torch.utils import data
from myModel import WYJ_CNN
import os
from PIL import Image
import numpy as np
from torchvision import transforms as T
from torch.utils.data import DataLoader
transform = T.Compose([#HWC怎么变为CHW?
T.Resize(224), # 缩放图片(Image)，保持长宽比不变，最短边为224像素
T.CenterCrop(224), # 从图片中间切出224*224的图片
T.ToTensor(), # 将图片(Image)转成Tensor，归一化至[0, 1]
T.Normalize(mean=[.5, .5, .5], std=[.5, .5, .5]) # 标准化至[-1, 1]，规定均值和标准差
])
class DogCat(data.Dataset):
def __init__(self, root, transforms=None):
imgs = os.listdir(root)
# 所有图片的绝对路径
# 这里不实际加载图片，只是指定路径，当调用__getitem__时才会真正读图片
self.imgs = [os.path.join(root, img) for img in imgs]
self.transforms = transforms
def __getitem__(self, index):
img_path = self.imgs[index]
label = 1 if 'dog' in img_path.split('/')[-1] else 0
data = Image.open(img_path)
if self.transforms:
data = self.transforms(data)
return data, label
def __len__(self):
return len(self.imgs)
net = WYJ_CNN()
CalcLoss = nn.CrossEntropyLoss()#loss用交叉熵来算
optimizer = torch.optim.SGD(net.parameters(), lr=0.05)#lr是学习率
trainset = DogCat('./train', transforms=transform)
trainloader = DataLoader(trainset, batch_size=100, shuffle=True, num_workers=0, drop_last=False)
testset = DogCat('./test', transforms=transform)
testloader = DataLoader(testset, batch_size=10, shuffle=False, num_workers=0, drop_last=False)
for cnt in range(10):
sum=0
for batch_datas, batch_labels in trainloader:
sum=sum+1
outputs = net(batch_datas)
loss = CalcLoss(outputs, batch_labels)
optimizer.zero_grad() # 注意清空优化器的梯度，防止累计
loss.backward()
optimizer.step()
print("training",cnt," ",sum*100)
if np.isnan(np.isnan(outputs.detach().numpy().any())):
print("NAN error!")
break;
torch.save(net, "my_cnn.nn")
net = torch.load("my_cnn.nn")
id = [];
label = [];
i=0
for batch_datas, batch_labels in testloader:
print(batch_datas.shape)
outputs = net(batch_datas)
id.extend(np.arange(i*10+1,(i+1)*10+1))
label.extend(outputs[:,1].detach().numpy())
i = i + 1
print("testing", i*10)
myAns = pd.DataFrame({'id': id, 'label': label})
myAns.to_csv("myAns.csv", index=False, sep=',')
放一个飞桨版本
改出来了，踩的坑不少，到时候另开一个总结一下
用的是resnet50，训练效果挺好的，排到了中间位置
import math
import paddle
import paddle.nn as nn
import pandas as pd
import numpy as np
from myModel import WYJ_CNN
import os
from PIL import Image
import numpy as np
import paddle.vision.transforms as T
from paddle.io import DataLoader
from IPython.display import display
transform = T.Compose([
T.Resize(224),
T.RandomResizedCrop(224,scale=(0.6,1.0),ratio=(0.8,1.0)),
T.RandomHorizontalFlip(),
T.ColorJitter(brightness=0.5, contrast=0, saturation=0, hue=0),
T.ColorJitter(brightness=0, contrast=0.5, saturation=0, hue=0),
T.ToTensor(),
T.Normalize(mean=[.5, .5, .5], std=[.5, .5, .5])
])
class DogCat(paddle.io.Dataset):
def __init__(self, root, transforms=None):
imgs = os.listdir(root)
# 所有图片的绝对路径
# 这里不实际加载图片，只是指定路径，当调用__getitem__时才会真正读图片
self.imgs = [os.path.join(root, img) for img in imgs]
self.transforms = transforms
def __getitem__(self, index):
img_path = self.imgs[index]
label = 1 if 'dog' in img_path.split('/')[-1] else 0
data = Image.open(img_path)
if self.transforms:
data = self.transforms(data)
return data, label ,img_path
def __len__(self):
return len(self.imgs)
paddle.device.set_device("gpu:0")
net = paddle.vision.models.resnet50(pretrained=True)
net.fc = nn.Sequential(nn.Linear(2048,2))
CalcLoss = nn.CrossEntropyLoss()#loss用交叉熵来算
optimizer = paddle.optimizer.SGD(learning_rate =0.01 , parameters=net.parameters() )#lr是学习率
net_state_dict=paddle.load('net.pdparams')
net.set_state_dict(net_state_dict)
optimizer_state_dict=paddle.load('optimizer.pdopt')
optimizer.set_state_dict(optimizer_state_dict)
trainset = DogCat('./train', transforms=transform)
a_list = paddle.io.random_split(trainset, [int(len(trainset)*0.8), len(trainset)-int(len(trainset)*0.8)])
trainloader0 = DataLoader(a_list[0], batch_size=32, shuffle=True, num_workers=0, drop_last=False)
trainloader1 = DataLoader(a_list[1], batch_size=32, shuffle=True, num_workers=0, drop_last=False)
testset = DogCat('./test', transforms=transform)
testloader = DataLoader(testset, batch_size=10, shuffle=False, num_workers=0, drop_last=False)
for cnt in range(3):
sum=0
for batch_datas, batch_labels ,name in trainloader0:
sum=sum+1
outputs = net(batch_datas)
loss = CalcLoss(outputs, batch_labels)
optimizer.clear_grad() # 注意清空优化器的梯度，防止累计
loss.backward()
optimizer.step()
#print(name[0],batch_labels[0])
if sum%100==0:
print("training",cnt," ",sum*32)
if np.isnan(np.isnan(outputs.detach().numpy().any())):
print("NAN error!")
break;
total_loss = 0 # 保存这次测试总的loss
with paddle.no_grad(): # 下面不需要反向传播，所以不需要自动求导
for batch_datas, batch_labels ,name in trainloader1:
outputs = net(batch_datas)
loss = CalcLoss(outputs, batch_labels)
total_loss += loss # 累计误差
print("第{}次训练的Loss:{}".format(cnt + 1, total_loss))
paddle.save(net.state_dict(),'net.pdparams')
paddle.save(optimizer.state_dict(),'optimizer.pdopt')
id = [];
label = [];
ans_dict={}
i=0
for batch_datas, batch_labels ,name in testloader:
#print(batch_datas.shape)
outputs = net(batch_datas)
m = paddle.nn.Softmax()
outputs=m(outputs)
for j in range(len(outputs)):
num = ''.join([x for x in name[j] if x.isdigit()])
ans_dict[num]=outputs[j][1].item()
i = i + 1
if i%100==0:
print("testing", i*10)
for i in range(len(ans_dict)):
id.append(i+1)
label.append(ans_dict.get(str(i+1)))
myAns = pd.DataFrame({'id': id, 'label': label})
myAns.to_csv("myAns.csv", index=False, sep=',')
