如何将notMNIST转成MNIST格式

相信了解机器学习的对MNIST不会陌生，Google的工程师Yaroslav Bulatov 创建了notMNIST，它和MNIST类似，图像28x28，也有10个Label（A-J）。

在Tensorflow中已经封装好了读取MNIST数据集的函数 read_data_sets()，

from tensorflow.contrib.learn.python.learn.datasets.mnist import read_data_sets
mnist = read_data_sets("data", one_hot=True, reshape=False, validation_size=0)

但是由于notMNIST的格式和MNIST的格式不是完全相同，所以基于tensorflow创建的针对MNIST的模型并不能直接读取notMNIST的图片。

Github上有人编写了格式转换代码（https://github.com/davidflanagan/notMNIST-to-MNIST），转换后可直接使用read_data_sets()完成读取，这样模型代码的变动就不会很大。本文是对在阅览完代码后所做的注释。

import numpy, imageio, glob, sys, os, random
#Imageio 提供简单的用于读写图像数据的接口
#glob 功能类似于文件搜索，查找文件只用到三个匹配符：”*”, “?”, “[]”。”*”匹配0个或多个字符；”?”匹配单个字符；”[]”匹配指定范围内的字符，如：[0-9]匹配数字。
def get_labels_and_files(folder, number):
  # Make a list of lists of files for each label
  filelists = []
  for label in range(0,10):
    filelist = []
    filelists.append(filelist);
    dirname = os.path.join(folder, chr(ord('A') + label))
    #label实际为0-9，chr(ord('A') + label)返回A-J
    #拼接路径dirname=folder/[A-J]
    for file in os.listdir(dirname):
    #返回一个装满当前路径中文件名的list
      if (file.endswith('.png')):
        fullname = os.path.join(dirname, file)
        if (os.path.getsize(fullname) > 0):
          filelist.append(fullname)
        else:
          print('file ' + fullname + ' is empty')
    # sort each list of files so they start off in the same order
    # regardless of how the order the OS returns them in
    filelist.sort()

  # Take the specified number of items for each label and
  # build them into an array of (label, filename) pairs
  # Since we seeded the RNG, we should get the same sample each run
  labelsAndFiles = []
  for label in range(0,10):
    filelist = random.sample(filelists[label], number)
    #随机采样 设定个数的文件名
    for filename in filelist:
      labelsAndFiles.append((label, filename))
      #Python的元组与列表类似，不同之处在于元组的元素不能修改。元组使用小括号，列表使用方括号。
  return labelsAndFiles

def make_arrays(labelsAndFiles):
  images = []
  labels = []
  for i in range(0, len(labelsAndFiles)):

    # display progress, since this can take a while
    if (i % 100 == 0):
      sys.stdout.write("\r%d%% complete" % ((i * 100)/len(labelsAndFiles)))
      #\r 返回第一个指针，覆盖前面的内容
      sys.stdout.flush()

    filename = labelsAndFiles[i][1]
    try:
      image = imageio.imread(filename)
      images.append(image)
      labels.append(labelsAndFiles[i][0])
    except:
      # If this happens we won't have the requested number
      print("\nCan't read image file " + filename)

  count = len(images)
  imagedata = numpy.zeros((count,28,28), dtype=numpy.uint8)
  labeldata = numpy.zeros(count, dtype=numpy.uint8)
  for i in range(0, len(labelsAndFiles)):
    imagedata[i] = images[i]
    labeldata[i] = labels[i]
  print("\n")
  return imagedata, labeldata

def write_labeldata(labeldata, outputfile):
  header = numpy.array([0x0801, len(labeldata)], dtype='>i4')
  with open(outputfile, "wb") as f:
  #以二进制写模式打开
  #这里使用了 with 语句，不管在处理文件过程中是否发生异常，都能保证 with 语句执行完毕后已经关闭了打开的文件句柄
    f.write(header.tobytes())
    #写入二进制数
    f.write(labeldata.tobytes())

def write_imagedata(imagedata, outputfile):
  header = numpy.array([0x0803, len(imagedata), 28, 28], dtype='>i4')
  with open(outputfile, "wb") as f:
    f.write(header.tobytes())
    f.write(imagedata.tobytes())
    


def main(argv):
  # Uncomment the line below if you want to seed the random
  # number generator in the same way I did to produce the
  # specific data files in this repo.
  # random.seed(int("notMNIST", 36))
  #当我们设置相同的seed，每次生成的随机数相同。如果不设置seed，则每次会生成不同的随机数

  labelsAndFiles = get_labels_and_files(argv[1], int(argv[2]))
  #随机排序
  random.shuffle(labelsAndFiles)
  
  imagedata, labeldata = make_arrays(labelsAndFiles)
  write_labeldata(labeldata, argv[3])
  write_imagedata(imagedata, argv[4])

if __name__=='__main__':
#Make a script both importable and executable
#如果我们是直接执行某个.py文件的时候，该文件中那么”__name__ == '__main__'“是True
#如果被别的模块import，__name__！='__main__'，这样main()就不会执行

  main(sys.argv)

使用方法

下载解压notMNIST：

curl -o notMNIST_small.tar.gz http://yaroslavvb.com/upload/notMNIST/notMNIST_small.tar.gz
curl -o notMNIST_large.tar.gz http://yaroslavvb.com/upload/notMNIST/notMNIST_large.tar.gz
tar xzf notMNIST_small.tar.gz
tar xzf notMNIST_large.tar.gz

运行转换代码：

python convert_to_mnist_format.py notMNIST_small 1000 data/t10k-labels-idx1-ubyte data/t10k-images-idx3-ubyte
python convert_to_mnist_format.py notMNIST_large 6000 data/train-labels-idx1-ubyte data/train-images-idx3-ubyte
gzip data/*ubyte