如何將notMNIST轉成MNIST格式

相信了解機器學習的對MNIST不會陌生，Google的工程師Yaroslav Bulatov 創建了notMNIST，它和MNIST類似，圖像28x28，也有10個Label（A-J）。

在Tensorflow中已經封裝好了讀取MNIST數據集的函數 read_data_sets()，

from tensorflow.contrib.learn.python.learn.datasets.mnist import read_data_sets
mnist = read_data_sets("data", one_hot=True, reshape=False, validation_size=0)

但是由於notMNIST的格式和MNIST的格式不是完全相同，所以基於tensorflow創建的針對MNIST的模型並不能直接讀取notMNIST的圖片。

Github上有人編寫了格式轉換代碼（https://github.com/davidflanagan/notMNIST-to-MNIST），轉換后可直接使用read_data_sets()完成讀取，這樣模型代碼的變動就不會很大。本文是對在閱覽完代碼后所做的注釋。

import numpy, imageio, glob, sys, os, random
#Imageio 提供簡單的用於讀寫圖像數據的接口
#glob 功能類似於文件搜索，查找文件只用到三個匹配符：”*”, “?”, “[]”。”*”匹配0個或多個字符；”?”匹配單個字符；”[]”匹配指定范圍內的字符，如：[0-9]匹配數字。
def get_labels_and_files(folder, number):
  # Make a list of lists of files for each label
  filelists = []
  for label in range(0,10):
    filelist = []
    filelists.append(filelist);
    dirname = os.path.join(folder, chr(ord('A') + label))
    #label實際為0-9，chr(ord('A') + label)返回A-J
    #拼接路徑dirname=folder/[A-J]
    for file in os.listdir(dirname):
    #返回一個裝滿當前路徑中文件名的list
      if (file.endswith('.png')):
        fullname = os.path.join(dirname, file)
        if (os.path.getsize(fullname) > 0):
          filelist.append(fullname)
        else:
          print('file ' + fullname + ' is empty')
    # sort each list of files so they start off in the same order
    # regardless of how the order the OS returns them in
    filelist.sort()

  # Take the specified number of items for each label and
  # build them into an array of (label, filename) pairs
  # Since we seeded the RNG, we should get the same sample each run
  labelsAndFiles = []
  for label in range(0,10):
    filelist = random.sample(filelists[label], number)
    #隨機采樣 設定個數的文件名
    for filename in filelist:
      labelsAndFiles.append((label, filename))
      #Python的元組與列表類似，不同之處在於元組的元素不能修改。元組使用小括號，列表使用方括號。
  return labelsAndFiles

def make_arrays(labelsAndFiles):
  images = []
  labels = []
  for i in range(0, len(labelsAndFiles)):

    # display progress, since this can take a while
    if (i % 100 == 0):
      sys.stdout.write("\r%d%% complete" % ((i * 100)/len(labelsAndFiles)))
      #\r 返回第一個指針，覆蓋前面的內容
      sys.stdout.flush()

    filename = labelsAndFiles[i][1]
    try:
      image = imageio.imread(filename)
      images.append(image)
      labels.append(labelsAndFiles[i][0])
    except:
      # If this happens we won't have the requested number
      print("\nCan't read image file " + filename)

  count = len(images)
  imagedata = numpy.zeros((count,28,28), dtype=numpy.uint8)
  labeldata = numpy.zeros(count, dtype=numpy.uint8)
  for i in range(0, len(labelsAndFiles)):
    imagedata[i] = images[i]
    labeldata[i] = labels[i]
  print("\n")
  return imagedata, labeldata

def write_labeldata(labeldata, outputfile):
  header = numpy.array([0x0801, len(labeldata)], dtype='>i4')
  with open(outputfile, "wb") as f:
  #以二進制寫模式打開
  #這里使用了 with 語句，不管在處理文件過程中是否發生異常，都能保證 with 語句執行完畢后已經關閉了打開的文件句柄
    f.write(header.tobytes())
    #寫入二進制數
    f.write(labeldata.tobytes())

def write_imagedata(imagedata, outputfile):
  header = numpy.array([0x0803, len(imagedata), 28, 28], dtype='>i4')
  with open(outputfile, "wb") as f:
    f.write(header.tobytes())
    f.write(imagedata.tobytes())
    


def main(argv):
  # Uncomment the line below if you want to seed the random
  # number generator in the same way I did to produce the
  # specific data files in this repo.
  # random.seed(int("notMNIST", 36))
  #當我們設置相同的seed，每次生成的隨機數相同。如果不設置seed，則每次會生成不同的隨機數

  labelsAndFiles = get_labels_and_files(argv[1], int(argv[2]))
  #隨機排序
  random.shuffle(labelsAndFiles)
  
  imagedata, labeldata = make_arrays(labelsAndFiles)
  write_labeldata(labeldata, argv[3])
  write_imagedata(imagedata, argv[4])

if __name__=='__main__':
#Make a script both importable and executable
#如果我們是直接執行某個.py文件的時候，該文件中那么”__name__ == '__main__'“是True
#如果被別的模塊import，__name__！='__main__'，這樣main()就不會執行

  main(sys.argv)

使用方法

下載解壓notMNIST：

curl -o notMNIST_small.tar.gz http://yaroslavvb.com/upload/notMNIST/notMNIST_small.tar.gz
curl -o notMNIST_large.tar.gz http://yaroslavvb.com/upload/notMNIST/notMNIST_large.tar.gz
tar xzf notMNIST_small.tar.gz
tar xzf notMNIST_large.tar.gz

運行轉換代碼：

python convert_to_mnist_format.py notMNIST_small 1000 data/t10k-labels-idx1-ubyte data/t10k-images-idx3-ubyte
python convert_to_mnist_format.py notMNIST_large 6000 data/train-labels-idx1-ubyte data/train-images-idx3-ubyte
gzip data/*ubyte