MXNet：读取图像数据集Fasion-MNIST

质量声明：原创文章，内容质量问题请评论吐槽。如对您产生干扰，可私信删除。
主要参考：python 读取 MNIST 数据集并解析为图片文件

---

---

摘要： MXNet实践: 以Fasion-MNIST为例，下载并读取数据集

---

图像分类数据集中最常⽤的是⼿写数字识别数据集MNIST ，但⼤部分模型在MNIST上的分类精度都超过了95%，选用图像内容更加复杂的数据集可以更直观地观察算法之间的差异，Fashion-MNIST克隆了MNIST的所有外在特征，⼀共包括了10个类别，分别为t-shirt（T恤）、 trouser（裤⼦）、 pullover（套衫）、dress（连⾐裙）、 coat（外套）、 sandal（凉鞋）、 shirt（衬衫）、 sneaker（运动鞋）、 bag（包）和ankle boot（短靴）

下载数据集

• GitHub ： https://github.com/zalandoresearch/fashion-mnist

• Loading data with machine learning libraries

• MXNet 自带下载工具:

  from mxnet.gluon import data as gdata
  
  root='/content/drive/My Drive/Colab/datasets/fashion_mnist'
  mnist_train = gdata.vision.FashionMNIST(root, train=True)
  mnist_test = gdata.vision.FashionMNIST(root, train=False)
  

下载后的压缩文件:

• train-images-idx3-ubyte.gz 训练集图片 - 55000 张 训练图片, 5000 张 验证图片
• train-labels-idx1-ubyte.gz 训练集标签
• t10k-images-idx3-ubyte.gz 测试集图片 - 10000 张 图片
• t10k-labels-idx1-ubyte.gz 测试集标签

解压

gzip -d XXX

解压后的文件夹：

• train-images-idx3-ubyte
• train-labels-idx1-ubyte
• t10k-images-idx3-ubyte
• t10k-labels-idx1-ubyte

解析为图片文件，按标签存放

• 最终的文件结构
  

• MNIST转换为图片的python代码

import struct
import numpy as np
import os
import cv2

def decode_idx3_ubyte(idx3_ubyte_file):
    with open(idx3_ubyte_file, 'rb') as f:
        print('解析文件：', idx3_ubyte_file)
        fb_data = f.read()
    offset = 0
    fmt_header = '>iiii'    # 以大端法读取4个 unsinged int32
    magic_number, num_images, num_rows, num_cols = struct.unpack_from(fmt_header, fb_data, offset)
    print('魔数：{}，图片数：{}'.format(magic_number, num_images))
    offset += struct.calcsize(fmt_header)
    fmt_image = '>' + str(num_rows * num_cols) + 'B'
    images = np.empty((num_images, num_rows, num_cols))
    for i in range(num_images):
        im = struct.unpack_from(fmt_image, fb_data, offset)
        images[i] = np.array(im).reshape((num_rows, num_cols))
        offset += struct.calcsize(fmt_image)
    return images

def decode_idx1_ubyte(idx1_ubyte_file):
    with open(idx1_ubyte_file, 'rb') as f:
        print('解析文件：', idx1_ubyte_file)
        fb_data = f.read()
    offset = 0
    fmt_header = '>ii'  # 以大端法读取两个 unsinged int32
    magic_number, label_num = struct.unpack_from(fmt_header, fb_data, offset)
    print('魔数：{}，标签数：{}'.format(magic_number, label_num))
    offset += struct.calcsize(fmt_header)
    labels = []
    fmt_label = '>B'    # 每次读取一个 byte
    for i in range(label_num):
        labels.append(struct.unpack_from(fmt_label, fb_data, offset)[0])
        offset += struct.calcsize(fmt_label)
    return labels

def check_folder(folder):
    if not os.path.exists(folder):
        os.mkdir(folder)
        print(folder)
    else:
        if not os.path.isdir(folder):
            os.mkdir(folder)

def export_img(exp_dir, img_ubyte, lable_ubyte):
    check_folder(exp_dir)
    images = decode_idx3_ubyte(img_ubyte)
    labels = decode_idx1_ubyte(lable_ubyte)
    nums = len(labels)
    for i in range(nums):
        img_dir = os.path.join(exp_dir, str(labels[i]))
        check_folder(img_dir)
        img_file = os.path.join(img_dir, str(i)+'.png')
        imarr = images[i]
        cv2.imwrite(img_file, imarr)


def parser_mnist_data(data_dir):
    train_dir = os.path.join(data_dir, 'train')
    train_img_ubyte = os.path.join(data_dir, 'train-images-idx3-ubyte')
    train_label_ubyte = os.path.join(data_dir, 'train-labels-idx1-ubyte')
    export_img(train_dir, train_img_ubyte, train_label_ubyte)
    test_dir = os.path.join(data_dir, 'test')
    test_img_ubyte = os.path.join(data_dir, 't10k-images-idx3-ubyte')
    test_label_ubyte = os.path.join(data_dir, 't10k-labels-idx1-ubyte')
    export_img(test_dir, test_img_ubyte, test_label_ubyte)

if __name__ == '__main__':
    data_dir = os.path.join("fashion-mnist/")
    data_dir = os.path.expanduser(data_dir) 
    parser_mnist_data(data_dir)

• 此时得到的图片文件夹是以标签0,1,...,9命名的，可选择进行如下转换

import os

text_labels = ['t-shirt', 'trouser', 'pullover', 'dress', 'coat',
               'sandal', 'shirt', 'sneaker', 'bag', 'ankle boot']
# 训练集
path = "fashion-mnist/train"
dirs = os.listdir(path)
for oldname in dirs:
    newname = text_labels[int(oldname)]
    os.rename(oldname, newname)
# 测试集
path = "fashion-mnist/test"
dirs = os.listdir(path)
for oldname in dirs:
    newname = text_labels[int(oldname)]
    os.rename(oldname, newname)

读入数据集

dataset_dir = "~/.mxnet/datasets/fashion-mnist"
train_imgs = gdata.vision.ImageFolderDataset(os.path.join(dataset_dir, 'train'))
test_imgs = gdata.vision.ImageFolderDataset(os.path.join(dataset_dir, 'test'))

此时train_imgs和test_imgs类型均为datasets.ImageFolderDataset, 标签已被转化为数值型. 通过synsets属性可查看标签对应的类别名,如:

trainset.synsets = [‘ankle boot’, ‘bag’, ‘coat’, ‘dress’, ‘pullover’, ‘sandal’, ‘shirt’, ‘sneaker’, ‘t-shirt’, ‘trouser’]

数据增广

transformer = gdata.vision.transforms.Compose([  
    gdata.vision.transforms.RandomFlipLeftRight(),
    gdata.vision.transforms.ToTensor(),
    gdata.vision.transforms.Normalize(),])
trainset = trainset.transform_first(transformer)
testset = testset.transform_first(transformer)

构建小批量数据数据生成器

batch_size = 256
num_workers = 0 if sys.platform.startswith('win32') else 4
train_iter = gdata.DataLoader(trainset, batch_size, shuffle=True, num_workers=num_workers)
test_iter = gdata.DataLoader(testset, batch_size, shuffle=True, num_workers=num_workers)