Cifar-10:
import pickle
import glob
import numpy as np
def Dataloader():
data_list = glob.glob("data_batch_*")
for data in data_list:
data = pickle.load(open(data, 'rb'), encoding='bytes')
labels, data, filenames = data[b'labels'], data[b'data'], data[b'filenames']
labels, data = map(np.array, [labels, data])
try:
Data = np.r_[Data, data]
Labels = np.r_[Labels, labels]
except:
Data = data
Labels = labels
np.save("data/data.npy", Data)
np.save("data/label.npy", Labels)
if __name__ == "__main__":
Dataloader()
Mnist:
import numpy as np
import struct
def loadImageSet(filename):
binfile = open(filename, 'rb') # 讀取二進制文件
buffers = binfile.read()
head = struct.unpack_from('>IIII', buffers, 0) # 取前4個整數,返回一個元組
offset = struct.calcsize('>IIII') # 定位到data開始的位置
imgNum = head[1]
width = head[2]
height = head[3]
bits = imgNum * width * height # data一共有60000*28*28個像素值
bitsString = '>' + str(bits) + 'B' # fmt格式:'>47040000B'
imgs = struct.unpack_from(bitsString, buffers, offset) # 取data數據,返回一個元組
binfile.close()
imgs = np.reshape(imgs, [imgNum, width * height]) # reshape爲[60000,784]型數組
return imgs,head
def loadLabelSet(filename):
binfile = open(filename, 'rb') # 讀二進制文件
buffers = binfile.read()
head = struct.unpack_from('>II', buffers, 0) # 取label文件前2個整形數
labelNum = head[1]
offset = struct.calcsize('>II') # 定位到label數據開始的位置
numString = '>' + str(labelNum) + "B" # fmt格式:'>60000B'
labels = struct.unpack_from(numString, buffers, offset) # 取label數據
binfile.close()
labels = np.reshape(labels, [labelNum]) # 轉型爲列表(一維數組)
return labels,head
if __name__ == "__main__":
file1= './train-images.idx3-ubyte'
file2= './train-labels.idx1-ubyte'
imgs,data_head = loadImageSet(file1)
labels,labels_head = loadLabelSet(file2)
np.save("data/data.npy", imgs)
np.save("data/label.npy", labels)