Kaggle_Digit Recognizer_DCNN

Kaggle-Digit Recognizer-DCNN

date:2019-01-22

參考網頁

注意點

BN層編譯部分:
Batchnormalization的axis參數在卷積層和池化層應爲1（4維數據）
Dense層保持默認即可
但是對於keras 2,2,4和tensorflow-gpu 1.12會出現錯誤
可參照鏈接對對應文件進行修改,文件修改如下：

if ndim(mean) > 1:
	#mean = tf.reshape(mean, (-1))
	#added by zwg 20190121
	mean = tf.reshape(mean, [-1])
if ndim(var) > 1:
	#var = tf.reshape(var, (-1))
	#added by zwg 20190121
	var = tf.reshape(var, [-1])
if beta is None:
	beta = zeros_like(mean)
elif ndim(beta) > 1:
	#beta = tf.reshape(beta, (-1))
	#added by zwg 20190121
	beta = tf.reshape(beta, [-1])
if gamma is None:
	gamma = ones_like(mean)
elif ndim(gamma) > 1:
	#gamma = tf.reshape(gamma, (-1))
	#added by zwg 20190121
	gamma = tf.reshape(gamma, [-1])

Import all required libraries

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
XS
import matplotlib.pyplot as plt
%matplotlib inline

from keras.models import Sequential
from keras.layers import Dense , Dropout , Lambda, Flatten
from keras.optimizers import Adam ,RMSprop
from sklearn.model_selection import train_test_split
from keras import  backend as K
from keras.preprocessing.image import ImageDataGenerator

# Input data files are available in the "input/" directory.
# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory

from subprocess import check_output
print(check_output(["ls", "../input"]).decode("utf8"))

Using TensorFlow backend.


sample_submission.csv
test.csv
train.csv

Load Train and Test data

# create the training & test sets, skipping the header row with [1:]
train = pd.read_csv("../input/train.csv")
print(train.shape)
train.head()

(42000, 785)

	label	pixel0	pixel1	pixel2	pixel3	pixel4	pixel5	pixel6	pixel7	pixel8	...	pixel774	pixel775	pixel776	pixel777	pixel778	pixel779	pixel780	pixel781	pixel782	pixel783
0	1	0	0	0	0	0	0	0	0	0	...	0	0	0	0	0	0	0	0	0	0
1	0	0	0	0	0	0	0	0	0	0	...	0	0	0	0	0	0	0	0	0	0
2	1	0	0	0	0	0	0	0	0	0	...	0	0	0	0	0	0	0	0	0	0
3	4	0	0	0	0	0	0	0	0	0	...	0	0	0	0	0	0	0	0	0	0
4	0	0	0	0	0	0	0	0	0	0	...	0	0	0	0	0	0	0	0	0	0

5 rows × 785 columns

test= pd.read_csv("../input/test.csv")
print(test.shape)
test.head()

(28000, 784)

	pixel0	pixel1	pixel2	pixel3	pixel4	pixel5	pixel6	pixel7	pixel8	pixel9	...	pixel774	pixel775	pixel776	pixel777	pixel778	pixel779	pixel780	pixel781	pixel782	pixel783
0	0	0	0	0	0	0	0	0	0	0	...	0	0	0	0	0	0	0	0	0	0
1	0	0	0	0	0	0	0	0	0	0	...	0	0	0	0	0	0	0	0	0	0
2	0	0	0	0	0	0	0	0	0	0	...	0	0	0	0	0	0	0	0	0	0
3	0	0	0	0	0	0	0	0	0	0	...	0	0	0	0	0	0	0	0	0	0
4	0	0	0	0	0	0	0	0	0	0	...	0	0	0	0	0	0	0	0	0	0

5 rows × 784 columns

X_train = (train.iloc[:,1:].values).astype('float32') # all pixel values
y_train = train.iloc[:,0].values.astype('int32') # only labels i.e targets digits
X_test = test.values.astype('float32')
X_train

array([[0., 0., 0., ..., 0., 0., 0.],
       [0., 0., 0., ..., 0., 0., 0.],
       [0., 0., 0., ..., 0., 0., 0.],
       ...,
       [0., 0., 0., ..., 0., 0., 0.],
       [0., 0., 0., ..., 0., 0., 0.],
       [0., 0., 0., ..., 0., 0., 0.]], dtype=float32)

y_train

array([1, 0, 1, ..., 7, 6, 9], dtype=int32)

Data Visualization

X_train = X_train.reshape(X_train.shape[0], 28, 28)

for i in range(6, 9):
    plt.subplot(330 + (i+1))
    plt.imshow(X_train[i], cmap=plt.get_cmap('gray'))
    plt.title(y_train[i]);

#expand 1 more dimention as 1 for colour channel gray
X_train = X_train.reshape(X_train.shape[0], 28, 28,1)
X_train.shape

(42000, 28, 28, 1)

X_test = X_test.reshape(X_test.shape[0], 28, 28,1)
X_test.shape

(28000, 28, 28, 1)

Preprocessing the digit images

Feature Standardization

mean_px = X_train.mean().astype(np.float32)
std_px = X_train.std().astype(np.float32)

def standardize(x): 
    return (x-mean_px)/std_px

One Hot encoding for labels

from keras.utils.np_utils import to_categorical
y_train= to_categorical(y_train)
num_classes = y_train.shape[1]
plt.title(y_train[9])
plt.plot(y_train[9])
plt.xticks(range(10));

/usr/local/lib/python3.5/dist-packages/matplotlib/text.py:1191: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison
  if s != self._text:

Designing Neural Network Architecture

# fix random seed for reproducibility
seed = 43
np.random.seed(seed)

Linear Model

#Lambda layer:sum, average, exponentiation etc.
#Flatten will transform input into 1D array
#Dense: Full-connect layer
from keras.models import  Sequential
from keras.layers.core import  Lambda , Dense, Flatten, Dropout
from keras.callbacks import EarlyStopping
from keras.layers import BatchNormalization, Convolution2D , MaxPooling2D

model= Sequential()
model.add(Lambda(standardize,input_shape=(28,28,1)))
model.add(Flatten())
model.add(Dense(10, activation='softmax'))
print("input shape ",model.input_shape)
print("output shape ",model.output_shape)

input shape  (None, 28, 28, 1)
output shape  (None, 10)

Compile network

# Add a loss layer 
# Add an optimizer
# Add a metrics to monitor the performance of the network
from keras.optimizers import RMSprop
model.compile(optimizer=RMSprop(lr =0.001),
             loss='categorical_crossentropy',
             metrics=['accuracy'])

from keras.preprocessing import image
gen = image.ImageDataGenerator()

Cross Validation

from sklearn.model_selection import train_test_split
X = X_train
y = y_train
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.10, random_state=42)
batches = gen.flow(X_train, y_train, batch_size=64)
val_batches = gen.flow(X_val, y_val, batch_size=64)

history = model.fit_generator(generator=batches, steps_per_epoch=batches.n, epochs=1,
                              validation_data=val_batches, validation_steps=val_batches.n)

Epoch 1/1
37800/37800 [==============================] - 198s 5ms/step - loss: 0.2401 - acc: 0.9342 - val_loss: 0.3306 - val_acc: 0.9112

history_dict = history.history
history_dict.keys()

dict_keys(['val_acc', 'loss', 'val_loss', 'acc'])

import matplotlib.pyplot as plt
%matplotlib inline
loss_values = history_dict['loss']
val_loss_values = history_dict['val_loss']
epochs = range(1, len(loss_values) + 1)

# "bo" is for "blue dot"
plt.plot(epochs, loss_values, 'bo')
# "b+" is for "blue crosses"
plt.plot(epochs, val_loss_values, 'b+')
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.show()

plt.clf() # Clear figure
acc_values = history_dict['acc']
val_acc_values = history_dict['val_acc']

plt.plot(epochs, acc_values, 'bo')
plt.plot(epochs, val_acc_values, 'b+')
plt.xlabel('Epochs')
plt.ylabel('Accuracy')
plt.show()

Fully Connected Model

# 構建兩層全連接層：
# 前層負激活數據（Relu）
# 後層輸出分類
def get_fc_model():
    model = Sequential([
        Lambda(standardize, input_shape=(28,28,1)), # standardize
        Flatten(), # Change data into 1-d array
        Dense(512, activation='relu'),
        Dense(10, activation='softmax')
    ])
    model.compile(optimizer='Adam', loss='categorical_crossentropy',
                 metrics=['accuracy'])
    return model

fc = get_fc_model()
fc.optimizer.lr=0.01

# history = fc.fit_generator(generator=batches, steps_per_epoch=batches.n,
#                           epochs=1, validation_data=val_batches, 
#                           validation_steps=val_batches.n)

Convolutional Neural Network

from keras.layers import Convolution2D, MaxPooling2D

def get_cnn_model():
    model = Sequential([
        Lambda(standardize, input_shape=(28,28,1)),
        Convolution2D(32, (3,3), activation='relu'),
        Convolution2D(32, (3,3), activation='relu'),
        MaxPooling2D(),
        Convolution2D(64, (3,3), activation='relu'),
        Convolution2D(64, (3,3), activation='relu'),
        MaxPooling2D(),
        Flatten(),
        Dense(512, activation='relu'),
        Dense(10, activation='softmax')
    ])
    model.compile(Adam(), loss='categorical_crossentropy',
                  metrics=['accuracy'])
    return model

model = get_cnn_model()
model.optimizer.lr = 0.01

# history = model.fit_generator(generator=batches,steps_per_epoch=batches.n,
#                              epochs=1, validation_data=val_batches,
#                              validation_steps = val_batches.n
#                             )

Data Augmentation

# Different data aumentation techniques:
# Cropping
# Rotating
# Scaling
# Translating
# Flipping
# Adding Gaussian noise to input images etc.
gen = ImageDataGenerator(rotation_range=8, width_shift_range=0.08,
                        shear_range=0.3, height_shift_range=0.08,
                        zoom_range=0.08)
batches = gen.flow(X_train, y_train, batch_size=64)
val_batches = gen.flow(X_val, y_val, batch_size=64)

model.optimizer.lr = 0.001
# history = model.fit_generator(generator=batches,
#                              steps_per_epoch=batches.n, epochs=1,
#                             validation_data=val_batches,
#                             validation_steps=val_batches.n)

Adding Batch Normalization

from keras.layers.normalization import BatchNormalization

def get_bn_model():
    model = Sequential([
        Lambda(standardize, input_shape=(28,28,1)),
        Convolution2D(32,(3,3), activation='relu'),
        BatchNormalization(axis=1),
        Convolution2D(32,(3,3), activation='relu'),
        MaxPooling2D(),
        BatchNormalization(axis=1),
        Convolution2D(64,(3,3), activation='relu'),
        BatchNormalization(axis=1),
        Convolution2D(64,(3,3), activation='relu'),
        MaxPooling2D(),
        Flatten(),
        BatchNormalization(),
        Dense(512, activation='relu'),
        BatchNormalization(),
        Dense(10, activation='softmax')
        ])
    model.compile(Adam(), loss='categorical_crossentropy', metrics=['accuracy'])
    return model

model= get_bn_model()
model.optimizer.lr=0.01
history=model.fit_generator(generator=batches, steps_per_epoch=batches.n, epochs=1,validation_data=val_batches, validation_steps=val_batches.n)

Epoch 1/1
37800/37800 [==============================] - 2453s 65ms/step - loss: 0.0350 - acc: 0.9902 - val_loss: 0.0485 - val_acc: 0.9898

Submitting Predictions to Kaggle

model.optimizer.lr = 0.01
gen = image.ImageDataGenerator()
batches = gen.flow(X, y, batch_size=64)
history = model.fit_generator(generator=batches, steps_per_epoch=batches.n, epochs=3)

Epoch 1/3
42000/42000 [==============================] - 2573s 61ms/step - loss: 0.0128 - acc: 0.9980
Epoch 2/3
42000/42000 [==============================] - 2224s 53ms/step - loss: 0.0195 - acc: 0.9984
Epoch 3/3
42000/42000 [==============================] - 2937s 70ms/step - loss: 0.0241 - acc: 0.9983

predictions = model.predict_classes(X_test, verbose=0)
submissions = pd.DataFrame({"ImageId": list(range(1, len(predictions)+1)),
                           "Label": predictions})
submissions.to_csv("DR.csv", index=False, header=True)