第4.6.2節，我覺得我家破電腦確實到極限了，~~~
import tensorflow as tf
import tensorlayer as tl
from tensorlayer.layers import *
import numpy as np
import os, io, time
model_file_name = "./model_cifar10_tfrecord.ckpt"
resume = False # load model, resume from previous checkpoint?
X_train, y_train, X_test, y_test = tl.files.load_cifar10_dataset(shape=(-1, 32, 32, 3), plotable=False)
print('X_train.shape', X_train.shape) # (50000, 32, 32, 3)
print('y_train.shape', y_train.shape) # (50000,)
print('X_test.shape', X_test.shape) # (10000, 32, 32, 3)
print('y_test.shape', y_test.shape) # (10000,)
print('X %s y %s' % (X_test.dtype, y_test.dtype))
print('~~~~~~~~~data information~~~~~~~~~~~~')
def data_to_tfrecord(images, labels, filename):
if os.path.isfile(filename):
print("%s exists" % filename)
return
print("Converting data into %s ..." % filename)
cwd = os.getcwd()
writer = tf.python_io.TFRecordWriter(filename)
for index, img in enumerate(images):
img_raw = img.tobytes()
## Visualize a image
# tl.visualize.frame(np.asarray(img, dtype=np.uint8), second=1, saveable=False, name='frame', fig_idx=1236)
label = int(labels[index])
## Convert the bytes back to image as follow:
# image = Image.frombytes('RGB', (32, 32), img_raw)
# image = np.fromstring(img_raw, np.float32)
# image = image.reshape([32, 32, 3])
# tl.visualize.frame(np.asarray(image, dtype=np.uint8), second=1, saveable=False, name='frame', fig_idx=1236)
example = tf.train.Example(
features=tf.train.Features(
feature={
"label": tf.train.Feature(int64_list=tf.train.Int64List(value=[label])),
'img_raw': tf.train.Feature(bytes_list=tf.train.BytesList(value=[img_raw])),
}))
writer.write(example.SerializeToString())
writer.close()
## Save data into TFRecord files
data_to_tfrecord(images=X_train, labels=y_train, filename="train.cifar10")
data_to_tfrecord(images=X_test, labels=y_test, filename="test.cifar10")
def read_and_decode(filename, is_train=None):
filename_queue = tf.train.string_input_producer([filename])
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
features = tf.parse_single_example(
serialized_example, features={
'label': tf.FixedLenFeature([], tf.int64),
'img_raw': tf.FixedLenFeature([], tf.string),
})
img = tf.decode_raw(features['img_raw'], tf.float32)
img = tf.reshape(img, [32, 32, 3])
if is_train == True:
img = tf.random_crop(img, [24, 24, 3])
img = tf.image.random_flip_left_right(img)
img = tf.image.random_brightness(img, max_delta=63)
img = tf.image.random_contrast(img, lower=0.2, upper=1.8)
img = tf.image.per_image_standardization(img)
elif is_train == False:
img = tf.image.resize_image_with_crop_or_pad(img, 24, 24)
img = tf.image.per_image_standardization(img)
elif is_train == None:
img = img
label = tf.cast(features['label'], tf.int32)
return img, label
batch_size = 128
model_file_name = "./model_cifar10_advanced.ckpt"
resume = False # load model, resume from previous checkpoint?
with tf.device('/cpu:0'):
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
x_train_, y_train_ = read_and_decode("train.cifar10", True)
x_test_, y_test_ = read_and_decode("test.cifar10", False)
x_train_batch, y_train_batch = tf.train.shuffle_batch(
[x_train_, y_train_], batch_size=batch_size, capacity=2000, min_after_dequeue=1000, num_threads=32) # set the number of threads here
# for testing, uses batch instead of shuffle_batch
x_test_batch, y_test_batch = tf.train.batch([x_test_, y_test_], batch_size=batch_size, capacity=50000, num_threads=32)
def model(x_crop, y_, reuse):
W_init = tf.truncated_normal_initializer(stddev=5e-2)
W_init2 = tf.truncated_normal_initializer(stddev=0.04)
b_init2 = tf.constant_initializer(value=0.1)
with tf.variable_scope("model", reuse=reuse):
net = tl.layers.InputLayer(x_crop, name='input')
net = tl.layers.Conv2d(net, 64, (5, 5), (1, 1), act=tf.nn.relu, padding='SAME', W_init=W_init, name='cnn1')
net = tl.layers.SignLayer(net)
net = tl.layers.MaxPool2d(net, (3, 3), (2, 2), padding='SAME', name='pool1')
net = tl.layers.LocalResponseNormLayer(net, depth_radius=4, bias=1.0, alpha=0.001 / 9.0, beta=0.75, name='norm1')
net = tl.layers.BinaryConv2d(net, 64, (5, 5), (1, 1), act=tf.nn.relu, padding='SAME', W_init=W_init, name='cnn2')
net = tl.layers.LocalResponseNormLayer(net, depth_radius=4, bias=1.0, alpha=0.001 / 9.0, beta=0.75, name='norm2')
net = tl.layers.MaxPool2d(net, (3, 3), (2, 2), padding='SAME', name='pool2')
net = tl.layers.FlattenLayer(net, name='flatten') # output: (batch_size, 2304)
net = tl.layers.SignLayer(net)
net = tl.layers.BinaryDenseLayer(net, n_units=384, act=tf.nn.relu, W_init=W_init2, b_init=b_init2, name='d1relu') # output: (batch_size, 384)
net = tl.layers.SignLayer(net)
net = tl.layers.BinaryDenseLayer(net, n_units=192, act=tf.nn.relu, W_init=W_init2, b_init=b_init2, name='d2relu') # output: (batch_size, 192)
net = tl.layers.DenseLayer(net, n_units=10, act=tf.identity, W_init=W_init2, name='output') # output: (batch_size, 10)
y = net.outputs
ce = tl.cost.cross_entropy(y, y_, name='cost')
L2 = 0
for p in tl.layers.get_variables_with_name('relu/W', True, True):
L2 += tf.contrib.layers.l2_regularizer(0.004)(p)
cost = ce + L2
# correct_prediction = tf.equal(tf.argmax(tf.nn.softmax(y), 1), y_)
correct_prediction = tf.equal(tf.cast(tf.argmax(y, 1), tf.int32), y_)
acc = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
return net, cost, acc
def model_batch_norm(x_crop, y_, reuse, is_train):
""" Batch normalization should be placed before rectifier. """
W_init = tf.truncated_normal_initializer(stddev=5e-2)
W_init2 = tf.truncated_normal_initializer(stddev=0.04)
b_init2 = tf.constant_initializer(value=0.1)
with tf.variable_scope("model", reuse=reuse):
tl.layers.set_name_reuse(reuse)
net = InputLayer(x_crop, name='input')
net = tl.layers.Conv2d(net, 64, (5, 5), (1, 1), padding='SAME', W_init=W_init, b_init=None, name='cnn1')
net = tl.layers.BatchNormLayer(net, is_train, act=tf.nn.relu, name='batch1')
net = tl.layers.MaxPool2d(net, (3, 3), (2, 2), padding='SAME', name='pool1')
net = tl.layers.Conv2d(net, 64, (5, 5), (1, 1), padding='SAME', W_init=W_init, b_init=None, name='cnn2')
net = tl.layers.BatchNormLayer(net, is_train, act=tf.nn.relu, name='batch2')
net = tl.layers.MaxPool2d(net, (3, 3), (2, 2), padding='SAME', name='pool2')
net = tl.layers.FlattenLayer(net, name='flatten') # output: (batch_size, 2304)
net = tl.layers.DenseLayer(net, n_units=384, act=tf.nn.relu, W_init=W_init2, b_init=b_init2, name='d1relu') # output: (batch_size, 384)
net = tl.layers.DenseLayer(net, n_units=192, act=tf.nn.relu, W_init=W_init2, b_init=b_init2, name='d2relu') # output: (batch_size, 192)
net = tl.layers.DenseLayer(net, n_units=10, act=tf.identity, W_init=W_init2, name='output') # output: (batch_size, 10)
y = net.outputs
ce = tl.cost.cross_entropy(y, y_, name='cost')
L2 = 0
for p in tl.layers.get_variables_with_name('relu/W', True, True):
L2 += tf.contrib.layers.l2_regularizer(0.004)(p)
cost = ce + L2
correct_prediction = tf.equal(tf.cast(tf.argmax(y, 1), tf.int32), y_)
acc = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
return net, cost, acc
# with tf.device('/gpu:0'):
# network, cost, acc, = model(x_train_batch, y_train_batch, False)
# _, cost_test, acc_test = model(x_test_batch, y_test_batch, True)
# network, cost, acc, = model(x_train_batch, y_train_batch, False)
# _, cost_test, acc_test = model(x_test_batch, y_test_batch, True)
network, cost, acc, = model_batch_norm(x_train_batch, y_train_batch, False, is_train=True)
_, cost_test, acc_test = model_batch_norm(x_test_batch, y_test_batch, True, is_train=False)
# n_epoch = 50000
n_epoch = 5
learning_rate = 0.0001
print_freq = 1
n_step_epoch = int(len(y_train) / batch_size)
n_step = n_epoch * n_step_epoch
# with tf.device('/gpu:0'):
# train_op = tf.train.AdamOptimizer(learning_rate).minimize(cost)
train_op = tf.train.AdamOptimizer(learning_rate).minimize(cost)
tl.layers.initialize_global_variables(sess)
if resume:
print("Load existing model " + "!" * 10)
saver = tf.train.Saver()
saver.restore(sess, model_file_name)
network.print_params(False)
network.print_layers()
print(' learning_rate: %f' % learning_rate)
print(' batch_size: %d' % batch_size)
print(' n_epoch: %d, step in an epoch: %d, total n_step: %d' % (n_epoch, n_step_epoch, n_step))
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
step = 0
for epoch in range(n_epoch):
start_time = time.time()
train_loss, train_acc, n_batch = 0, 0, 0
for s in range(n_step_epoch):
err, ac, _ = sess.run([cost, acc, train_op])
step += 1
train_loss += err
train_acc += ac
n_batch += 1
if epoch + 1 == 1 or (epoch + 1) % print_freq == 0:
print("Epoch %d : Step %d-%d of %d took %fs" % (epoch, step, step + n_step_epoch, n_step, time.time() - start_time))
print(" train loss: %f" % (train_loss / n_batch))
print(" train acc: %f" % (train_acc / n_batch))
test_loss, test_acc, n_batch = 0, 0, 0
for _ in range(int(len(y_test) / batch_size)):
err, ac = sess.run([cost_test, acc_test])
test_loss += err
test_acc += ac
n_batch += 1
print(" test loss: %f" % (test_loss / n_batch))
print(" test acc: %f" % (test_acc / n_batch))
if (epoch + 1) % (print_freq * 5) == 0:
print("Save model " + "!" * 10)
saver = tf.train.Saver()
save_path = saver.save(sess, model_file_name)
tl.files.save_npz(network.all_params, name='4.6.2_model.npz', sess=sess)
# and restore it as follow:
# tl.files.load_and_assign_npz(sess=sess, name='model.npz', network=network)
coord.request_stop()
coord.join(threads)
sess.close()
老樣子，僅5個n_epoch就快等死人了~~
[TL] Load or Download cifar10 > data\cifar10
X_train.shape (50000, 32, 32, 3)
y_train.shape (50000,)
X_test.shape (10000, 32, 32, 3)
y_test.shape (10000,)
X float32 y int32
~~~~~~~~~data information~~~~~~~~~~~~
Converting data into train.cifar10 ...
Converting data into test.cifar10 ...
[TL] InputLayer model/input: (128, 24, 24, 3)
[TL] Conv2dLayer model/cnn1: shape:(5, 5, 3, 64) strides:(1, 1, 1, 1) pad:SAME act:identity
[TL] BatchNormLayer model/batch1: decay:1.000000 epsilon:0.000010 act:relu is_train:False
[TL] PoolLayer model/pool1: ksize:[1, 3, 3, 1] strides:[1, 2, 2, 1] padding:SAME pool:max_pool
[TL] Conv2dLayer model/cnn2: shape:(5, 5, 64, 64) strides:(1, 1, 1, 1) pad:SAME act:identity
[TL] BatchNormLayer model/batch2: decay:1.000000 epsilon:0.000010 act:relu is_train:False
[TL] PoolLayer model/pool2: ksize:[1, 3, 3, 1] strides:[1, 2, 2, 1] padding:SAME pool:max_pool
[TL] FlattenLayer model/flatten: 2304
[TL] DenseLayer model/d1relu: 384 relu
[TL] DenseLayer model/d2relu: 192 relu
[TL] DenseLayer model/output: 10 identity
[TL] [*] geting variables with relu/W
[TL] got 0: model/d1relu/W:0 (2304, 384)
[TL] got 1: model/d2relu/W:0 (384, 192)
[TL] InputLayer model/input: (128, 24, 24, 3)
[TL] Conv2dLayer model/cnn1: shape:(5, 5, 3, 64) strides:(1, 1, 1, 1) pad:SAME act:identity
[TL] BatchNormLayer model/batch1: decay:0.000000 epsilon:0.000010 act:relu is_train:False
[TL] PoolLayer model/pool1: ksize:[1, 3, 3, 1] strides:[1, 2, 2, 1] padding:SAME pool:max_pool
[TL] Conv2dLayer model/cnn2: shape:(5, 5, 64, 64) strides:(1, 1, 1, 1) pad:SAME act:identity
[TL] BatchNormLayer model/batch2: decay:0.000000 epsilon:0.000010 act:relu is_train:False
[TL] PoolLayer model/pool2: ksize:[1, 3, 3, 1] strides:[1, 2, 2, 1] padding:SAME pool:max_pool
[TL] FlattenLayer model/flatten: 2304
[TL] DenseLayer model/d1relu: 384 relu
[TL] DenseLayer model/d2relu: 192 relu
[TL] DenseLayer model/output: 10 identity
[TL] [*] geting variables with relu/W
[TL] got 0: model/d1relu/W:0 (2304, 384)
[TL] got 1: model/d2relu/W:0 (384, 192)
[TL] param 0: model/cnn1/W_conv2d:0 (5, 5, 3, 64) float32_ref
[TL] param 1: model/batch1/beta:0 (64,) float32_ref
[TL] param 2: model/batch1/gamma:0 (64,) float32_ref
[TL] param 3: model/batch1/moving_mean:0 (64,) float32_ref
[TL] param 4: model/batch1/moving_variance:0 (64,) float32_ref
[TL] param 5: model/cnn2/W_conv2d:0 (5, 5, 64, 64) float32_ref
[TL] param 6: model/batch2/beta:0 (64,) float32_ref
[TL] param 7: model/batch2/gamma:0 (64,) float32_ref
[TL] param 8: model/batch2/moving_mean:0 (64,) float32_ref
[TL] param 9: model/batch2/moving_variance:0 (64,) float32_ref
[TL] param 10: model/d1relu/W:0 (2304, 384) float32_ref
[TL] param 11: model/d1relu/b:0 (384,) float32_ref
[TL] param 12: model/d2relu/W:0 (384, 192) float32_ref
[TL] param 13: model/d2relu/b:0 (192,) float32_ref
[TL] param 14: model/output/W:0 (192, 10) float32_ref
[TL] param 15: model/output/b:0 (10,) float32_ref
[TL] num of params: 1068682
[TL] layer 0: model/cnn1/Identity:0 (128, 24, 24, 64) float32
[TL] layer 1: model/batch1/Relu:0 (128, 24, 24, 64) float32
[TL] layer 2: model/pool1:0 (128, 12, 12, 64) float32
[TL] layer 3: model/cnn2/Identity:0 (128, 12, 12, 64) float32
[TL] layer 4: model/batch2/Relu:0 (128, 12, 12, 64) float32
[TL] layer 5: model/pool2:0 (128, 6, 6, 64) float32
[TL] layer 6: model/flatten:0 (128, 2304) float32
[TL] layer 7: model/d1relu/Relu:0 (128, 384) float32
[TL] layer 8: model/d2relu/Relu:0 (128, 192) float32
[TL] layer 9: model/output/Identity:0 (128, 10) float32
learning_rate: 0.000100
batch_size: 128
n_epoch: 5, step in an epoch: 390, total n_step: 1950
Epoch 0 : Step 390-780 of 1950 took 515.877306s
train loss: 3.543596
train acc: 0.346534
test loss: 2.900318
test acc: 0.453225
Epoch 1 : Step 780-1170 of 1950 took 501.759281s
train loss: 2.757720
train acc: 0.439463
test loss: 2.425451
test acc: 0.517829
Epoch 2 : Step 1170-1560 of 1950 took 500.589279s
train loss: 2.387224
train acc: 0.493369
test loss: 2.143382
test acc: 0.548177
Epoch 3 : Step 1560-1950 of 1950 took 510.120896s
train loss: 2.156212
train acc: 0.523518
test loss: 1.943987
test acc: 0.575621
Epoch 4 : Step 1950-2340 of 1950 took 502.445682s
train loss: 1.988670
train acc: 0.543169
test loss: 1.789174
test acc: 0.599659
Save model !!!!!!!!!!
[TL] [*] 4.6.2_model.npz saved
[Finished in 2801.4s]
沒好機器的時候，參數都調不動啊~希望有路過的土豪機，能來試下20000的n_epoch~~~生成文件如下：
