Tensorflow中如何加載數據

版權聲明：本文爲博主原創文章，未經博主允許不得轉載。 https://blog.csdn.net/zSean/article/details/76474658

在Tensorflow中通過以下3中方式進行讀取數據：1.預加載數據(preloaded data)；2.填充數據(feeding)；3.從文件讀取數據(reading from file);

1.預加載數據：通常通過定義常量或變量來保存所有數據，缺點：由於直接將數據嵌入數據流圖中，當數據量過大時，過於消耗內存；

import tensorflow as tf
x1 = tf.constant([2,3,4])
x2 =tf.constant([4,2,1])
y = tf.add(x1,x2)

2.填充數據：由python產生數據，再把數據填充後端； 缺點：數據量大，消耗內存；以及數據類型轉換過於消耗內存；

import tensorflow as tf
a1 = tf.placeholder(tf.int16)
a2 = tf.placeholder(tf.int16)
b = tf.add(a1,a2)
li1 = [2,3,4]
li2 = [4,0,1]
with tf.Session() as sess:
      print(sess.run(b,feed_dict={a1:li1,a2:li2}))

3.從文件讀取數據：(1)首先將樣本數據寫入TFReords二進制文件；(2)再從隊列中讀取；

實現第一步：

from __future__ import absolute_import    
from __future__ import division
from __future__ import print_function
import argparse
import os
import tensorflow as tf
from tensorflow.contrib.learn.python.learn.datasets import mnist

FLAGS = None
def _int64_feature(value):   #生成整數類型
  return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))

def _bytes_feature(value):   #生成字符類型
  return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))

def convert_to(data_set, name):
  images = data_set.images
  labels = data_set.labels
  num_examples = data_set.num_examples
  if images.shape[0] != num_examples:
    raise ValueError('Images size %d does not match label size %d.' %
                     (images.shape[0], num_examples))
  rows = images.shape[1]
  cols = images.shape[2]
  depth = images.shape[3]
  filename = os.path.join(FLAGS.directory, name + '.tfrecords')
  print('Writing', filename)
  writer = tf.python_io.TFRecordWriter(filename)
  for index in range(num_examples):
    image_raw = images[index].tostring()        #將圖像矩陣轉化爲一個字符串
    example = tf.train.Example(features=tf.train.Features(feature={
        'height': _int64_feature(rows),           #寫入協議緩衝區，height,width,depth,label 編碼成int64類型，image——raw編碼成二進制
        'width': _int64_feature(cols),
        'depth': _int64_feature(depth),
        'label': _int64_feature(int(labels[index])),
        'image_raw': _bytes_feature(image_raw)}))
    writer.write(example.SerializeToString())
  writer.close()

def main(argv):
  data_sets = mnist.read_data_sets(FLAGS.directory,            #獲取數據
                                   dtype=tf.uint8,
                                   reshape=False,
                                   validation_size=FLAGS.validation_size)
  convert_to(data_sets.train, 'train')                       #將數據轉換成tf.train.Example類型
  convert_to(data_sets.validation, 'validation')
  convert_to(data_sets.test, 'test')
if __name__ == '__main__':
  parser = argparse.ArgumentParser()
  parser.add_argument(
      '--directory',
      type=str,
      default='D:/tmp/data',
  )
  parser.add_argument(
      '--validation_size',
      type=int,
      default=5000,
  )
  FLAGS = parser.parse_args()
  tf.app.run()

從文件中讀取並解析一個樣本：

def read_and_decode(filename_queue):
    reader = tf.TFRecordReader()
    _,serialized_example =reader.read(filename_queue)
    features = tf.parse_single_example(
        serialized_example,
        features={
            'image_raw':tf.FixedLenFeature([],tf.string),
            'label':tf.FixedLenFeature([],tf.int64),
        })
    image = tf.decode_raw(features['image_raw'],tf.uint8)
    image.set_shape([mnist.IMAGE_PIXELS])
    image = tf.cast(image,tf.float32)*(1./255)-0.5
    label = tf.cast(features['label'],tf.int32)
    return image,label