python h5py 创建无限数据集、在无限数据集上添加新的数据

# -*- coding:utf-8 -*-

import h5py
import numpy as np

file = h5py.File('myunlim_dataset.hdf5','a')

dt_unlimt = file.create_dataset('data',[10,224,224,3],maxshape=[None,224,224,3],chunks=True,
compression='gzip',compression_opts=7) # 创建一个数据集对象，先把各项参数设置好，后续就不能改了，后面直接通过索引写入数据，数据会自动根据这些属性设置进行压缩存储。

# generate example data
a = np.arange(10*224*224*3).reshape(10,224,224,3)

# write a into dataset 'data',remenmber dt_unlimt is a 'data' dataset obj.
dt_unlimt[0:10] = a # write in by slice !

# now, we need to write new data to dataset 'data', and shape is [100,224,224,3],for example:

b = np.arange(100*224*224*3).reshape(100,224,224,3)

# before we write in new data, we need to resize dataset 'data' 's total shape first:
dt_unlimt.resize((110,224,224,3)) # 这里的resize的参数是dataset的总的shape，不仅仅是新写入数据集的shape。
dt_unlimt[10:110] = b # write in by slice! 记住，python切片是左闭右开的

# 写完数据不要忘记关闭！
file.close()

# That's OK !