0. python打包并上传到hdfs
# 安装Python
mkdir Python
export PYTHON_ROOT=~/Python
tar -xvf Python-3.6.8.tgz
pushd Python-3.6.8
./configure --prefix="{PYTHON_ROOT}" --enable-unicode=cs4
make && make install
popd
rm -rf Python-3.6.8.tgz
# Python打包
export PYTHON_ROOT=~/Python
pushd "${PYTHON_ROOT}"
zip -r Python.zip *
popd
# 推送到hdfs
hadoop fs -put Python.zip /usr/dm/tools/
1. 模型本地分发
以Keras保存的.h5模型为例.
#!/bin/bash
# 输入输出
INPUT_DIR="hdfs://ns3-backup/usr/dm/data/predict_feature/dt=20191201/part_*"
output_DIR="hdfs://ns3-backup/usr/dm/data/predict_feature_results/dt=20191201/"
# Python包
PYTHON="hdfs://ns3-backup/usr/dm/tools/Python.zip#Python"
# 模型位置
MODEL="./models.h5"
# hadoop stream命令
hadoop jar /usr/local/hadoop-2.7.3/share/hadoop/tools/lib/hadoop-streaming-2.7.3.jar \
-archives ${PYTHON} \
-input ${INPUT_DIR} \
-output ${OUTPUT_DIR} \
-mapper "Python/bin/python3 mapper.py" \
-mapper "Python/bin/python3 reducer.py ${MODEL}" \
-jobconf mapred.map.tasks=1000 \
-jobconf mapred.reduce.tasks=1000 \
-jobconf mapred.job.name="predict" \
-jobconf mapreduce.map.memory.mb=4096 \
-jobconf mapreduce.reduce.memory.mb=4096 \
-inputformat com.sina.hadoop.rcfile.RCFileAsTextInputFormat \ # 输入为RCFile格式
-file mapper.py \
-file reducer.py \
-file ${MODEL}
2. 模型从HDFS分发
还是以Keras保存的.h5模型为例.
#!/bin/bash
# 输入输出
INPUT_DIR="hdfs://ns3-backup/usr/dm/data/predict_feature/dt=20191201/part_*"
output_DIR="hdfs://ns3-backup/usr/dm/data/predict_feature_results/dt=20191201/"
# Python包
PYTHON="hdfs://ns3-backup/usr/dm/tools/Python.zip#Python"
# 模型位置
MODEL="hdfs://ns3-backup/usr/dm/model/models.h5#models"
# hadoop stream命令
hadoop jar /usr/local/hadoop-2.7.3/share/hadoop/tools/lib/hadoop-streaming-2.7.3.jar \
-archives ${PYTHON} \
-input ${INPUT_DIR} \
-output ${OUTPUT_DIR} \
-mapper "Python/bin/python3 mapper.py" \
-mapper "Python/bin/python3 reducer.py models" \
-jobconf mapred.map.tasks=1000 \
-jobconf mapred.reduce.tasks=1000 \
-jobconf mapred.job.name="predict" \
-jobconf mapreduce.map.memory.mb=4096 \
-jobconf mapreduce.reduce.memory.mb=4096 \
-inputformat com.sina.hadoop.rcfile.RCFileAsTextInputFormat \ # 输入为RCFile格式
-file mapper.py \
-file reducer.py \
-cacheFile ${MODEL}
3. 带多个目录的模型从HDFS分发
从本地使用 -file分发模型和从HDFS使用-cacheFile分发模型最终都是在同一级目录下使用。使用tf.saved_model.load读取tpb模型时存在需要读取model下的saved_model.pb和assets及variables两个目录,模型结果如下:
models -| saved_model.pb
-| aseets -| key
-| variables -| variables.data-00000-of-00001
-| variables.index
我们可以使用与python用法一样的压缩方式进行加载.
模型打包:
cd model && zip -r models.zip *
hadoop fs -put models.zip /usr/dm/data/models
hadoop streaming命令如下:
#!/bin/bash
# 输入输出
INPUT_DIR="hdfs://ns3-backup/usr/dm/data/predict_feature/dt=20191201/part_*"
output_DIR="hdfs://ns3-backup/usr/dm/data/predict_feature_results/dt=20191201/"
# Python包
PYTHON="hdfs://ns3-backup/usr/dm/tools/Python.zip#Python"
# 模型位置
MODEL="hdfs://ns3-backup/usr/dm/model/models.zip#models"
# hadoop stream命令
hadoop jar /usr/local/hadoop-2.7.3/share/hadoop/tools/lib/hadoop-streaming-2.7.3.jar \
-archives ${PYTHON}, ${MODEL} \
-input ${INPUT_DIR} \
-output ${OUTPUT_DIR} \
-mapper "Python/bin/python3 mapper.py" \
-mapper "Python/bin/python3 reducer.py models" \
-jobconf mapred.map.tasks=1000 \
-jobconf mapred.reduce.tasks=1000 \
-jobconf mapred.job.name="predict" \
-jobconf mapreduce.map.memory.mb=4096 \
-jobconf mapreduce.reduce.memory.mb=4096 \
-inputformat com.sina.hadoop.rcfile.RCFileAsTextInputFormat \ # 输入为RCFile格式
-file mapper.py \
-file reducer.py