#!/bin/bash
#HAdoop函數
function hadoop(){
echo "${needmountdisk[0]}磁盤掛載檢查"
#獲取掛載文件夾前一個文件夾
file=${needmountfile[i]}
filenum=`echo ${file#*e}`
BeforeMountfilenum=`printf "%02d" $((10#$filenum-1))`
BeforeMountfile=storage$BeforeMountfilenum
if mountpoint -q /mnt/${needmountfile[i]}
then
echo "修改${needmountfile[i]}目錄權限"
chmod -R 770 /mnt/${needmountfile[i]} && chown -R hdfs:hadoop /mnt/${needmountfile[i]}
echo "備份hdfs-site.xml到/tmp/"
rsync -azrL --delete --force /etc/hadoop/hdfs-site.xml /tmp/
echo "修改hdfs-site.xml配置文件"
ip=`ifconfig -a | grep -A 1 "bond0" | grep inet | awk '{print $2}' | awk -F ":" '{print $2}'`
if [[ ${needmountfile[i]} == storage01 ]];
then
#mkdir -p /run/hadoop && chown hdfs:hadoop /run/hadoop && su - hdfs -c 'hadoop-daemon.sh start datanode' && supervisorctl start all
#掛載磁盤爲storage01，將插入到/mnt/storage02/datum/datanode上面
sed -i '/\/mnt\/storage02\/datum\/datanode,/i\\ /mnt/storage01/datum/datanode,' /etc/hadoop/hdfs-site.xml
su - hdfs -c "hdfs dfsadmin -reconfig datanode $ip:50020 start"
su - hdfs -c "hdfs dfsadmin -reconfig datanode $ip:50020 status"
else
#掛載文件夾插入到前一文件夾後面
#mkdir -p /run/hadoop && chown hdfs:hadoop /run/hadoop && su - hdfs -c 'hadoop-daemon.sh start datanode' && supervisorctl start all
sed -i "/\/mnt\/$BeforeMountfile\/datum\/datanode,/a\\ /mnt\/${needmountfile[i]}\/datum\/datanode," /etc/hadoop/hdfs-site.xml
sed -i "/\/mnt\/$BeforeMountfile\/datum\/datanode,/a\\ /mnt\/${needmountfile[i]}\/datum\/datanode," /etc/hadoop/hdfs-site.xml
su - hdfs -c "hdfs dfsadmin -reconfig datanode $ip:50020 start"
su - hdfs -c "hdfs dfsadmin -reconfig datanode $ip:50020 status"
fi
echo "----------End----------"
break
else
echo "${needmountfile[i]}掛載失敗"
echo "/etc/fatab備份還原"
rsync -azrL --delete --force /tmp/fstab /etc/
echo "----------End----------"
exit 0
fi
}
#kafka函數
function kafka(){
echo "${needmountdisk[0]}磁盤掛載檢查"
if mountpoint -q /mnt/${needmountfile[i]}
then
echo "創建kafka文件環境"
mkdir -p /mnt/${needmountfile[i]}/datum
mkdir -p /mnt/${needmountfile[i]}/datum/kafka_data/
chmod -R 770 /mnt/${needmountfile[i]} && chown -R kafka:hadoop /mnt/${needmountfile[i]}
echo "----------End----------"
break
else
echo "${needmountfile[i]}掛載失敗"
echo "/etc/fatab備份還原"
rsync -azrL --delete --force /tmp/fstab /etc/
echo "----------End----------"
exit 0
fi
}
#yarn函數
function yarn(){
echo "/mnt/storage00磁盤掛載檢查"
if mountpoint -q /mnt/storage00
then
echo "創建yarn文件環境"
mkdir -p /mnt/storage00/logs
mkdir -p /mnt/storage00/logs/yarn
chown -R yarn:hadoop /mnt/storage00/logs/
echo "----------End----------"
break
else
echo "${needmountfile[i]}掛載失敗"
echo "/etc/fatab備份還原"
rsync -azrL --delete --force /tmp/fstab /etc/
echo "----------End----------"
exit 0
fi
}
#系統盤
systemdisk=`fdisk -l | grep -i Linux | awk '{print $1}'| head -n 1 | sed 's/.$//'`
#獲取所有磁盤盤符
alldisk=(`fdisk -l | grep -i disk | grep dev | sort -k 4 | awk '{print $2}' | awk -F ":" '{print $1}'| grep -v $systemdisk`)
#獲取所有磁盤盤符數量
alldisksum=${#alldisk[@]}
#創建文件夾
for i in `seq -w 0 40`
do
if [[ 10#$i -lt $alldisksum ]];
then
mkdir -p /mnt/storage$i
fi
done
#獲取已掛載磁盤盤符
fsdisk=(`df -h | grep mnt | awk '{print $1}'`)
#獲取mnt下文件夾數量
allfilesum=`ls /mnt/ | grep storage| wc -l`
#獲取已經mount文件夾數量
mountfilesum=`df -h | grep mnt | wc -l`
#獲取mnt下所有文件夾
allfile=(`ls /mnt/ | grep storage`)
#獲取已掛載文件夾
mountfile=(`df -h | grep mnt | awk '{print $6}'`)
#遍歷獲取未掛載文件夾
declare -a needmountfile
t=0
flag=0
for nomountfile in "${allfile[@]}"
do
for file in "${mountfile[@]}"
do
if [[ "/mnt/${nomountfile}" == "${file}" ]]; then
flag=1
break
fi
done
if [[ $flag -eq 0 ]]; then
needmountfile[t]=$nomountfile
t=$((t+1))
else
flag=0
fi
done
#未掛載文件夾數量
needmountfilesum=${#needmountfile[@]}
#遍歷獲取未掛載盤符
declare -a needmountdisk
t=0
flag=0
for mountdisk in "${alldisk[@]}"
do
for disk in "${fsdisk[@]}"
do
if [[ "${mountdisk}" == "${disk}" ]]; then
flag=1
break
fi
done
if [[ $flag -eq 0 ]]; then
needmountdisk[t]=$mountdisk
t=$((t+1))
else
flag=0
fi
done
#未掛載磁盤數量
mountdisknum=${#needmountdisk[@]}
if [[ $mountdisknum -ge 1 ]]; then
#掛載磁盤
for (( i = 0; i <= $mountdisknum-1 ; i++ )); do
OLDUUID=`cat /etc/fstab | grep ${needmountfile[i]} | awk -F '"' '{print $2}'`
fs=`blkid | awk -F '"' '{print $4}' | awk 'END {print}'`
echo "----------Begin----------"
echo "磁盤${needmountdisk[i]}開始格式化"
mkfs.$fs ${needmountdisk[i]}
if [ $? -eq 0 ];
then
PCIE=`fdisk -l | grep ${needmountdisk[i]} | awk '{print $3}'`
disk=`fdisk -l | grep ${needmountdisk[i]} | awk '{print $3}'`
#判斷磁盤是否爲PCIE
if [[ ${needmountdisk[i]} == nvme* ]] && [[ `expr $PCIE "==" 1.8` -eq 1 ]];
then
echo "盤符${needmountdisk[i]}格式化完成"
rsync -azrL --delete --force /etc/fstab /tmp/
NEWUUID=`blkid | grep ${needmountdisk[i]} | awk -F '"' '{print $2}'`
if [ ! -n "$OLDUUID" ];
then
echo "添加${needmountdisk[i]}磁盤UUID到/etc/fatab中需掛載的文件/mnt/${needmountfile[i]}"
echo "UUID="$NEWUUID" /mnt/storage00 $fs defaults,nofail,noatime,nodiratime 0 0 #${needmountdisk[i]}" >> /etc/fstab
#sed -i "$a\UUID="$NEWUUID" /mnt/storage00 $fs defaults,nofail,noatime,nodiratime 0 0 #${needmountdisk[i]}" /etc/fstab
echo "磁盤掛載到${needmountfile[i]}文件夾"
mount -a
#hadoop
#kafka
#yarn
else
echo "添加${needmountdisk[i]}磁盤UUID到/etc/fatab中需掛載的文件/mnt/storage00"
sed -i "s/$OLDUUID/$NEWUUID/" /etc/fstab
echo "磁盤掛載到${needmountfile[i]}文件夾"
mount -a
#hadoop
#kafka
#yarn
fi
#unset needmountfile[storage00]
continue
#判斷磁盤大小是否爲960G
elif [[ `expr $disk \> 890.0` -eq 1 ]] && [[ `expr $disk \> 960.0` -eq 0 ]];
then
echo "盤符${needmountdisk[i]}格式化完成"
rsync -azrL --delete --force /etc/fstab /tmp/
NEWUUID=`blkid | grep ${needmountdisk[i]} | awk -F '"' '{print $2}'`
if [ ! -n "$OLDUUID" ];
then
echo "添加${needmountdisk[i]}磁盤UUID到/etc/fatab中需掛載的文件/mnt/storage00"
echo "UUID="$NEWUUID" /mnt/storage00 $fs defaults,nofail,noatime,nodiratime 0 0 #${needmountdisk[i]}" >> /etc/fstab
echo "磁盤掛載到${needmountfile[i]}文件夾"
mount -a
#hadoop
#kafka
yarn
else
echo "添加${needmountdisk[i]}磁盤UUID到/etc/fatab中需掛載的文件/mnt/storage00"
sed -i "s/$OLDUUID/$NEWUUID/" /etc/fstab
echo "磁盤掛載到${needmountfile[i]}文件夾"
mount -a
#hadoop
#kafka
yarn
fi
unset needmountfile[storage00]
continue
else
echo "盤符${needmountdisk[i]}格式化完成"
rsync -azrL --delete --force /etc/fstab /tmp/
NEWUUID=`blkid | grep ${needmountdisk[i]} | awk -F '"' '{print $2}'`
if [ ! -n "$OLDUUID" ];
then
echo "添加${needmountdisk[i]}磁盤UUID到/etc/fatab中需掛載的文件/mnt/${needmountfile[i]}"
echo "UUID="$NEWUUID" /mnt/${needmountfile[i]} $fs defaults,nofail,noatime,nodiratime 0 0 #${needmountdisk[i]}" >> /etc/fstab
echo "磁盤掛載到${needmountfile[i]}文件夾"
mount -a
#hadoop
#kafka
#yarn
rm -rf /tmp/hdfs-site.xml
else
echo "添加${needmountdisk[i]}磁盤UUID到/etc/fatab中需掛載的文件/mnt/${needmountfile[i]}"
sed -i "s/$OLDUUID/$NEWUUID/" /etc/fstab
echo "磁盤掛載到${needmountfile[i]}文件夾"
mount -a
#hadoop
#kafka
#yarn
rm -rf /tmp/hdfs-site.xml
fi
fi
else
echo "${needmountdisk[i]}磁盤格式化失敗"
break
fi
done
else
#沒有可以掛載的磁盤
echo “磁盤均已掛載”
echo “請檢查硬盤是否被系統識別”
exit 0
fi