HPC 算力測試

#安裝前配置

curl 168.7.10.2:8000/local.repo > /etc/yum.repos.d/kylin_aarch64.repo
yum install -y libatomic environment-modules vim nfs-utils &>/dev/null
mkdir -p /workspace
mount 168.10.1.20:/workspace /workspace

#加載環境變量
cat>/etc/profile.d/hpc.sh<<'EOF'
source /usr/share/Modules/init/bash
module use /workspace/kylin/software/modules/
module load compilers/bisheng/2.5.0/bisheng2.5.0
module load mpi/hmpi/1.2.1/bisheng2.5.0
module load kml/1.7.0/kml_1.7.0.1_aarch64
EOF

grep /usr/share/Modules/init/bash /etc/profile || (echo "source /usr/share/Modules/init/bash">>/etc/profile && source /usr/share/Modules/init/bash)

#####################
# osu 測試
# 測試帶寬和延遲
#####################

# 1.測試項BW
mpidir=/workspace/kylin/software/osu/5.9/bisheng2.5.0_mpi1.2.1/libexec/osu-micro-benchmarks/mpi
gpu01=168.10.1.21

mpirun --allow-run-as-root -oversubscribe -np 2 -N 1 \
--host $gpu01 \
-mca pml ucx -mca btl ^vader,tcp,openib,uct \
-mca io romio321 \
${mpidir}/pt2pt/osu_bw


# 2.測試項Latency
mpidir=/workspace/kylin/software/osu/5.9/bisheng2.5.0_mpi1.2.1/libexec/osu-micro-benchmarks/mpi
gpu01=168.10.1.21

mpirun --allow-run-as-root -oversubscribe -np 2 -N 1 \
--host $gpu01 \
-mca pml ucx -mca btl ^vader,tcp,openib,uct \
-mca io romio321 \
${mpidir}/pt2pt/osu_latency



# 3.測試項Bcast
mpidir=/workspace/kylin/software/osu/5.9/bisheng2.5.0_mpi1.2.1/libexec/osu-micro-benchmarks/mpi
gpu01=168.10.1.21

mpirun --allow-run-as-root -oversubscribe -np 2 -N 1 \
--host $gpu01 \
-mca pml ucx \
-mca btl ^vader,tcp,openib,uct \
-mca io romio321 \
${mpidir}/collective/osu_bcast



# 4.測試項allreduce
mpidir=/workspace/kylin/software/osu/5.9/bisheng2.5.0_mpi1.2.1/libexec/osu-micro-benchmarks/mpi
gpu01=168.10.1.21

mpirun --allow-run-as-root -oversubscribe -np 2 -N 1 \
--host $gpu01 \
-mca pml ucx \
-mca btl ^vader,tcp,openib,uct \
-mca io romio321 \
${mpidir}/collective/osu_allreduce


# 5.測試項alltoall
mpidir=/workspace/kylin/software/osu/5.9/bisheng2.5.0_mpi1.2.1/libexec/osu-micro-benchmarks/mpi
gpu01=168.10.1.21

mpirun --allow-run-as-root -oversubscribe -np 2 -N 1 \
--host $gpu01 \
-mca pml ucx \
-mca btl ^vader,tcp,openib,uct \
-mca io romio321 \
${mpidir}/collective/osu_alltoall



###############################
# IMB基準測試
# 測試節點性能，網絡延時和吞吐量
###############################


#IMB PingPong測試
IMB_DIR=/workspace/kylin/software/imb/mpi-benchmarks-master/src_c
hostfile_dir=/workspace/kylin/software/imb/hostfile

mpirun --allow-run-as-root \
-x PATH=$PATH -x LD_LIBRARY_PATH=$LD_LIBRARY_PATH \
-mca pml ucx -mca btl ^vader,tcp,openib,uct -mca io romio321 \
-x UCX_TLS=self,sm \
-npernode 2 -hostfile $hostfile_dir \
${IMB_DIR}/IMB-MPI1 PingPong



############################################
#
# STREAM內存帶寬性能測量
# STREAM支持Copy、Scale、Add、Triad
#
############################################

STREAM_DIR=/workspace/kylin/software/STREAM-master

cd $STREAM_DIR
./stream_c.exe


############################################
#
#    HPL 測試
#
############################################

hpl_dir=/workspace/kylin/software/hpl/2.3/bin/kunpeng
hostfile_dir=/workspace/kylin/software/imb/hostfile

cd $hpl_dir
mpirun --allow-run-as-root --oversubscribe \
-x OMP_NUM_THREADS=8 \
-x LD_LIBRARY_PATH -x PATH -x PWD \
-map-by ppr:4:node:pe=2 \
-hostfile $hostfile_dir \
-mca pml ucx \
-mca btl ^vader,tcp,openib,uct \
-mca io romio321 \
-x UCX_TLS=self,sm \
./xhpl



############################################
#
# IOR 測試
#
############################################
IOR_DIR=/workspace/kylin/software/ior/bin
mkdir -p /workspace/kylin/software/apps/IOR/test/

# 1.1單節點單進程寫
mpirun --allow-run-as-root -np 1 -N 1 \
-mca pml ucx \
-mca btl ^vader,tcp,openib,uct \
-mca io romio321 -x UCX_TLS=self,sm \
${IOR_DIR}/ior \
-i 3 -w -t 1M -b 16g -F -e -k -o \
/workspace/kylin/software/apps/IOR/test/ior1M1T1.data

# 1.2單節點單進程讀
mpirun --allow-run-as-root -np 1 -N 1 \
-mca pml ucx \
-mca btl ^vader,tcp,openib,uct \
-mca io romio321 \
-x UCX_TLS=self,sm \
-x UCX_NET_DEVICES=mlx5_0:1 \
${IOR_DIR}/ior -i 3 -r -t 1M -b 16g -F -e -k -o \
/workspace/kylin/software/apps/IOR/test/ior1M1T1.data
 

IOR_DIR=/workspace/kylin/software/ior/bin
mkdir -p /workspace/kylin/software/apps/IOR/test/
hostfile_dir=/workspace/kylin/software/ior/hostfile
cat>/workspace/kylin/software/ior/hostfile<<'EOF'
168.1.1.1
....
168.1.1.10
EOF

# 2.1多節點多進程寫
mpirun --allow-run-as-root -np 16 -N 4 \
-hostfile ${hostfile_dir}  \
-mca pml ucx -mca btl ^vader,tcp,openib,uct \
-mca io romio321 \
-x UCX_TLS=self,sm \
-x UCX_NET_DEVICES=mlx5_0:1 \
${IOR_DIR}/ior -i 3 -w -t 1M -b 16g -F -e -k -o \
/workspace/kylin/software/apps/IOR/test/ior1M1T1.data

# 2.2多節點多進程讀
mpirun --allow-run-as-root -np 16 -N 4 \
-hostfile ${hostfile_dir}  \
-mca pml ucx \
-mca btl ^vader,tcp,openib,uct \
-mca io romio321 \
-x UCX_TLS=self,sm \
-x UCX_NET_DEVICES=mlx5_0:1 \
${IOR_DIR}/ior -i 3 -r -t 1M -b 16g -F -e -k -o \
/workspace/kylin/software/apps/IOR/test/ior1M1T1.data