HPC 算力测试

#安装前配置

curl 168.7.10.2:8000/local.repo > /etc/yum.repos.d/kylin_aarch64.repo
yum install -y libatomic environment-modules vim nfs-utils &>/dev/null
mkdir -p /workspace
mount 168.10.1.20:/workspace /workspace

#加载环境变量
cat>/etc/profile.d/hpc.sh<<'EOF'
source /usr/share/Modules/init/bash
module use /workspace/kylin/software/modules/
module load compilers/bisheng/2.5.0/bisheng2.5.0
module load mpi/hmpi/1.2.1/bisheng2.5.0
module load kml/1.7.0/kml_1.7.0.1_aarch64
EOF

grep /usr/share/Modules/init/bash /etc/profile || (echo "source /usr/share/Modules/init/bash">>/etc/profile && source /usr/share/Modules/init/bash)

#####################
# osu 测试
# 测试带宽和延迟
#####################

# 1.测试项BW
mpidir=/workspace/kylin/software/osu/5.9/bisheng2.5.0_mpi1.2.1/libexec/osu-micro-benchmarks/mpi
gpu01=168.10.1.21

mpirun --allow-run-as-root -oversubscribe -np 2 -N 1 \
--host $gpu01 \
-mca pml ucx -mca btl ^vader,tcp,openib,uct \
-mca io romio321 \
${mpidir}/pt2pt/osu_bw


# 2.测试项Latency
mpidir=/workspace/kylin/software/osu/5.9/bisheng2.5.0_mpi1.2.1/libexec/osu-micro-benchmarks/mpi
gpu01=168.10.1.21

mpirun --allow-run-as-root -oversubscribe -np 2 -N 1 \
--host $gpu01 \
-mca pml ucx -mca btl ^vader,tcp,openib,uct \
-mca io romio321 \
${mpidir}/pt2pt/osu_latency



# 3.测试项Bcast
mpidir=/workspace/kylin/software/osu/5.9/bisheng2.5.0_mpi1.2.1/libexec/osu-micro-benchmarks/mpi
gpu01=168.10.1.21

mpirun --allow-run-as-root -oversubscribe -np 2 -N 1 \
--host $gpu01 \
-mca pml ucx \
-mca btl ^vader,tcp,openib,uct \
-mca io romio321 \
${mpidir}/collective/osu_bcast



# 4.测试项allreduce
mpidir=/workspace/kylin/software/osu/5.9/bisheng2.5.0_mpi1.2.1/libexec/osu-micro-benchmarks/mpi
gpu01=168.10.1.21

mpirun --allow-run-as-root -oversubscribe -np 2 -N 1 \
--host $gpu01 \
-mca pml ucx \
-mca btl ^vader,tcp,openib,uct \
-mca io romio321 \
${mpidir}/collective/osu_allreduce


# 5.测试项alltoall
mpidir=/workspace/kylin/software/osu/5.9/bisheng2.5.0_mpi1.2.1/libexec/osu-micro-benchmarks/mpi
gpu01=168.10.1.21

mpirun --allow-run-as-root -oversubscribe -np 2 -N 1 \
--host $gpu01 \
-mca pml ucx \
-mca btl ^vader,tcp,openib,uct \
-mca io romio321 \
${mpidir}/collective/osu_alltoall



###############################
# IMB基准测试
# 测试节点性能，网络延时和吞吐量
###############################


#IMB PingPong测试
IMB_DIR=/workspace/kylin/software/imb/mpi-benchmarks-master/src_c
hostfile_dir=/workspace/kylin/software/imb/hostfile

mpirun --allow-run-as-root \
-x PATH=$PATH -x LD_LIBRARY_PATH=$LD_LIBRARY_PATH \
-mca pml ucx -mca btl ^vader,tcp,openib,uct -mca io romio321 \
-x UCX_TLS=self,sm \
-npernode 2 -hostfile $hostfile_dir \
${IMB_DIR}/IMB-MPI1 PingPong



############################################
#
# STREAM内存带宽性能测量
# STREAM支持Copy、Scale、Add、Triad
#
############################################

STREAM_DIR=/workspace/kylin/software/STREAM-master

cd $STREAM_DIR
./stream_c.exe


############################################
#
#    HPL 测试
#
############################################

hpl_dir=/workspace/kylin/software/hpl/2.3/bin/kunpeng
hostfile_dir=/workspace/kylin/software/imb/hostfile

cd $hpl_dir
mpirun --allow-run-as-root --oversubscribe \
-x OMP_NUM_THREADS=8 \
-x LD_LIBRARY_PATH -x PATH -x PWD \
-map-by ppr:4:node:pe=2 \
-hostfile $hostfile_dir \
-mca pml ucx \
-mca btl ^vader,tcp,openib,uct \
-mca io romio321 \
-x UCX_TLS=self,sm \
./xhpl



############################################
#
# IOR 测试
#
############################################
IOR_DIR=/workspace/kylin/software/ior/bin
mkdir -p /workspace/kylin/software/apps/IOR/test/

# 1.1单节点单进程写
mpirun --allow-run-as-root -np 1 -N 1 \
-mca pml ucx \
-mca btl ^vader,tcp,openib,uct \
-mca io romio321 -x UCX_TLS=self,sm \
${IOR_DIR}/ior \
-i 3 -w -t 1M -b 16g -F -e -k -o \
/workspace/kylin/software/apps/IOR/test/ior1M1T1.data

# 1.2单节点单进程读
mpirun --allow-run-as-root -np 1 -N 1 \
-mca pml ucx \
-mca btl ^vader,tcp,openib,uct \
-mca io romio321 \
-x UCX_TLS=self,sm \
-x UCX_NET_DEVICES=mlx5_0:1 \
${IOR_DIR}/ior -i 3 -r -t 1M -b 16g -F -e -k -o \
/workspace/kylin/software/apps/IOR/test/ior1M1T1.data
 

IOR_DIR=/workspace/kylin/software/ior/bin
mkdir -p /workspace/kylin/software/apps/IOR/test/
hostfile_dir=/workspace/kylin/software/ior/hostfile
cat>/workspace/kylin/software/ior/hostfile<<'EOF'
168.1.1.1
....
168.1.1.10
EOF

# 2.1多节点多进程写
mpirun --allow-run-as-root -np 16 -N 4 \
-hostfile ${hostfile_dir}  \
-mca pml ucx -mca btl ^vader,tcp,openib,uct \
-mca io romio321 \
-x UCX_TLS=self,sm \
-x UCX_NET_DEVICES=mlx5_0:1 \
${IOR_DIR}/ior -i 3 -w -t 1M -b 16g -F -e -k -o \
/workspace/kylin/software/apps/IOR/test/ior1M1T1.data

# 2.2多节点多进程读
mpirun --allow-run-as-root -np 16 -N 4 \
-hostfile ${hostfile_dir}  \
-mca pml ucx \
-mca btl ^vader,tcp,openib,uct \
-mca io romio321 \
-x UCX_TLS=self,sm \
-x UCX_NET_DEVICES=mlx5_0:1 \
${IOR_DIR}/ior -i 3 -r -t 1M -b 16g -F -e -k -o \
/workspace/kylin/software/apps/IOR/test/ior1M1T1.data