Linux记录-shell获取hdfs used使用

#!/bin/bash

export JAVA_HOME=xxx
export HADOOP_HOME=xxx
export HADOOP_CONF_DIR=xxx

cd /home/hdfs/xxx

mv host.txt host.txt_`date +"%Y-%m-%d"`
mv host.txt_`date +"%Y-%m-%d"` host/

mv hostdfs.txt hostdfs.txt_`date +"%Y-%m-%d"`
mv hostdfs.txt_`date +"%Y-%m-%d"` host/

rm -f ip.txt
rm -f allhostname.txt
rm -f hostname.txt
rm -f iphostname.txt
rm -f dfused.txt
rm -f minhost.txt
rm -f maxhost.txt
rm -f host.txt
rm -f nohup.out
rm -f dfsreport.txt

echo "start checking hdfs used rate"
$HADOOP_HOME/bin/hdfs dfsadmin -report > dfsreport.txt
cat dfsreport.txt | grep -i "50010" | awk -F ":" '{print $2}'| awk '{print $1}' > ip.txt
cat dfsreport.txt | grep -i "hostname" | awk -F ": " '{print $2}' > allhostname.txt
cat dfsreport.txt  | grep "DFS Used%" | awk -F ": " '{print $2}' | awk -F "%" '{print $1}' > dfused.txt
livesum=$(cat dfsreport.txt  | grep "Live datanodes" | awk '{print $3}' | awk -F "):" '{print $1}' | awk -F "(" '{print $2}')
echo $livesum
sed -i '1d' dfused.txt
let livesum+=1
sed -i $livesum',$d' dfused.txt 
sed -i $livesum',$d' ip.txt
linesum=$(cat ip.txt | wc -l)
echo $linesum
harr=($(awk '{print $1}' ip.txt))
darr=($(awk '{print $1}' dfused.txt)) 
if [ $linesum -gt 0 ]
  then
  for (( i = 0 ; i < ${#harr[@]} ; i++ ))
  do
     for (( j = 0 ; j < ${#darr[@]} ; j++ ))
     do
       if [ $i -eq $j ]
       then
         echo ${harr[$i]} ":" ${darr[$j]} >> hostdfs.txt
       fi
     done
 done
else
   echo "Not Live Datanodes"
fi
hharr=($(cat hostdfs.txt | awk '{print $3}' |sort -ru | tail -n 60)) 
ddarr=($(cat hostdfs.txt | awk '{print $3}' |sort -ru | head -n 100))
for (( k = 0; k< ${#hharr[@]} ; k++ ))
do
    if [[ $(echo $(cat hostdfs.txt) | grep "${hharr[$k]}") != "" ]]
    then
    cat hostdfs.txt | grep "${hharr[$k]}" | awk 'NR==1{print $1}' >> minhost.txt
    fi
done
for (( m = 0; m< ${#ddarr[@]} ; m++ ))
do
    if [[ $(echo $(cat hostdfs.txt) | grep "${ddarr[$m]}") != "" ]]
    then
    cat hostdfs.txt | grep "${ddarr[$m]}" | awk 'NR==1{print $1}' >> maxhost.txt
    fi
done

cat maxhost.txt | awk '{print $1}' >> host.txt
cat minhost.txt | awk '{print $1}' >> host.txt

narr=($(cat allhostname.txt | awk '{print $1}'))
for (( k = 0; k< ${#harr[@]}; k++ ))
do
  for (( n = 0; n < ${#narr[@]}; n++))
  do
    if [ $k -eq $n ]
    then
      echo ${harr[$k]} ":" ${narr[$n]} >>iphostname.txt
    fi
  done
done

hostarr=($(cat host.txt | awk '{print $1}'))
for (( c = 0; c < ${#hostarr[@]} ; c++ ))
do
    if [[ $(echo $(cat iphostname.txt) | grep "${hostarr[$c]}") != "" ]]
    then
      cat iphostname.txt | grep "${hostarr[$c]}" | awk 'NR==1{print $3}' >> hostname.txt
    fi
done

max_rate=$(cat dfused.txt | sort -r | head -n 1 | awk '{print int($0)}')
avg_used_rate=$(cat dfused.txt | awk '{e+=$1}END{print e/NR}' | awk '{print int($0)}' )
let max_avg_diff=max_rate-avg_used_rate

if [ $max_avg_diff -gt 5 ]
   then
   jps | grep -i "balancer"
   if [ $? -eq 0 ]
   then
      kill -9 $($JAVA_HOME/bin/jps | grep -i "balancer" | awk 'NR==1{print $1}')
   fi
   $HADOOP_HOME/bin/hdfs dfs -mv /system/balancer.id /system/balancer.id_`date +"%Y-%m-%d-%H-%M"` 
   nohup $HADOOP_HOME/bin/hdfs  balancer  -policy datanode -threshold 5 -include -f host.txt > /home/hdfs/balancer/rebalancer.log 2>&1 &
else
  echo "Nothing to do"
fi

$ nohup hdfs balancer \
-Ddfs.datanode.balance.max.concurrent.moves = 10 \
-Ddfs.balancer.dispatcherThreads = 1024 \
-Ddfs.balance.bandwidthPerSec = 1073741824

#此配置用于限制允许Datanode平衡群集的最大并发块移动数
dfs.datanode.balance.max.concurrent.moves, default is 5
#带宽
dfs.datanode.balance.bandwidthPerSec, default is 1048576 (=1MB/s)
dfsadmin -setBalancerBandwidth <bandwidth in bytes per second>
#mover线程数
dfs.balancer.moverThreads, default is 1000
#datanode传输的最大线程数
dfs.datanode.max.transfer.threads
修改dfs.datanode.max.transfer.threads=4096 (如果运行HBase的话建议为16384)，
指定用于在DataNode间传输block数据的最大线程数，老版本的对应参数为dfs.datanode.max.xcievers。
#平衡策略，默认为datanode
[-policy <policy>]
blockpool: Cluster is balanced if each pool in each node is balanced.
datanode: Cluster is balanced if each datanode is balanced.
#阈值
[-threshold <threshold>] [1.0, 100.0]
#包含列表
[-include [-f <hosts-file> | <comma-separated list of hosts>]]
#排除列表
[-exclude [-f <hosts-file> | <comma-separated list of hosts>]]
#最大移动数据大小
dfs.balancer.max-size-to-move, default is 10737418240 (=10GB)