计算数据传输花费时间和传输速度

计算数据传输花费时间和传输速度

　　昨天，快要下班了，竟然给出了代码需求，这能忍吗？必须干它呀，拿着需求灰溜溜地回家了。大概原型是这样滴：

有个shell脚本，将指定后缀的文件从一个文件夹复制/移动到另一个文件夹，并检验md5值，并计算复制数据文件的速度和花费时间。

首先分析一下需求吧！

 

分析

阶段一：找出指定后缀

ls -a (路径) | grep ".*\.yuv\|.*\.264\|.*\.mjpeg\| &"

阶段二：复制文件

提供思路：从找出后缀的这些文件中，for循环复制/移动。

阶段三：检验md5值

md5sum 文件 |awk -F' ' '{print $1}'

提供思路：设置两个md5值检测，分别是复制文件前后，然后通过前后md5值进行比对，并挑出MD5值不同的数据文件。

阶段四：复制数据的速度和所花费的时间

starttime=`date +'%Y-%m-%d %H:%M:%S'`
#执行程序
endtime=`date +'%Y-%m-%d %H:%M:%S'`
start_seconds=$(date --date="$starttime" +%s);
end_seconds=$(date --date="$endtime" +%s);
echo "本次运行时间： "$((end_seconds-start_seconds))"s"

当然我把它封装了，哈哈，即调即用... ...

starttime=`date +'%Y-%m-%d %H:%M:%S'`

Time_calculation() {
    endtime=`date +'%Y-%m-%d %H:%M:%S'`
    start_seconds=$(date --date="$starttime" +%s);
    end_seconds=$(date --date="$endtime" +%s);
}

这个阶段需要注意的是：有的文件大小过小，执行速度一闪而过，就会出现分母为0的情况，所以要在这做个if判断。等于（-eq），不等于（-ne），大于（-gt）,小于（-lt）通过时间差来计算。

这里还有个小知识点：就是shell中使用除法，保留几位小数的问题。【关键代码】

$(printf "%.2f" `echo "scale=2;$(du -m $i  |awk -F' ' '{print $1}')/$((end_seconds-start_seconds))"|bc`) "M/s"

阶段五：完整的代码

#!/bin/sh


data_p=/root/sd/p3/data_p/
dir_p=/root/sd/p3/dir_p/


# 没有目标路径要创建
if [ ! -d $dir_p ]; then
        mkdir $dir_p
fi

# 筛选出以 .yuv .264 .mjpeg 后缀的文件
data_file=$(ls -a $(data_p) | grep ".*\.yuv\|.*\.264\|.*\.mjpeg\| &")

starttime=`date +'%Y-%m-%d %H:%M:%S'`

Time_calculation() {
    endtime=`date +'%Y-%m-%d %H:%M:%S'`
    start_seconds=$(date --date="$starttime" +%s);
    end_seconds=$(date --date="$endtime" +%s);
}

for i in $data_file
do
    md_data_file=`md5sum $i|awk -F' ' '{print $1}'`
    cp $i $dir_p
    Time_calculation

# 判断数据传输花费的时间是否为0，以防出现除数为0的情况
    if [ $((end_seconds-start_seconds)) -eq 0 ]
    then
        echo "$i is High speed!" "~" "$i transmission time: "$((end_seconds-start_seconds))"s"
    elif [ $((end_seconds-start_seconds)) -gt 0 ]
    then
        echo "$i transmission_speed is " $(printf "%.2f" `echo "scale=2;$(du -m $i  |awk -F' ' '{print $1}')/$((end_seconds-start_seconds))"|bc`) "M/s"
    fi

    md_dir_file=`md5sum $i|awk -F' ' '{print $1}'`
    if [ $md_data_file == $md_dir_file ]
    then
        echo "$i MD5 is ok!"
    elif [ $md_data_file != $md_dir_file ]
    then
        echo "$i MD5 is no ok!"
    fi
        
    

done

Time_calculation
echo "All operations time: "$((end_seconds-start_seconds))"s" 

然后就可以执行了... ...

阶段六：二次优化

增加了删除源目录的文件功能，业务上修改了部分代码

#!/bin/sh


data_p=/root/sd/p3/data_p/
dir_p=/root/sd/p3/dir_p/


# 没有目标路径要创建
if [ ! -d $dir_p ]; then
        mkdir $dir_p
fi

starttime=`date +'%Y-%m-%d %H:%M:%S'`

Time_calculation() {
    endtime=`date +'%Y-%m-%d %H:%M:%S'`
    start_seconds=$(date --date="$starttime" +%s);
    end_seconds=$(date --date="$endtime" +%s);
}

cd $data_p

for i in *
do
    md_data_file=`md5sum $i|awk -F' ' '{print $1}'`
    cp $i $dir_p  && sync
    Time_calculation
    

# 判断数据传输花费的时间是否为0，以防出现除数为0的情况
    if [ $((end_seconds-start_seconds)) -eq 0 ]
    then
        echo "$i is High speed!" "~" "$i transmission time: "$((end_seconds-start_seconds))"s"
    elif [ $((end_seconds-start_seconds)) -gt 0 ]
    then
        echo "$i transmission_speed is " $(printf "%.2f" `echo "scale=2;$(du -m $i  |awk -F' ' '{print $1}')/$((end_seconds-start_seconds))"|bc`) "M/s"
    fi

# 校验传输前后文件的MD5值，判断MD5值是否正确
    md_dir_file=`md5sum $i|awk -F' ' '{print $1}'`
    if [ $md_data_file == $md_dir_file ]
    then
        echo "$i MD5 is ok!"
    elif [ $md_data_file != $md_dir_file ]
    then
        echo "$i MD5 is no ok!"
    fi
    
    echo "$i has been moved to $dir_p , Source directory's $i is deleting..."
    rm $data_p/$i
done
Time_calculation
echo "All operations time: "$((end_seconds-start_seconds))"s" 

阶段七：代码改良

#!/bin/sh

source_dir='/root/sd/p3/test'
target_dir='/root/emmc/p4/test_read_write'

[ -z "$1" ] && loop_time=100 || loop_time=$1  # 测试次数

[ -n "$2" ] && mode=$2 || mode='cp'

[ ! -d $target_dir ] && mkdir -p $target_dir



function do_test_use_MD5()
{

    cd $source_dir
    for file_to_test in *

    do
        s_md5=$(echo $file_to_test|awk -F '[-.]' '{print $2}'  | tr '[a-z]' '[A-Z]' )  # 拿出原文件md5
        #echo "file_size_info=$(ls -lh $file_to_test | awk '{print $5}')"
        file_size_info=$(ls -lh $file_to_test | awk '{print $5}')
        file_size_number=${file_size_info:0:-1}
        file_size_unit=${file_size_info: -1}

        # 单位为G的，转换为M
        [ $file_size_unit = "G" ] && { file_size_unit='M'; file_size_number=$( echo "$file_size_number * 1024" | bc ) ; }
        start_time=$(date +%"s")
        if [ "$mode" = "cp" ]; then
                echo -n  "cp $file_to_test to  $target_dir ....."
                cp $file_to_test $target_dir && sync || echo "cp $file_to_test failed!"
                echo 1 > /proc/sys/vm/drop_caches  &>/dev/null
                end_time=$(date +%"s")
                d_md5=$(md5sum $target_dir/$file_to_test |awk '{print $1}'| tr '[a-z]' '[A-Z]')
                [ $s_md5 = $d_md5 ] && echo -n " cp successed!" || echo "md5sum not match ($s_md5, $d_md5), cp failed!"
        fi

        if [ $mode = 'dd' ]; then
                echo -n  "dd $file_to_test to  $target_dir ....."
                dd if=$file_to_test of=$target_dir/$file_to_test conv=fsync && sync || echo "dd  $file_to_test failed!"
                echo 1 > /proc/sys/vm/drop_caches &>/dev/null
                end_time=$(date +%"s")
                d_md5=$(md5sum $target_dir/$file_to_test |awk '{print $1}'|tr '[a-z]' '[A-Z]')
                [ $s_md5 = $d_md5 ] && echo -n "dd successed!" || echo "md5sum not match ($s_md5, $d_md5), dd failed!"
        fi

        duration=$((end_time - start_time))
        [ $duration -eq 0 ] && duration=1
        echo "size = $file_size_number"
        speed=$(echo "scale=2; $file_size_number / $duration" | bc )
        echo "speed = $speed $file_size_unit/s"
        rm -f $target_dir/$file_to_test

    done
}



function do_test()
{

        cd $source_dir
        for file_to_test in *
        
        do
                #s_md5=$(echo $file_to_test|awk -F '[-.]' '{print $2}'  | tr '[a-z]' '[A-Z]' )  # 拿出原文件md5
                #echo "file_size_info=$(ls -lh $file_to_test | awk '{print $5}')"
                file_size_info=$(ls -lh $file_to_test | awk '{print $5}')
                file_size_number=${file_size_info:0:-1}
                file_size_unit=${file_size_info: -1}

                # 单位为G的，转换为M
                [ $file_size_unit = "G" ] && { file_size_unit='M'; file_size_number=$( echo "$file_size_number * 1024" | bc ) ; }
                start_time=$(date +%"s")
                if [ "$mode" = "cp" ]; then
                        echo -n  "cp $file_to_test  ....."
                        cp $file_to_test $target_dir && sync
                        echo 1 > /proc/sys/vm/drop_caches  &>/dev/null
                        end_time=$(date +%"s")

                fi

                if [ $mode = 'dd' ]; then
                        echo -n  "dd $file_to_test ....."
                        dd if=$file_to_test of=$target_dir/$file_to_test conv=fsync && sync
                        echo 1 > /proc/sys/vm/drop_caches &>/dev/null
                        end_time=$(date +%"s")
                        d_md5=$(md5sum $target_dir/$file_to_test |awk '{print $1}'|tr '[a-z]' '[A-Z]')

                fi

                duration=$((end_time - start_time))
                [ $duration -eq 0 ] && duration=1

                speed=$(echo "scale=2; $file_size_number / $duration" | bc )
                echo  -n "speed = $speed $file_size_unit/s"
                cmp $file_to_test $target_dir/$file_to_test &&  echo -e "\033[32m SUCCESSED \033[0m" ||  echo -e "\033[31m FAILED \033[0m"
                rm -f $target_dir/$file_to_test && sync
                echo $"rm -f $target_dir/$file_to_test"
        echo 1 > /proc/sys/vm/drop_caches &>/dev/null

        done
}


#for i in $(seq(1 $loop_time))
for i in $(seq 1 $loop_time)
do
        echo "==== 当前第 $i 次测试 ======"
        do_test

done

 阶段八：最终优化

#!/bin/sh

source_dir='/root/sd/p3/'
target_dir='/root/emmc/p4/test_read_write'

[ -z "$1" ] && loop_time=100 || loop_time=$1  # 测试次数

[ -n "$2" ] && mode=$2 || mode='cp'

[ ! -d $target_dir ] && mkdir -p $target_dir

function msg_red()
{
    echo -e "\033[31m $1 \033[0m"
}

function msg_green()
{
    echo -e "\033[32m $1 \033[0m"
}

function msg_blue()
{
    echo -e "\033[34m $1 \033[0m"
}


function do_test_use_MD5()
{

    cd $source_dir
    for file_to_test in *
    do
        s_md5=$(echo $file_to_test|awk -F '[-.]' '{print $2}'  | tr '[a-z]' '[A-Z]' )  # 拿出原文件md5
        #echo "file_size_info=$(ls -lh $file_to_test | awk '{print $5}')"
        file_size_info=$(ls -lh $file_to_test | awk '{print $5}')
        file_size_number=${file_size_info:0:-1}
        file_size_unit=${file_size_info: -1}

        # 单位为G的，转换为M
        [ $file_size_unit = "G" ] && { file_size_unit='M'; file_size_number=$( echo "$file_size_number * 1024" | bc ) ; }
        start_time=$(date +%"s")
        if [ "$mode" = "cp" ]; then
                echo -n  "cp $file_to_test to  $target_dir ....."
                cp $file_to_test $target_dir && sync || echo "cp $file_to_test failed!"
                echo 3 > /proc/sys/vm/drop_caches  &>/dev/null
                end_time=$(date +%"s")
                d_md5=$(md5sum $target_dir/$file_to_test |awk '{print $1}'| tr '[a-z]' '[A-Z]')
                [ $s_md5 = $d_md5 ] && echo -n " cp successed!" || echo "md5sum not match ($s_md5, $d_md5), cp failed!"
        fi

        if [ $mode = 'dd' ]; then
                echo -n  "dd $file_to_test to  $target_dir ....."
                dd if=$file_to_test of=$target_dir/$file_to_test conv=fsync && sync || echo "dd  $file_to_test failed!"
                echo 3 > /proc/sys/vm/drop_caches &>/dev/null
                end_time=$(date +%"s")
                d_md5=$(md5sum $target_dir/$file_to_test |awk '{print $1}'|tr '[a-z]' '[A-Z]')
                [ $s_md5 = $d_md5 ] && echo -n "dd successed!" || echo "md5sum not match ($s_md5, $d_md5), dd failed!"
        fi

        duration=$((end_time - start_time))
        [ $duration -eq 0 ] && duration=1
        #echo "$file_size_number / $duration"
        speed=$(echo "scale=2; $file_size_number / $duration" | bc )
        echo "speed = $speed $file_size_unit/s"
        rm -f $target_dir/$file_to_test

    done
}



function do_test()
{
        [ -n "$1" ] && test_time=$1 || test_time=10 # 每个文件测试多少次，默认10次
        cd $source_dir
        for file_to_test in *
        do
                #s_md5=$(echo $file_to_test|awk -F '[-.]' '{print $2}'  | tr '[a-z]' '[A-Z]' )  # 拿出原文件md5
                #echo "file_size_info=$(ls -lh $file_to_test | awk '{print $5}')"
                test_total=0
                test_pass=0
                test_fail=0
                
                
                file_size_info=$(ls -lh $file_to_test | awk '{print $5}')
                file_size_number=${file_size_info:0:-1}
                file_size_unit=${file_size_info: -1}

                # 单位为G的，转换为M
                [ $file_size_unit = "G" ] && { file_size_unit='M'; file_size_number=$( echo "$file_size_number * 1024" | bc ) ; }
                    
                for i in $(seq 1 $test_time)
                do
                    
                    start_time=$(date +%"s")
                    if [ "$mode" = "cp" ]; then
                            echo -n  "当前第$i次测试: cp $file_to_test  ....."
                            cp $file_to_test $target_dir && sync
                            sync
                            echo 1 > /proc/sys/vm/drop_caches  &>/dev/null
                            end_time=$(date +%"s")
                    fi

                    if [ $mode = 'dd' ]; then
                            echo -n  "当前第$i次测试: dd $file_to_test ....."
                            dd if=$file_to_test of=$target_dir/$file_to_test conv=fsync && sync
                            sync
                            echo 3 > /proc/sys/vm/drop_caches &>/dev/null
                            end_time=$(date +%"s")
                    fi

                    duration=$((end_time - start_time))
                    [ $duration -eq 0 ] && duration=1

                    speed=$(echo "scale=2; $file_size_number / $duration" | bc )
                    echo  -n "speed = $speed $file_size_unit/s"
                    result=$(cmp $file_to_test $target_dir/$file_to_test)
                    if [ $? -eq 0 ]; then
                        msg_green " SUCCESSED! "
                        test_pass=$((test_pass+1))
                    else
                        msg_red "FAILED! -> $result "
                        test_fail=$((test_fail+1))
                    fi
                    
                    test_total=$((test_total+1))
                    rm -f $target_dir/$file_to_test && sync
                    sync
                    echo 3 > /proc/sys/vm/drop_caches &>/dev/null
                done
                msg_blue "====== $file_to_test 一共测试$test_total次，成功$test_pass次，失败$test_fail次 ======"
        done
}


#for i in $(seq(1 $loop_time))
for t in $(seq 1 $loop_time)
do
        echo ">>>>>>>>>>>>>>>>>> 当前第 $t 轮测试 <<<<<<<<<<<<<<<<<<<<<<<<<<<<"
        do_test 10

done