对rsync进行封装的shell脚本

抓取

#!/bin/bash

. push.sh

# 错误处理:尝试查找备份文件
function onError()
{
        local errFile="err"
        local serverInfo=($1 $2 $3) # ip username password
        local logInfo=($4 $5)       # srcPath destPath

        local srcPath=${logInfo[0]}
        logInfo[0]="disk1/"${srcPath##*/}".gz" #备份文件路径
        logInfo[1]=${logInfo[1]}".gz"

        push ${serverInfo[*]} ${logInfo[*]} $errFile

        if [ -s $errFile ]; then
                echo "the file ${logInfo[0]} was not found"
        fi
}

# 抓取指定日期指定时间的日志文件
function catchLogByHour()
{
        local errFile="err"
        local serverInfo=($1 $2 $3) # serverInfo=(ip username password)
        local logInfo=($4 $5)       # logInfo=(srcPath destPath)
        local endHour=$6

        local i=0
        while [ $i -le $endHour ];
        do
                hour=$i
                if [ $hour -lt 10 ]; then
                        hour="0"$hour
                fi

                local hourLogInfo
                hourLogInfo[0]=${logInfo[0]}"."$hour
                hourLogInfo[1]=${logInfo[1]}"."$hour

                push ${serverInfo[*]} ${hourLogInfo[*]} $errFile

                if [ -s $errFile ]; then
                        onError ${serverInfo[*]} ${hourLogInfo[*]}
                fi

                i=`expr $[$i+1]`
        done
}

# for test
# logDate="20131229"
# serverInfo=([1]="172.18.11.200" [2]="log" [3]="wappplog")
# logInfo=([1]="logs/syslog/project_log."$logDate [2]="./data/project_log."$logDate [3]="2")
# catchLogByHour  ${serverInfo[*]} ${logInfo[*]}

 

推送

#!/bin/bash

MAX_RE_CONNECT=3
SLEEP_TIME=2

# 实际推送动作
function pushAction()
{
        local ip=$1
        local user=$2
        local password=$3

        local path=$4
        local localPath=$5
        local error=$6

        :>$error # 清空文件相当于rm,touch命令
        export RSYNC_PASSWORD="$password"
        rsync -Cavz $user@$ip::$path $localPath > $error 2>&1

        # 判断是否发送错误
        grep "rsync error" $error > $error"_copy"
        cp $error"_copy" $error
        rm -f $error"_copy"
}

# 失败重新尝试，一旦超过最大尝试次数认为无法完成
function push()
{

        local serverInfo=($1 $2 $3) # ip username password
        local logInfo=($4 $5)       # srcPath destPath
        local error=$6

        local i=0
        while [ $i -lt $MAX_RE_CONNECT ]
        do
                pushAction ${serverInfo[*]} ${logInfo[*]} $error

                # no error      
                if [ ! -s $error ]; then
                        break
                fi

                echo "try again..."
                sleep $SLEEP_TIME
                i=`expr $[$i+1]`
        done
}

# for test
# serverInfo=([1]="172.18.11.201" [2]="log" [3]="wappplog")
# fileInfo=([1]="logs/syslog/project_log.20131229.16" [2]="./lishujun.log" [3]="aaa")
# push ${serverInfo[*]} ${fileInfo[*]}

 

 main.sh

#! /bin/sh

cd `dirname $0`

. catch.sh

if [ $# -eq 0 ]; then
        # 取昨天数据
        logDate=`date -d"1 day ago" +'%Y%m%d'`
        hour=23
else
        # 取指定日期时间

        today=`date +'%Y%m%d'`
        logDate=$1
        if [ "$1" == "$today" ]; then
                # 取当天数据已产生的数据
                hour=`date -d "1 hour ago" +"%H"`
        else
                # 取历史数据
                hour=23
        fi
fi

mkdir -p "./logs/project/$logDate"
serverInfo=([1]="172.18.10.104" [2]="stat" [3]="3gppstat")
logInfo=([1]="logs/project/project_log."$logDate [2]="./logs/project/$logDate/project_log."$logDate [3]=$hour)
catchLogByHour  ${serverInfo[*]} ${logInfo[*]}