用shell实现linux系统应用文件清理工具
用shell实现linux系统文件清理工具
1：原始需求
在系统运维中，会产生大量应用备份文件、落地文件等，这些文件需要定时清理。一般来说，都是使用crontab 拉起一个脚本来清理。类似这样：
30 0,12 * * * find /data02/rating_bak/loading -type f -name "gprs_ol*gz" -atime +1 -exec rm {} \;
但是如果机器很多，目录多，这种方式非常不利于管理，而且也没有一个清晰的配置，来知道什么机器上哪个目录的文件留存时间。这个工具用于在数据库中存一个配置，然后在每台机器上，配置一个定时脚本执行即可。
实现的基本方式为：用一个shell从数据库中读取配置，根据文件的最后修改时间，大于X的mv到带日期的子目录中，大于Y的压缩，大于Z的删除。用crontab拉起这个脚本。
2：数据库表配置
create table WH_BAKFILE_CFG
(
BAKID NUMBER, --主键ID
IP VARCHAR2(30), --IP地址
SCANPWD VARCHAR2(300), --扫描目录
BAKPWD VARCHAR2(300), --备份目录
FILECH VARCHAR2(100), --文件特征
MVTIME NUMBER, --文件移动时间 单位分
ZIPTIME NUMBER, --文件压缩时间 单位分
DELTIME number, --文件删除时间 单位分
Needcheck number --是否需要校验文件被占用 1需要校验
)
配置举例：
|
BAKID |
IP |
SCANPWD |
BAKPWD |
FILECH |
MVTIME |
ZIPTIME |
DELTIME |
NEEDCHECK |
|
91 |
172.20.31.24 |
/data02/rating_bak/upcheck/migrate_sd |
|
* |
1440 |
2880 |
46080 |
|
|
10 |
172.20.31.24 |
/app/billapp/user/wxk/mybak |
|
* |
1440 |
2880 |
8640 |
|
|
11 |
172.20.31.24 |
/data02/rating_bak/upfile/d |
|
* |
1440 |
5760 |
46080 |
|
|
13 |
172.20.31.24 |
/data02/rating_bak/upfile/q |
|
* |
1440 |
5760 |
46080 |
|
|
14 |
172.20.31.24 |
/data02/rating_bak/upfile/vpmn |
|
* |
1440 |
5760 |
46080 |
|
3：相关代码
A：读取配置，拉起处理shell。
从数据库中获取配置，写入${IP}.cfg，生成$(bakid).lock，避免处理shell被多次拉起，然后拉起处理shell。
#!/bin/sh
CONNSTR="conn dz/dz_xxxxx@nbilldb"
LOGFILE=./getbakinfo.log
##得到本机IP
IP=""
getip()
{
if [[ x"${IP}" = x ]];then
IP=`/sbin/ifconfig |grep "inet addr"| cut -f 2 -d ":" |sed -n '1p' |awk '{print $1}'`
fi
}
##记录日志
writelog()
{
echo `date +%Y-%m-%d" "%X` "$1" >>${LOGFILE}
}
##SQL
selectsql()
{
sqlplus -S /nolog <<EOF
set heading off feedback off pagesize 0 verify off echo off linesize 3000
${CONNSTR}
${1};
commit;
exit
EOF
}
MYWORKSHELL=$0
if [[ ${MYWORKSHELL} = "getbakinfo.sh" ]] ; then
MYWORKPATH=.
else
MYWORKPATH=${MYWORKSHELL%\/*}
fi
cd $MYWORKPATH
##判断连接
sql="select 'HELLOSMK' from dual"
selectresult=`selectsql "${sql}"`
if [[ ! "${selectresult}" = "HELLOSMK" ]] ; then
writelog "error 连接数据库失败!:${selectresult}"
else
##写配置文件
getip
rm ${IP}.cfg
sql="select trim(bakid)||'##'||trim(t.scanpwd)||'##'||trim(t.bakpwd)||'##'||trim(t.filech)||'##'||t.mvtime||'##'||t.ziptime||'##'||t.deltime||'##'||nvl(t.needcheck,0) \
from wh_bakfile_cfg t where t.ip ='${IP}' "
#selectsql "${sql}"
#IFS=$'\x0A'
selectresult=`selectsql "${sql}"`
##逐条处理
for result in ${selectresult}
do
bakid=`echo ${result} |awk -F "##" '{print $1}'`
scanpwd=`echo ${result} |awk -F "##" '{print $2}'`
bakpwd=`echo ${result} |awk -F "##" '{print $3}'`
filech=`echo ${result} |awk -F "##" '{print $4}'`
mvtime=`echo ${result} |awk -F "##" '{print $5}'`
ziptime=`echo ${result} |awk -F "##" '{print $6}'`
deltime=`echo ${result} |awk -F "##" '{print $7}'`
needcheck=`echo ${result} |awk -F "##" '{print $8}'`
echo ${bakid},${scanpwd},${bakpwd},${filech},${mvtime},${ziptime},${deltime},${needcheck}>>${IP}.cfg
done
fi
cat ${IP}.cfg |while read filestr
do
bakid=`echo ${filestr} |awk -F "," '{print $1}'`
scanpwd=`echo ${filestr} |awk -F "," '{print $2}'`
bakpwd=`echo ${filestr} |awk -F "," '{print $3}'`
filech=`echo ${filestr} |awk -F "," '{print $4}'`
mvtime=`echo ${filestr} |awk -F "," '{print $5}'`
ziptime=`echo ${filestr} |awk -F "," '{print $6}'`
deltime=`echo ${filestr} |awk -F "," '{print $7}'`
needcheck=`echo ${filestr} |awk -F "," '{print $8}'`
##目录检查
if [ -d "${scanpwd}" ] ; then
##锁检查
if [ ! -f "${bakid}.lock" ]; then
touch "${bakid}.lock"
nohup ${MYWORKPATH}/whbakfile.sh "${bakid}" "${scanpwd}" "${bakpwd}" "${filech}" "${mvtime}" "${ziptime}" "${needcheck}" "${MYWORKPATH}" "${deltime}" >> ${LOGFILE} 2>&1 &
fi
else
writelog "error目录不存在!:${scanpwd}"
fi
done
B:文件处理shell
#!/bin/sh
ddtime=`date +%Y%m%d`
LOGFILE=$HOME/log/bakfile/bakfile${ddtime}.log
bakid=$1
scanpwd=$2
bakpwd=$3
filech=$4
mvtime=$5
ziptime=$6
needcheck=$7
MYWORKPATH=$8
deltime=$9
cd $MYWORKPATH
##记录日志
writelog()
{
echo `date +%Y-%m-%d\ %H:%M:%S` "$1" >>${LOGFILE}
}
cd $MYWORKPATH
if [[ x"${bakpwd}" = x ]];then
bakpwd=${scanpwd}
fi
if [[ x"${deltime}" = x ]];then
deltime=9999999
fi
if [[ x"${ziptime}" = x ]];then
ziptime=9999999
fi
if [[ x"${filech}" = x ]];then
filech="*"
fi
###mv
find ${scanpwd} -maxdepth 1 -type f -name "${filech}" -mmin +${mvtime} |while read filename
do
filetime=`ls -l ${filename} --time-style '+%Y%m%d'|awk '{print $6}'`
if [ ! -d "${bakpwd}/${filetime}" ]; then
mkdir -p ${bakpwd}/${filetime}
fi
##check
checkstat=0
if [[ ${needcheck} -eq 1 ]] ; then
checkstat=`lsof|grep filename |wc -l`
fi
if [[ ${checkstat} -eq 0 ]] ; then
mv ${filename} ${bakpwd}/${filetime}
writelog "mv ${filename} ${bakpwd}/${filetime}"
fi
done
###gzip
if [[ ! ${ziptime} -eq 9999999 ]] ; then
find ${bakpwd} -type f -name "${filech}" -mmin +${ziptime}|grep -v .gz|grep ${bakpwd}/20|while read filename
do
gzip ${filename}
writelog "gzip ${filename}"
done
fi
###rm
if [[ ! ${deltime} -eq 9999999 ]] ; then
find ${bakpwd} -type f -name "${filech}" -mmin +${deltime}|grep ${bakpwd}/20|while read filename
do
rm ${filename}
writelog "rm ${filename}"
done
fi
##rmdir
if [[ ! ${deltime} -eq 9999999 ]] ; then
find ${bakpwd} -type d |grep ${bakpwd}/20|while read pathname
do
pathnum=`ls ${pathname} | wc -l`
if [[ ${pathnum} -eq 0 ]] ; then
rmdir ${pathname}
writelog "rmdir ${pathname}"
fi
done
fi
rm ${bakid}.lock
