使用Python对数据备份文件进行整理

最近公司上了一个转储服务器，用于存储所有应用系统的数据文件备份，由于每天的备份文件都放在一个固定的文件夹，不需要的文件也会传过来，日后不方便整理，于是便使用Python脚本，将需要的每个文件每日进行归类整理。

一、配置文件格式

{
    "otcdDBBAK": {
        "foldername": "otcdDBBAK",               --源文件目录最后一级的文件夹名
        "backupType": 0,                         --备份类型（0-每周一至周五备份；1-每天备份；2、每周一至周六备份）
        "src_path": "/home/oracle/dmpbak/",      --源文件需要整理的文件夹
        "target_path": "/home/oracle/backup/",   --目标存放的目录
        "fileName": "otc_yyyymmdd",              --文件名
        "fileExtension": [                       --文件后缀
            ".tar.gz"
        ]
    }
}

二、脚本内容

脚本中，将需要的执行信息输出到执行日志中，便于后面监控。

脚本执行目的：

1、对源文件进行整理

2、对部分文件进行MD5码值校验

#!/usr/bin/python
# -*- coding: utf-8 -*-
from datetime import datetime
from datetime import timedelta
import json
import os
import subprocess
from glob import glob
import hashlib


# 获取前一天的日期，获取两种格式数据，yyyymmdd和yyyymm
def getYesterday():
    today = datetime.today()
    oneday = timedelta(days=1)
    yesterday1 = (today - oneday).strftime('%Y%m%d')
    yesterday2 = (today - oneday).strftime('%Y%m')
    return yesterday1, yesterday2


def check_md5_file_list(files):
    # files输入的是一个字典，获取字典中fileExtension 包含.md5后缀的元素，并组成一个新的字典
    value = '.*' + 'md5'
    # 获取需要校验的文件字典集合
    for i in list(files):
        import re
        for f in files[i]["fileExtension"]:
            result = re.findall(value, f)
        if len(result) == 0:
            del files[i]
    return files


def get_md5(file):
    # 计算文件的MD5值，返回一个字符串
    md5 = hashlib.md5()
    with open(file, 'rb') as fobj:
        while True:
            data = fobj.read(4096)
            if not data:
                break
            md5.update(data)
    return md5.hexdigest()


def read_md5(file):
    # 读取文件的数据，返回一个字符串
    with open(file, 'r') as data:
        rd = data.read()
        md5 = rd[0:32]
    return md5


def get_src_file(cf):
    # 根据备份策略
    # week 取值为0-6 表示周一到周日
    # 当week 取值为0时，获取conf/config,json 中{backupType}为1的数据
    # 当week 取值为6时，获取conf/config.json 中{backupType}不为0的数据
    week = datetime.weekday(datetime.today())
    with open(cf, 'r') as f:
        json_data = json.load(f)
        for i in list(json_data):
            if week == 0 and json_data[i]["backupType"] != 1:
                del json_data[i]
                continue
            elif week == 6 and json_data[i]["backupType"] == 0:
                del json_data[i]
                continue
            else:
                continue
    return json_data


def mv_file_everyday(files, logfile, yd1, yd2):
    # 遍历新的字典，获取源文件和目标目录，执行创建文件夹的命令以及mv文件的命令
    for i in files:
        target_path = files[i]["target_path"] + \
            files[i]["foldername"] + '/' + yd2 + '/'
        # 目标文件夹不存在是需要创建
        if not os.path.exists(target_path):
            os.makedirs(target_path)
            warn = datetime.now().strftime("%Y-%m-%d %H:%M:%S ") + \
                "daily mv task warnning: Folder created: " + '"'+target_path+'"'
            print(warn)
            subprocess.call(f"echo {warn} >> {logfile}", shell=True)
        # 开始处理源文件
        for j in files[i]["fileExtension"]:
            src_file = (files[i]["src_path"] + files[i]["foldername"] +
                        '/' + files[i]["fileName"] + j).replace("yyyymmdd", yd1)
            try:
                try:
                    # 1、src_file 中有通配符的话，需要进行转换
                    if '*' in src_file:
                        src_file = glob(src_file)[0]
                except Exception as e:
                    error = datetime.now().strftime("%Y-%m-%d %H:%M:%S ") + \
                    " daily mv task error: Execution failed " + str(e) + src_file
                    subprocess.call(f"echo {error} >> {logfile}", shell=True)
                # 2、判断源文件是否存在，如果不存在则将信息写入日志，如需监控则从日志中提取信息
                if not os.path.exists(src_file):
                    error = datetime.now().strftime("%Y-%m-%d %H:%M:%S ") + \
                        "daily mv task error srcfile: no such file: " + '"'+src_file+'"'
                    subprocess.call(f"echo {error} >> {logfile}", shell=True)
                    # subprocess.call(['touch', src_file])
                # 3、开始整理文件操作，将源文件mv 到目标文件夹中
                # 不直接mv，分开使用cp + rm 的方式，通过分析日志，可以便于mv文件失败后，辨别正确的文件
                else:
                    try:
                        subprocess.call(["cp", src_file, target_path])
                        success_cp = datetime.now().strftime("%Y-%m-%d %H:%M:%S ") + \
                            "daily mv task successfully copy file : " + '"'+src_file+'"'
                        subprocess.call(
                            f"echo {success_cp} >> {logfile}", shell=True)
                        subprocess.call(["rm", "-f", src_file])
                        success_rm = datetime.now().strftime("%Y-%m-%d %H:%M:%S ") + \
                            "daily mv task successfully delete files : " + '"'+src_file+'"'
                        subprocess.call(
                            f"echo {success_rm} >> {logfile}", shell=True)
                    except Exception as e:
                        error = datetime.now().strftime("%Y-%m-%d %H:%M:%S ") + \
                            "daily mv task error: Execution failed: " + str(e)
                        subprocess.call(
                            f"echo {error} >> {logfile}", shell=True)
            except Exception as s:
                error = datetime.now().strftime("%Y-%m-%d %H:%M:%S ") + \
                    " daily mv task error: Execution failed: " + str(s) + src_file
                subprocess.call(f"echo {error} >> {logfile}", shell=True)
    return


def check(Files):
    for j in Files:
        for fx in Files[j]["fileExtension"]:
            target_file = (Files[j]["target_path"] + Files[j]["foldername"] +
                           '/' + yd2 + '/' + Files[j]["fileName"] + fx).replace("yyyymmdd", yd1)
            try:
                if '*' in target_file:
                    target_file = glob(target_file)[0]
            except Exception as e:
                error = datetime.now().strftime("%Y-%m-%d %H:%M:%S ") + \
                    " check md5 task error: Execution failed " + str(e) + target_file
                subprocess.call(f"echo {error} >> {logfile}", shell=True)
            # 由于md5Files列表中，既包含源文件，也包含其.md5文件
            # 这里需要先删选出源文件，计算源文件的md5值
            # 再根据源文件名拼接得出其md5文件名，进行读取操作
            if not target_file.endswith('.md5'):
                # 首先判断文件是否存在
                if os.path.exists(target_file):
                    check_result = get_md5(target_file)
                    target_file_md5 = target_file + '.md5'
                    md5_result = read_md5(target_file_md5)
                    # 判断md5值是否一致
                    if check_result == md5_result:
                        check_md5_yes = datetime.now().strftime("%Y-%m-%d %H:%M:%S ") + \
                            ' check md5 task successfully: md5_consistent in ' + '"'+j+'"'
                        subprocess.call(
                            f"echo {check_md5_yes} >> {logfile}", shell=True)
                    else:
                        check_md5_no = datetime.now().strftime("%Y-%m-%d %H:%M:%S ") + \
                            ' check md5 task error: md5_inconsistent in ' + '"'+j+'"'
                        subprocess.call(
                            f"echo {check_md5_no} >> {logfile}", shell=True)
                else:
                    error_check_md5file = datetime.now().strftime("%Y-%m-%d %H:%M:%S ") + \
                        ' check md5 task error: file missing in ' + '"'+j+'"'
                    subprocess.call(
                        f"echo {error_check_md5file} >> {logfile}", shell=True)
            else:
                # print(target_file)
                continue
    return


if __name__ == '__main__':
    logfile = '/home/oracle/task/PYbackup/conf/log.log'
    configfile = '/home/oracle/task/PYbackup/conf/config.json'
    yd1, yd2 = getYesterday()
    # 从json 文件中获取今天需要整理的文件信息
    files = get_src_file(configfile)
    # 开始执行每日整理操作
    dailyTaskMv = datetime.now().strftime("%Y-%m-%d %H:%M:%S ") + "daily mv task begin......"
    subprocess.call(f"echo {dailyTaskMv} >> {logfile}", shell=True)
    mv_file_everyday(files, logfile, yd1, yd2)
    task_finished = datetime.now().strftime("%Y-%m-%d %H:%M:%S ") + \
        "mv file everyday task finished!"
    subprocess.call(f"echo {task_finished} >> {logfile}", shell=True)

    dailyTaskCheckMd5 = datetime.now().strftime("%Y-%m-%d %H:%M:%S ") + "daily check md5 task begin......"
    subprocess.call(f"echo {dailyTaskCheckMd5} >> {logfile}", shell=True)
    # 获取需要校验MD5值的元素信息
    md5Files = check_md5_file_list(files)
    # 开始校验md5值
    check(md5Files)
    check_md5_finished = datetime.now().strftime("%Y-%m-%d %H:%M:%S ") + \
        "check md5 task finished!"
    subprocess.call(f"echo {check_md5_finished} >> {logfile}", shell=True)

三、zabbix对执行进行监控

# 1、监控脚本：
$ cat zabbix_check.sh
#!/bin/sh
datetime=$(date +%F)
cat $1 | grep $datetime | grep "$2" | wc -l

# 2、zabbix配置文件配置
cat /etc/zabbix_agent.d/userparameter.conf
UnsafeUserParameters=1
UserParameter=logcheck[*],sh zabbix_check.sh $1 "$2"

# 3、zabbix前台新增监控项，设置键值
logcheck[/home/oracle/task/PYbackup/conf/log.log,"error"]