centos7环境下shell脚本对nginx日志过滤实现恶意ip的封堵

centos7环境下shell脚本对nginx日志过滤实现恶意ip的封堵

shell + iptables实现恶意访问ip的封堵

shell脚本读取nginx的日志，对最近的5分钟ip进行过滤，如果某个ip访问特定接口超过一定次数就把这个ip拉黑

1.编写监控nginx日志脚本

[root@web:/usr/local/worksh/monitor_ip]# cat monitor_ip.sh
#!/bin/bash
#
# 日志日期格式：[17/Jul/2020:06:10:

# 存放过去5分钟临时日志
filter_log="/data/www/logs/last_filve_min_access.log"
current_log="/data/www/logs/nginx_log/access/my.chinasoft.com_access.log"

# 每次都需要清理一次，否则越累积越多
echo ''> ${filter_log}


# 这是刚开始的写法，这样会有一个问题，当到小时、天、月、年的分界点时会漏掉其中的5分钟日志
function filter_last_five_min_log
{
    # 将过去5分钟的日志单独写入一个文件中
    # # Fri Jul 17 06:17:35 PDT 2020
    day=`date +%d`
    month=`env LANG=en_US.UTF-8 date|awk '{print $2}'`
    year=`env LANG=en_US.UTF-8 date|awk '{print $6}'`
    current_hour=`date +%H`

    # 当前开始，5分钟内的时间
    current_min=`date +%M`
    last_one_min=`date -d "1 minute ago" +"%M"`
    last_two_min=`date -d "2 minute ago" +"%M"`
    last_three_min=`date -d "3 minute ago" +"%M"`
    last_four_min=`date -d "4 minute ago" +"%M"`
    last_five_min=`date -d "5 minute ago" +"%M"`
    for min_num in $current_min $last_one_min $last_two_min $last_three_min $last_four_min $last_five_min;do
        if [ $min_num != $current_min ];then
        /bin/cat ${current_log} |grep "/chinasoft/login?ramdom="| grep "\[${day}/${month}/${year}:${current_hour}:${min_num}" >> ${filter_log}
        fi
    done
}


function current_minute_log
{
    local day=`date +%d`
    local month=`date +%b`
    local year=`date +%Y`
    local current_hour=`date +%H`
    local min_num=`date +%M`
    /bin/cat ${current_log} |grep "/chinasoft/login?ramdom="| grep "\[${day}/${month}/${year}:${current_hour}:${min_num}" >> ${filter_log}
}

function one_minute_log
{
    local day=`date -d "1 minute ago" +"%d"`
    local month=`date -d "1 minute ago" +"%b"`
    local year=`date -d "1 minute ago" +"%Y"`
    local current_hour=`date -d "1 minute ago" +"%H"`
    local min_num=`date -d "1 minute ago" +"%M"`
    /bin/cat ${current_log} |grep "/chinasoft/login?ramdom="| grep "\[${day}/${month}/${year}:${current_hour}:${min_num}" >> ${filter_log}
}

function two_minute_log
{
    local day=`date -d "2 minute ago" +"%d"`
    local month=`date -d "2 minute ago" +"%b"`
    local year=`date -d "2 minute ago" +"%Y"`
    local current_hour=`date -d "2 minute ago" +"%H"`
    local min_num=`date -d "2 minute ago" +"%M"`
    /bin/cat ${current_log} |grep "/chinasoft/login?ramdom="| grep "\[${day}/${month}/${year}:${current_hour}:${min_num}" >> ${filter_log}
}

function three_minute_log
{
    local day=`date -d "3 minute ago" +"%d"`
    local month=`date -d "3 minute ago" +"%b"`
    local year=`date -d "3 minute ago" +"%Y"`
    local current_hour=`date -d "3 minute ago" +"%H"`
    local min_num=`date -d "3 minute ago" +"%M"`
    /bin/cat ${current_log} |grep "/chinasoft/login?ramdom="| grep "\[${day}/${month}/${year}:${current_hour}:${min_num}" >> ${filter_log}
}

function four_minute_log
{
    local day=`date -d "4 minute ago" +"%d"`
    local month=`date -d "4 minute ago" +"%b"`
    local year=`date -d "4 minute ago" +"%Y"`
    local current_hour=`date -d "4 minute ago" +"%H"`
    local min_num=`date -d "4 minute ago" +"%M"`
    /bin/cat ${current_log} |grep "/chinasoft/login?ramdom="| grep "\[${day}/${month}/${year}:${current_hour}:${min_num}" >> ${filter_log}
}

function five_minute_log
{
    local day=`date -d "5 minute ago" +"%d"`
    local month=`date -d "5 minute ago" +"%b"`
    local year=`date -d "5 minute ago" +"%Y"`
    local current_hour=`date -d "5 minute ago" +"%H"`
    local min_num=`date -d "5 minute ago" +"%M"`
    /bin/cat ${current_log} |grep "/chinasoft/login?ramdom="| grep "\[${day}/${month}/${year}:${current_hour}:${min_num}" >> ${filter_log}
}

function count_last_logs
{
    # 将过去5分钟撞库的日志ip记录下来，并统计次数
    malice_access_ips="/data/www/logs/malice_access_ips.log"
    /bin/cat ${filter_log} |grep "/chinasoft/login?ramdom="|awk '{print $1}'|sort -r|uniq -c|sort -r -n|head -300 > ${malice_access_ips}
}


function count_filter_ip
{
    # 读取统计的ip次数，如果一个ip撞库次数超过100次，则防火墙拉黑
    while read line
    do
    access_num=`echo $line|awk '{print $1}'`
    access_ip=`echo $line|awk '{print $2}'`
    # 
    if [[ $access_num -gt 100 ]];
    then
        drop_ip=`iptables -nvL |grep ${access_ip}|awk '{print $8}'`
        if [[ ${access_ip} = ${drop_ip} ]];then
            # 如果已经被拉黑则跳过这个ip
            continue
        fi
        
        company_ip=`grep ${access_ip} /usr/local/worksh/monitor_ip/company_ips.txt`
        if [[ ${access_ip} = ${company_ip} ]];then
            # 如果是公司IP跳过
            continue
        fi

        iptables -I INPUT -s ${access_ip} -j DROP
        echo "access_nu="${access_num} "access_ip="${access_ip}
    fi
    done < ${malice_access_ips}
}


function main
{

    # 筛选过去5分钟的日志
    current_minute_log
    one_minute_log
    two_minute_log
    three_minute_log
    four_minute_log
    five_minute_log

    # 统计ip的次数
    count_last_logs
    # 过滤符合要求的ip
    count_filter_ip
}

main

2.计划任务
# 每5分钟统计一次访问日志，如果5分钟内同一个ip撞库超过100次就拉黑这个ip地址
*/5 * * * * /bin/bash /usr/local/worksh/monitor_ip/monitor_ip.sh >/dev/null 2>&1

# 防火墙的示例
[root@web:/usr/local/worksh/monitor_ip]# iptables -nvL
Chain INPUT (policy DROP 1786 packets, 81061 bytes)
 pkts bytes target     prot opt in     out     source               destination         
   20  1124 DROP       all  --  *      *       1.1.1.1                0.0.0.0/0           

# 公司IP地址
[root@web:/usr/local/worksh/monitor_ip]# cat company_ips.txt
1.1.1.1
1.1.1.2
1.1.1.3

日志示例
[root@web:/usr/local/worksh/monitor_ip]# tail -n 5 /data/www/logs/nginx_log/access/my.chinasoft.com_access.log
103.147.118.3 - - [17/Jul/2020:07:10:00 -0700] - - "POST /chinasoft/login?ramdom= HTTP/1.1" 200 65 "http://my.chinasoft.com/user/login" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36" 0.223
36.71.239.228 - - [17/Jul/2020:07:10:00 -0700] - - "GET /static/jslibs/jquery.uid.js HTTP/1.1" 200 497 "https://pdf.chinasoft.com/how-to/convert-pdf-to-word.html?gclid=CjwKCAjwmMX4BRAAEiwA-zM4JmOMdq6wDpFgAOMdKNbJv1tHWCiJmDEHlKbVmUtobEFuHZQF7FP6rBoC360QAvD_BwE" "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36" 0.000
91.74.26.54 - - [17/Jul/2020:07:10:00 -0700] - - "GET /static/jslibs/jquery.uid.js HTTP/1.1" 200 497 "https://web.chinasoft.com/" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36 Edg/83.0.478.64" 0.000
46.8.247.3 - - [17/Jul/2020:07:10:00 -0700] - - "POST /chinasoft/login?ramdom= HTTP/1.1" 200 65 "http://my.chinasoft.com/user/login" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36" 0.320
14.116.204.164 - - [17/Jul/2020:07:10:00 -0700] - - "POST /chinasoft/login?ramdom= HTTP/1.1" 200 65 "http://my.chinasoft.com/user/login" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36" 1.609