python网站访问统计
#!/usr/local/bin/python3 # coding:utf-8 # ==================================================== # Author: chang - EMail:changbo@hmg100.com # Last modified: 2017-5-13 # Filename: accesstimes.py # Description: real time analysis nginx log,base time, os, re, pymysql, Thread # blog:http://www.cnblogs.com/changbo # ==================================================== """ 需求:每隔1分钟读取nginx日志文件 notice: 模拟日志切割过程中初始化脚本参数 cp access.log access2017xxxx.log && echo > access.log && echo '0'> offset1.txt """ import time import os import re import pymysql from threading import Thread import logging # from django.db import connection logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s', datefmt='%a, %d %b %Y %H:%M:%S', filename='accesstimes.log', filemode='w') keypage = ['/sys/get_user.do'] keyIP = ['127.0.0.1', '119.28.61.122'] engdate = {'Jan': '1', 'Feb': '2', 'Mar': '3', 'Apr': '4', 'May': '5', 'Jun': '6', 'Jul': '7', 'Aug': '8', 'Sept': '9', 'Oct': '10', 'Nov': '11', 'Dec': '12'} def dateformat(nginxdate): day = (nginxdate.split('[')[1]).split('/')[0] month = engdate[(nginxdate.split('[')[1]).split('/')[1]] year = (nginxdate.split('[')[1]).split('/')[2] return year + '-' + month + '-' + day # write log offset def writeoffset(number): with open('offset1.txt', 'w+') as f3: f3.write(number) f3.flush() # get log offset def getoffset(): with open('offset1.txt') as f2: offset = f2.readline() return offset db = pymysql.connect("xxx.xxx.xxx.xxx", "xxx", "xxxxxx", "yunwei") cursor = db.cursor() # cleantable = 'TRUNCATE abnormal' def handleline(logline): susptmp = logline.split(" ") if len(susptmp) > 2: if susptmp[0] not in keyIP: del susptmp[1:3] if len(susptmp) > 2: ip = susptmp[0] time1 = (susptmp[1].split(':', 1))[1] dated = dateformat((susptmp[1].split(':', 1))[0]) # print(ip + '---', time1 + '---', date + '---', user + '---', passd + '---') sql = "INSERT INTO reqinfo(ip, timedd, datedd) VALUES('%s', '%s', '%s')" % (ip, time1, dated) try: cursor.execute(sql) db.commit() logging.debug('Insert success!') except Exception as e: logging.debug(e) # online analysis log def analysislog(): with open('access.log') as f1: while True: # get offset lastoffset = getoffset() # jump the Specify log line f1.seek(int(lastoffset)) # 获取该行偏移量 where = f1.tell() line = f1.readline() writeoffset(str(where)) if not line: time.sleep(10) f1.seek(where) else: # 处理该行,并获取改行的偏移量且写入文件 handleline(line) nowoffset = f1.tell() writeoffset(str(nowoffset)) if __name__ == '__main__': if not os.path.exists('offset1.txt'): with open("offset1.txt", 'w') as f: f.write('0') t1 = Thread(target=analysislog) t1.start()
#!/usr/local/bin/python3 # coding:utf-8 # ==================================================== # Author: chang - EMail:changbo@hmg100.com # Last modified: 2017-5-13 # Filename: reqcounts.py # Description: real time analysis nginx log,pymysql, Thread, logging # blog:http://www.cnblogs.com/changbo # ==================================================== import pymysql import logging from threading import Thread logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s', datefmt='%a, %d %b %Y %H:%M:%S', filename='reqcounts.log', filemode='w') db = pymysql.connect("xxx.xxxx.xxx.xxx", "xxxx", "xxxxx", "xxxxx") cursor = db.cursor() # cleantable = 'TRUNCATE tablename' def analysisdb(): listtime = [] listip = [] listdated = [] sql3 = 'SELECT ip, timedd, datedd FROM reqinfo' cursor.execute(sql3) results = cursor.fetchall() for row in results: listtime.append(row[1]) listip.append(row[0]) listdated.append(row[2]) try: # 统计1分钟内页面访问次数 sql1 = "SELECT count(*) from reqinfo where timedd='%s' and ip='%s' and datedd='%s'" % ( listtime[0], listip[0], listdated[0]) sql4 = "DELETE from reqinfo where timedd='%s' and ip='%s' and datedd='%s'" % ( listtime[0], listip[0], listdated[0]) sql5 = "DELETE FROM reqcounts WHERE timesddd=0" cursor.execute(sql1) datad = cursor.fetchone() sql2 = "INSERT INTO reqcounts(ip, timesddd, timeddd, dateddd) VALUES('%s' , '%s' , '%s', '%s')" % ( listip[0], datad[0], listtime[0], listdated[0]) cursor.execute(sql2) db.commit() logging.debug('-----Insert success -------') # delete already insert data of requinfo cursor.execute(sql4) db.commit() del listtime[0] del listip[0] del listdated[0] cursor.execute(sql5) db.commit() except Exception as e: logging.debug(e) if __name__ == '__main__': t2 = Thread(target=analysisdb) t2.start()
html 页面
<!DOCTYPE html> <html lang="en"> <head> <meta charset="UTF-8"> {% load static %} <link rel="stylesheet" type="text/css" href="{% static "css/style.css" %}" /> <title>访问统计</title> <script language="javascript" type="text/javascript"> window.onload = function(){ var oTable = document.getElementById("bbsTab"); for(var i=0;i<oTable.rows.length;i++){ oTable.rows[i].cells[0].innerHTML = (i+1); if(i%2==0) //偶数行 oTable.rows[i].className = "ys01"; } } </script> </head> <body> <div id="container"> <table class="zebra"> <caption>IP/s 并发量统计</caption> <thead> <tr> <th>序号</th> <th>ip</th> <th>登录次数</th> <th>时间</th> <th>日期</th> </tr> </thead> <tbody id="bbsTab"> {% for reqinfocount in reqinfocounts %} <tr> <td></td> <td>{{ reqinfocount.ip|safe }}</td> <td>{{ reqinfocount.timesddd|safe }}</td> <td>{{ reqinfocount.timeddd|safe }}</td> <td>{{ reqinfocount.dateddd|safe }}</td> </tr> {% endfor %} </tbody> </table> </div> </body> </html>
效果图
END!