nginx日志统计相关命令和python脚本
src=(s1,s2,s3,s3)
for i in ${src[@]
nginx访问量统计
1.根据访问IP统计UV
awk '{print $1}' access.log|sort | uniq -c |wc -l
嘉图的统计 awk '{print $1}'
/var/log/nginx/access.log|sort | uniq -c |wc -l
2.统计访问URL统计PV
awk '{print $7}' access.log|wc -l
3.查询访问最频繁的URL
awk '{print $7}' access.log|sort | uniq -c |sort -n -k 1 -r|more
4.查询访问最频繁的IP
awk '{print $1}' access.log|sort | uniq -c |sort -n -k 1 -r|more
5.根据时间段统计查看日志
cat access.log| sed -n '/14\/Mar\/2015:21/,/14\/Mar\/2015:22/p'|more
6．根据时间点统计查看日志.
[root@hemlock nginx]# sed -n '/22\/Apr\/2016:10:10:00/,/22\/Apr\/2016:10:11:00/p' access.log|awk '{print $1,$6}'|sort|uniq -c |sort -nr|more
查看某个时间点的
7.查看服务器并发的
[root@hemlock php-fpm]# netstat -n | awk '/^tcp/ {++S[$NF]} END {for(a in S) print a, S[a]}'
[root@hemlock ~]# netstat -ant | grep ESTABLISHED|more
查看连接从哪里来
8．Mongo查看单个IP信息的语法
. db.access20160417.find({"remote_addr" : "222.64.69.202"})
10.在10.1.0.9查看这个IP有没登陆网站
mysql> SELECT * from garmentoffice.login_log where ip_address='5.9.98.178';
11显示主要信息
11[root@hemlock nginx]# grep "120.236" access.log |awk '{print $1,$5,$6,$8,$9,$10,$11,$12,$13,$14,$15}' |sort |uniq -c |sort -nr|more
1287 120.236.141.135 "GET /websocket/GetMemberInfo 403 0.002 "http://www.haodingdan.com/company-335380.html?spm=10001" "Mozilla/5.0 (Windows NT 6.1; WOW64)
显示了ip数和ip 显示了useragent 显示了utl 显示了主页 显示了状态 和显示request_time=0.002 GET是访问的uri 后面url是referer
12.统计目前访问最多的uri 和ip和其他参数
awk '{print $1,$5,$6,$8,$9,$10,$11,$12,$13,$14,$15}' access.log |sort |uniq -c |sort -nr|more
13,知道ip查找来自哪个机房域名的网站
http://ip.chinaz.com/?qq-pf-to=pcqq.group
14数据库监控负载的地址
http://10.1.0.12:8080/render/?width=1133&height=650&target=collectd.quivertree.load.load.shortterm&target=collectd.lime.load.load.shortterm&target=collectd.apple.load.load.shortterm&from=-8hours&fontSize=14
15mongo查看最慢的所有信息中最慢的request_time
db.access.find({},{"http_x_forwarded_for":1,"request_time":1,"request":1,"time":1}).sort({"request_time":-1}).limit(10);
注意这里的-1 这里是以request_time排序并且限制出来10条 ，其实mongo最多也就一次出来20条信息
16mongo查看单个IP的信息这里是request.time排序
db.access.find({http_x_forwarded_for:'220.181.108.160'},{"time":1,_id:0,status:1,request_time:1,http_user_agent:1,request:1,http_x_forwarded_for:1}).sort({request_time:1})
17 阿里云工单地址
https://workorder.console.aliyun.com/console.htm#/ticket/detail/?ticketId=DG6M0A9
18.云监控安装步骤
https://help.aliyun.com/document_detail/cms/User_Manual/CloudServiceMonitor/CloudMonitorGuide.htm
选择安装新版云监控插件
19. mongodb数据库删除集合
db.collection.drop()删除一个集合的命令
比如 db.access20160511.drop()
20.centos7.1安装计划任务
yum install crontabs -y
21 show variables like "%gen%";
搜索mysql的某个配置
22 php的平滑重启
kill -USR2 `cat /run/php-fpm/php-fpm.pid`
23 mongo的正则匹配查询只要加一个$regex 就可以查询/sync/check/后面可能还很多。都可以查询得到
db.access20160622.find({"request":{$regex:"/sync/check"}}).limit(5)
24 cedar每日统计错误代码比如500 404
nohup php script.php backend/mongo_log/countStatus/access20160603 >> access.log &
25 查询某个字段相关一共有多少条数据
db.user_inbox_session.find({unread_count:-1}).count()
26 查看访问的request、
sh check_options.sh danxin2.haodingdan.com|awk '{print $6,$7,$15}'|sort|uniq -c |sort -nr|more
27阿里云app地址
https://push.console.aliyun.com/?spm=5176.doc34370.2.2.kJCWpr#/app/deviceStat/?appId=2308580&appKey=23287577&appName=%E5%8D%83%E8%AF%A2&source=
28 mongo的模糊查询
db.access20160701.find({"request":{$regex:"session_id=9/-1000038/0,0&to=-1000038&user_id=342322"}},{request:1,time:1,_id:0,request_time:1}).sort({request_time:-1})
29 查看mongo的索引
HaodingdanRepSet:PRIMARY> db.user_inbox_session.getIndexes();
db.purchase_intention.getIndexes();
30 查看mongo表中的一条信息i
db.user_inbox_session.find().limit(1);
31 如果需要屏蔽改成1 下面是屏蔽的写法，一定指定id ，要不然全部都会被屏蔽 不屏蔽是0 ·执行的时候给老大看下····
update haodingdan.com_base_info set is_shield = 1 where member_id in( 1212 ,12121,);
32 统计danxin2站有referer_http
cat /var/log/nginx/access.log|grep danxin2.haodingdan.com|awk '{print $10}'|sort |uniq -c |sort -nr|more
说明 $10正好是_这个referer位置
33 统计日志请求大于3就
sh query_time.sh 08:00 09:00 |grep /websocket/CheckOnline |awk '{if ($5 > 3) print $1,$2,$3,$4,$5,$6,$7,$8}'
34 查看是否有索引
show index from tblname;
35 svn拉取之前的版本代码下来，注意我这里只能拉取目录 不能拉取文件 -r后面的1862是指定show log最后一次的版本号
svn co -r 1862 svn://packages.garmentoffice.org/hdd_modules/business_objects/branches/1.6.0/src/Haodingdan/BusinessModule/
Python
#统计PV，UV
#-*- coding: UTF-8 -*- ips = [] with open(r'C:\Users\qinghesh\Desktop\view\nxd-frontal-gateway.2017-12-17.log') as f: for line in f: ip = line.split(',')[1].split(':')[1].strip('"') ips.append(ip) print('PV is {0}'.format(len(ips))) print('UV is {0}'.format(len(set(ips)))) print('The ip is {}'.format(set(ips))) #统计httpd code d = {} with open('access_log',encoding='utf-8') as f: for line in f: key = line.split()[8] d.setdefault(key,0) d[key] += 1 sum_request = 0 err_request = 0 for key,val in d.items(): print(key,val) if int(key) >= 400: err_request += val sum_request+= val print(sum_request) print('error rate: {0:.2f}%'.format(err_request * 100.0 / sum_request))
#统计url
from collections import Counter from collections import Iterator c = Counter() with open('access_log') as f: for line in f: c[line.split()[6]] += 1 print("Poular resources : {0}".format(c.most_common(3)))
