[100]awk运算-解决企业统计pv/ip问题
awk运算
awk以脚本方式运行
#!/bin/awk
BEGIN{
arr[1]="maotai";
arr[2]="maotai"
for(k in arr)
print k,arr[k]
}
[root@n1 ~]# awk -f a.sh
1 maotai
2 maomao
awk以命令行方法
[root@n1 ~]# awk 'BEGIN{arr[1]="maotai"; arr[2]="maotai";} END{for(k in arr) print k,arr[k]}' /etc/passwd
1 maotai
2 maotai
[root@n1 ~]# cat /etc/hosts|awk 'BEGIN{arr[1]="maotai"; arr[2]="maotai";} END{for(k in arr) print k,arr[k]}'
1 maotai
2 maotai
将文件用awk以数组方式输出
[root@n1 ~]# cat t3.log
1 maotai
2 maomao
[root@n1 ~]# awk '{S[$1]=$2}END{for(k in S) print k,S[k]}' t3.log
1 maotai
2 maomao
awk运用
top10url统计: 统计url(www.maotai.com)出现排名
思路先过滤url,然后再借助awk运算或者sort+uniq排序
[root@n1 ~]# cat url.log
http://www.maotai.com/index.html
http://www.maotai.com/1.html
http://post.maotai.com/index.html
http://mp3.maotai.com/3.html
http://www.maotai.com/1.html
http://post.maotai.com/2.html
[root@n1 ~]# awk -F '/' '{print $3}' url.log |sort -r|uniq -c
3 www.maotai.com
2 post.maotai.com
1 mp3.maotai.com
[root@n1 ~]# awk -F "/" '{}END{}'
[root@n1 ~]# awk -F '/' '{s[$3]=s[$3]+1}END{for(k in s) print s[k],k}' url.log |sort -r
3 www.maotai.com
2 post.maotai.com
1 mp3.maotai.com
- top10 url统计
[root@n1 ~]# awk -F '/' '{s[$3]++}END{for(k in s) print s[k],k}' url.log |sort -r|head
3 www.maotai.com
2 post.maotai.com
1 mp3.maotai.com
top10 ip统计: 统计web日志单ip访问请求数排名:爬虫
access.log
10.0.0.41 - - [03/Dec/2010:23:27:01 +0800] "HEAD /checkstatus.jsp HTTP/1.0" 200 -
10.0.0.43 - - [03/Dec/2010:23:27:01 +0800] "HEAD /checkstatus.jsp HTTP/1.0" 200 -
10.0.0.42 - - [03/Dec/2010:23:27:01 +0800] "HEAD /checkstatus.jsp HTTP/1.0" 200 -
10.0.0.46 - - [03/Dec/2010:23:27:02 +0800] "HEAD /checkstatus.jsp HTTP/1.0" 200 -
10.0.0.42 - - [03/Dec/2010:23:27:02 +0800] "HEAD /checkstatus.jsp HTTP/1.0" 200 -
10.0.0.47 - - [03/Dec/2010:23:27:02 +0800] "HEAD /checkstatus.jsp HTTP/1.0" 200 -
10.0.0.41 - - [03/Dec/2010:23:27:02 +0800] "HEAD /checkstatus.jsp HTTP/1.0" 200 -
10.0.0.47 - - [03/Dec/2010:23:27:02 +0800] "HEAD /checkstatus.jsp HTTP/1.0" 200 -
10.0.0.41 - - [03/Dec/2010:23:27:03 +0800] "HEAD /checkstatus.jsp HTTP/1.0" 200 -
10.0.0.46 - - [03/Dec/2010:23:27:03 +0800] "HEAD /checkstatus.jsp HTTP/1.0" 200 -
[root@n1 ~]# awk '{s[$1]++}END{for(k in s) print k,s[k]}' access.log |sort -rk2|head
10.0.0.41 3
10.0.0.47 2
10.0.0.46 2
10.0.0.42 2
10.0.0.43 1
tcp13种状态统计
awk '/^tcp/' a.log
方法1:
[root@n1 ~]# awk '/^tcp/ {print $NF}' a.log |sort|uniq -c |sort -rn|head
79 ESTABLISHED
7 CLOSE_WAIT
2 TIME_WAIT
方法2:
[root@n1 ~]# awk '/^tcp/ {S[$NF]++}END{for(k in S) print S[k],k}' a.log |sort -rn|head
79 ESTABLISHED
7 CLOSE_WAIT
2 TIME_WAIT
分析图片服务器日志,把日志(每个图片访问次数*图片大小的总和)排行,取top10,也就是计算每个url的总访问大小.
69.33.22.101 - - [08/Dec/2010:15:43:56 +0800] "GET /static/images/photos/2.jpg HTTP/1.1" 200 11299 "http://www.cnblogs.com/iiiiher/static/web/column/17/index.shtml?courseId=43" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729)"
59.33.26.105 - - [08/Dec/2010:15:43:56 +0800] "GET /static/images/photos/2.jpg HTTP/1.1" 200 11299 "http://www.cnblogs.com/iiiiher/static/web/column/17/index.shtml?courseId=43" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729)"
59.33.26.105 - - [08/Dec/2010:15:44:02 +0800] "GET /static/flex/vedioLoading.swf HTTP/1.1" 200 3583 "http://www.cnblogs.com/iiiiher/static/flex/AdobeVideoPlayer.swf?width=590&height=328&url=/`DYNAMIC`/2" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729)"
124.115.4.18 - - [08/Dec/2010:15:44:15 +0800] "GET /?= HTTP/1.1" 200 46232 "-" "-"
124.115.4.18 - - [08/Dec/2010:15:44:25 +0800] "GET /static/js/web_js.js HTTP/1.1" 200 4460 "-" "-"
124.115.4.18 - - [08/Dec/2010:15:44:25 +0800] "GET /static/js/jquery.lazyload.js HTTP/1.1" 200 1627 "-" "-"
输出格式: [访问次数*单个文件大小] [访问次数] 文件名(可带url)
- 输出url 大小
[root@n1 ~]# awk '{print $7"\t"$10}' image.log
/static/images/photos/2.jpg 11299
/static/images/photos/2.jpg 11299
/static/flex/vedioLoading.swf 3583
/?= 46232
/static/js/web_js.js 4460
/static/js/jquery.lazyload.js 1627
- 对输出结果统计次数
[root@n1 ~]# awk '{print $7"\t"$10}' image.log|sort | uniq -c
1 /?= 46232
1 /static/flex/vedioLoading.swf 3583
2 /static/images/photos/2.jpg 11299
1 /static/js/jquery.lazyload.js 1627
1 /static/js/web_js.js 4460
- 按照格式awk运算列后打印出: 方法1
[root@n1 ~]# awk '{print $7"\t"$10}' image.log |sort|uniq -c|awk '{print $1*$3"\t"$1"\t"$2}'|sort -rn|head
46232 1 /?=
22598 2 /static/images/photos/2.jpg
4460 1 /static/js/web_js.js
3583 1 /static/flex/vedioLoading.swf
1627 1 /static/js/jquery.lazyload.js
方法2:
[root@n1 ~]# awk '{print $7"\t" $10}' image.log|awk '{S[$1]+=$2;S1[$1]+=1}END{for(i in S) print S[i],S1[i],i}'|sort -rn|head -10
46232 1 /?=
22598 2 /static/images/photos/2.jpg
4460 1 /static/js/web_js.js
3583 1 /static/flex/vedioLoading.swf
1627 1 /static/js/jquery.lazyload.js
求某一列的和
[root@n1 sh]# cat /etc/passwd|awk -F ':' '{s+=$3}END{print s}'
2717
awk易错点: awk要用单引号
- awk要用单引号
netstat -an|awk '/^tcp/ {S[$NF]++}END{for(k in S) print S[k],k}'
- awk是双引号(会报错)
netstat -an|awk "/^tcp/ {s[$NF]++}END{for(i in s) print k,s[k]}"
