Python + Flask 实现Nginx Log信息统计

今日分享内容是基于：Python + Flask 实现接口接收 Disk 信息来进一步分享如何接收 Nginx Log 的信息。

原理：

通过 Python 调用 Shell 脚本去执行 Nginx Log 的相关命令，然后进行处理再请求 Requests 库来向后端定义好的接口来推送数据并展示。

Part1：收集端

 1 import os
 2 import requests
 3 import json
 4 import time
 5  
 6 url="http://10.8.31.61:5555/GetNginxResource"
 7 nginx_data={}
 8  
 9 def exec_cmd(logname):
10     nginx_cmd = "awk -F' ' '{print $21}' " + logname + " |sort |uniq -c"
11     response = os.popen(nginx_cmd)
12     nginx_data['data']=str(response.read()).replace("\n","")
13     #print(nginx_data)
14     response.close()
15     return nginx_data
16  
17 def httpPost(datas):
18     header = {"Content-Type":"application/json"}
19     resp_content = requests.post(url=url,data=json.dumps(datas),headers=header)
20     #resp_content = json.loads(resp_content.content)
21     print(resp_content.text)
22  
23 if __name__ == '__main__':
24     hostname=str(os.popen("hostname |awk -F'.' '{print $1}' |awk -F'-' '{print $2}'").read()).replace("\n","")
25     while(1):
26         data=exec_cmd("/home/q/nginx/logs/access_ars2.beta.shangri-la.com.log")
27         data1=data['data'].split(",")
28         dict1={}
29         for i in data1:
30             data2=i.strip().split(" ")
31             if(len(data2)>=2):
32                 key,value=data2[1],data2[0]
33                 dict1[key]=value
34             else:
35                 pass
36         listkey=['200','204','206','301','302','304','400','401','403','404','415','499','500','503']
37         dict2={}
38         for i in listkey:
39             if(i in dict1.keys()):
40                 dict2[i]=int(dict1[i])
41         print(dict2)
42         httpPost(dict(hostname=hostname,nginx_data=dict2))
43         time.sleep(3600)

Part2：接收端

 1 @resource.route('/GetNginxResource',methods=['POST'])
 2 def GetNginxResource():
 3     '''接收来自linux上传的数据'''
 4     query = request.get_json()
 5     hostname = query["hostname"]
 6     nginx_data = query["nginx_data"]
 7     createtime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
 8     sql = 'insert into nginx_log_info (hostname,nginx_data,create_time) VALUES '
 9     data = '("' + hostname + '","' + str(nginx_data) + '","' +  str(createtime) + '"'
10     end = data + ")"
11     sql = sql + end
12     print(sql)
13     db = conndb()
14     db.execute_sql(sql)
15     data = {'code': 200, 'message': 'success', 'status': '10000'}
16     return json.dumps(data)

Part3：展示端

这部分主要分为以下两块内容：

第一块是页面请求

<template>
    <div>
        <div class="crumbs">
            <el-breadcrumb separator="/">
                <el-breadcrumb-item>
                    <i class="el-icon-lx-cascades"></i> Nginx Log信息
                </el-breadcrumb-item>
            </el-breadcrumb>
        </div>
        <div class="container">
            <div class="handle-box">
                <el-input v-model="query.hostname" placeholder="环境" class="handle-input mr10" clearable @clear="clear_name"></el-input>
                <el-button type="primary" icon="el-icon-search" @click="handleSearch">搜索</el-button>
            </div>
            <el-table
                :data="tableData"
                border
                class="table"
                ref="multipleTable"
                header-cell-class-name="table-header">
                <el-table-column prop="id" label="ID" width="55" align="center"></el-table-column>
                <el-table-column prop="hostname" width="150" label="环境"></el-table-column>
                <el-table-column prop="nginx_data" width="800" label="nginx log统计信息"></el-table-column>
                <el-table-column prop="create_time"  label="创建时间"></el-table-column>
            </el-table>
          <el-pagination
            @size-change="handleSizeChange"
            @current-change="handleCurrentChange"
            :current-page="query.pageIndex"
            :page-sizes="[5, 10, 20, 30]"
            :page-size="query.pageSize"
            layout="total, sizes, prev, pager, next, jumper"
            :total="parseInt(pageTotal)">
          </el-pagination>
        </div>
    </div>
</template>
 
<script>
import server from '../api/request.js'
export default {
  name: 'InterfaceMem',
  data () {
    return {
      query: {
        hostname: '',
        pageIndex: 1,
        pageSize: 10
      },
      tableData: [],
      pageTotal: 0
    }
  },
  created () {
    this.getNginxData()
  },
  methods: {
    // 获取后端返回的真实数据
    getNginxData () {
      server({url: '/getNginxList', data: this.query, method: 'post'})
        .then(response => {
          console.log('**********')
          console.log(response)
          this.tableData = response.listdata
          console.log(this.tableData)
          console.log(this.tableData['nginx_data'])
          this.pageTotal = response.pageTotal || 5
        })
    },
    // 触发搜索按钮
    handleSearch () {
      server({url: '/getNginxList', data: this.query, method: 'post'})
        .then(response => {
          console.log(response)
          this.tableData = response.listdata
          console.log(this.tableData)
          this.pageTotal = response.pageTotal || 5
        })
    },
    // 分页导航
    handleSizeChange (val) {
      // console.log(val)
      this.$set(this.query, 'pageSize', val)
      // console.log(this.query)
      this.getNginxData()
    },
    // 翻页改变页码触发
    handleCurrentChange (val) {
      this.$set(this.query, 'pageIndex', val)
      this.getNginxData()
    },
    clear_name () {
      this.query.hostname = ''
      this.getNginxData()
    }
  }
}
</script>
 
<style scoped>
.handle-box {
    margin-bottom: 20px;
}
 
.handle-select {
    width: 120px;
}
 
.handle-input {
    width: 300px;
    display: inline-block;
}
.table {
    width: 100%;
    font-size: 14px;
}
.red {
    color: #ff0000;
}
.mr10 {
    margin-right: 10px;
}
.table-td-thumb {
    display: block;
    margin: auto;
    width: 40px;
    height: 40px;
}
</style>

第二块是后端请求处理

 1 @resource.route('/getNginxList',methods=['POST'])
 2 def getNginxList():
 3     '''fe的页面列表数据获取'''
 4     query = request.get_json()
 5     print(query)
 6     if (query["hostname"] == ""):
 7         sql1 = "select id,hostname,nginx_data,create_time from nginx_log_info  order by id DESC limit " + str(
 8             (query['pageIndex'] - 1) * query["pageSize"]) + "," + str(query["pageSize"])
 9         count_sql = "select count(*) from nginx_log_info"
10         colume_sql = "select id from nginx_log_info"
11  
12     else:
13         sql1 = "select id,hostname,nginx_data,create_time from nginx_log_info where hostname like '%" + str(query["hostname"]) + "%' order by id DESC" + " limit " + str(
14             (query['pageIndex'] - 1) * query["pageSize"]) + "," + str(query["pageSize"])
15         count_sql = "select count(*) from nginx_log_info where hostname like '%" + str(query["hostname"]) + "%' order by id DESC"
16         colume_sql = "select id from nginx_log_info"
17  
18     sql2 = "select id,hostname,nginx_data,create_time from nginx_log_info"
19     db = conndb()
20     listdata = db.get_data(sql1, sql2)
21     db = conndb()
22     result = db.get_data(count_sql, colume_sql)
23     print(result)
24     pageTotal = result[0]['id']
25     print(listdata)
26     print(pageTotal)
27     data = {'listdata': listdata, "pageTotal": pageTotal, "code": 200}
28     return json.dumps(data)

Part4：页面展示

欢迎关注【无量测试之道】公众号，回复【领取资源】

Python+Unittest框架API自动化、

Python+Pytest框架API自动化、

Python+Pandas+Pyecharts大数据分析、

Python+Selenium框架Web的UI自动化、

Python+Appium框架APP的UI自动化、

Python编程学习资源干货、

资源和代码免费送啦~
文章下方有公众号二维码，可直接微信扫一扫关注即可。

备注：我的个人公众号已正式开通，致力于IT互联网技术的分享。

包含：数据分析、大数据、机器学习、测试开发、API接口自动化、测试运维、UI自动化、性能测试、代码检测、编程技术等。

微信搜索公众号：“无量测试之道”，或扫描下方二维码：

添加关注，让我们一起共同成长！

posted on 2022-01-07 15:40 Wu_Candy 阅读(171) 评论(1) 编辑收藏举报