jq命令

以下为json格式的wp.log查询内容

{
  "_index": "security-log-waf4nginx-2021.08.17",
  "_type": "_doc",
  "_id": "7BhzUXsBveVSWlesuPXU",
  "_score": 2.5269058,
  "_source": {
    "server_port": "443",
    "appName": "qq-xflow-nginx.qq.com",
    "cluster_id": "0052cf59e33a4e931f87dbb56a908c82",
    "server_addr": "172.20.18.157",
    "request_length": 3333,
    "upstream_addr": "172.20.34.75:80",
    "http_referer": "https://m.qq.com/gp/83770757?templateType=C&bizOrigin=XM_ZAXFA_JJBJTT_CDBX_ZNSPPLH00015&adid=1701263627531278&creativeid=1701265104652331&creativetype=15&clickid=EKuQ7bGq6YIDGK3z4L7djPwDIP2FoLXdjOQBMAw44doBQiIyMDIxMDgxNjIzMzkxNzAxMDIxMjE0NjIxMzUwOTYxQjVDSMG4ApABAA&abt=qjts",
    "request_time": 0.014,
    "time": "2021-08-16 23:47:43",
    "source": "ngxAccess",
    "http_user_agent": "Mozilla/5.0 (Linux; Android 10; 8848 M6 Build/QKQ1.200127.002; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/74.0.3729.186 Mobile Safari/537.36 aweme_lite_150400 AppName/aweme_lite JsSdk/1.0 NetType/WIFI Channel/dylite_gdt_wz_yybwzl2 app_version/15.4.0 ByteLocale/zh-Hans-CN Region/CN AppSkin/black",
    "body_bytes_sent": 20,
    "clientIp": "120.219.4.61",
    "status": 200,
    "@version": "1",
    "tags": [
      "_dateparsefailure",
      "wp"
    ],
    "user_id": "c75a287b7093222fa9ba35ca1c9cc558",
    "nid": "a4774db3a883a6717ffc12a832ec38ce",
    "_dataType": "waf4nginx",
    "method": "GET",
    "scheme": "https",
    "request_uri": "/cloud_web_sdk.gif?data=%7B%22eve",
    "@timestamp": "2021-08-17T00:12:20.631Z",
    "host": "zhongan-xflow-nginx.zhongan.com",
    "s_geoip": {
      "country_name": "China",
      "location": {
        "lon": 113.7266,
        "lat": 34.7725
      },
      "continent_code": "AS",
      "country_code2": "CN"
    },
    "remote_addr": "120.219.4.61",
    "_dataFrom": "logstash"
  }
}

1、要查看json内容最简单的是使用.表达式，会打印json的原始内容

jq .  wp.log    
jq '.'  wp.log    显示文档全部内容 .表示文档本身

 2、查看文档中键为  _source 的内容

jq '._source' wp.log

{
  "server_port": "443",
  "appName": "qq-xflow-nginx.qq.com",
  "cluster_id": "0052cf59e33a4e931f87dbb56a908c82",
  "server_addr": "172.20.18.157",
  "request_length": 3333,
  "upstream_addr": "172.20.34.75:80",
  "http_referer": "https://m.qq.com/gp/83770757?templateType=C&bizOrigin=XM_ZAXFA_JJBJTT_CDBX_ZNSPPLH00015&adid=1701263627531278&creativeid=1701265104652331&creativetype=15&clickid=EKuQ7bGq6YIDGK3z4L7djPwDIP2FoLXdjOQBMAw44doBQiIyMDIxMDgxNjIzMzkxNzAxMDIxMjE0NjIxMzUwOTYxQjVDSMG4ApABAA&abt=qjts",
  "request_time": 0.014,
  "time": "2021-08-16 23:47:43",
  "source": "ngxAccess",
  "http_user_agent": "Mozilla/5.0 (Linux; Android 10; 8848 M6 Build/QKQ1.200127.002; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/74.0.3729.186 Mobile Safari/537.36 aweme_lite_150400 AppName/aweme_lite JsSdk/1.0 NetType/WIFI Channel/dylite_gdt_wz_yybwzl2 app_version/15.4.0 ByteLocale/zh-Hans-CN Region/CN AppSkin/black",
  "body_bytes_sent": 20,
  "clientIp": "120.219.4.61",
  "status": 200,
  "@version": "1",
  "tags": [
    "_dateparsefailure",
    "wp"
  ],
  "user_id": "c75a287b7093222fa9ba35ca1c9cc558",
  "nid": "a4774db3a883a6717ffc12a832ec38ce",
  "_dataType": "waf4nginx",
  "method": "GET",
  "scheme": "https",
  "request_uri": "/cloud_web_sdk.gif?data=%7B%22eve",
  "@timestamp": "2021-08-17T00:12:20.631Z",
  "host": "zhongan-xflow-nginx.zhongan.com",
  "s_geoip": {
    "country_name": "China",
    "location": {
      "lon": 113.7266,
      "lat": 34.7725
    },
    "continent_code": "AS",
    "country_code2": "CN"
  },
  "remote_addr": "120.219.4.61",
  "_dataFrom": "logstash"
}

3、查看文档中键_source 中键为tags的列表一个位置内容

[root@master3 tmp]# jq '._source.tags[0]' wp.log
"_dateparsefailure"

4、| 操作符号是jq中的过滤器，过滤格式通过{...}来构建对象和属性，可以嵌套访问属性，例如._source.tags

[root@master3 tmp]#jq '.|{aaa:._source.tags[0],bbb:._source.tags[1]}' wp.log   获取对应键的值，并设置自定义的键名
{

  "aaa": "_dateparsefailure",
  "bbb": "wp"
}

[]中如果为空表示获取所有的数组元素

5、根据Key对应的值过滤内容

jq '._source|select(.host=="zhongan-xflow-nginx.zhongan.com")' wp.log


tail -f  wp.log |jq '.|select(.host=="zhongan-xflow-nginx.zhongan.com"  and .status !=200)'

本例中只有一个字段，所以无法体现过滤的功能

 

jq也支持从JSON对象中删除键。删除后输出就不包含删除key的JSON对象。删除键使用del()函数,还是以dog.json为例

[root@master3 tmp]# jq 'del(._source)' wp.log
{
"_index": "security-log-waf4nginx-2021.08.17",
"_type": "_doc",
"_id": "7BhzUXsBveVSWlesuPXU",
"_score": 2.5269058
}

 

参考文档：https://stedolan.github.io/jq/manual/

https://devdocs.io/jq/