五、python-jsonpath、写日志、面向对象
(一)jsonpath
1、python需安装jsonpath模块
# 安装 pip install jsonpath
2、jsonpath使用
(1)模块导入使用:
# 模块提取方法 import json from jsonpath import jsonpath
(2)语法规则
JSONPath | Xpath | 描述 |
$ | / | 根节点 |
@ | . | 过滤器断言(filter predicate)处理的当前节点对象,类似于this |
.or[] | / | 取子节点 |
n/a | .. |
父元素,Jsonpath未支持 |
.. | // | 递归搜索,不管位置,选择所有符合条件的条件 |
* | * | 通配符,匹配所有的元素 |
n/a | @ | 属性访问字符,JsonPath不支持 |
[] | [] | 迭代器标示(可以在里面做简单的迭代操作,如数组下标,根据内容选值等) |
[,] | l | 连接操作符在XPath结果合并其它节点集合。JsonPath允许name或者数组索引 |
?() | [] | 应用过滤表示式,可进行过滤操作 |
() | n/a | 脚本表达式,使用在脚本引擎下面,XPath不支持 |
[start:end:step] | n/a | 数组分割操作,XPath不支持 |
n/a | () | Xpath分组,JsonPath不支持 |
注意:
- []在Xpath表达式总是从前面的路径来操作数组,索引是从1开始;
- 使用JsonPath的[]操作符操作一个对象或者数组,索引是从0开始的。
(3)实例:https://qun.qq.com/cgi-bin/qun_mgr/search_group_members
s={"ec":0,"errcode":0,"em":"","cache":0,"adm_num":3,"levelname":None,"mems":[{"uin":511402865,"role":0,"g":0,"join_time":1589360442,"last_speak_time":1600967808,"lv":{"point":0,"level":1},"card":"","tags":"-1","flag":0,"nick":"\u671d\u82b1\u5915\u62fe","qage":14,"rm":0},{"uin":475566024,"role":1,"g":0,"join_time":1589360443,"last_speak_time":1596195430,"lv":{"point":0,"level":1},"card":"","tags":"-1","flag":0,"nick":"CC","qage":15,"rm":0},{"uin":616745045,"role":1,"g":0,"join_time":1589360443,"last_speak_time":1589360443,"lv":{"point":0,"level":1},"card":"","tags":"-1","flag":0,"nick":"\u5927\u5e08\u5144","qage":14,"rm":0},{"uin":1473732204,"role":1,"g":0,"join_time":1589360443,"last_speak_time":1596699591,"lv":{"point":0,"level":1},"card":"","tags":"-1","flag":0,"nick":"\u5b89\u5927\u53d4","qage":10,"rm":0},{"uin":1930890111,"role":2,"g":-1,"join_time":1589360638,"last_speak_time":1589363741,"lv":{"point":0,"level":1},"card":"","tags":"-1","flag":0,"nick":"56","qage":9,"rm":0},{"uin":549313033,"role":2,"g":0,"join_time":1590131830,"last_speak_time":1597542612,"lv":{"point":0,"level":1},"card":"\u767d\u5b87\u9e4f","tags":"-1","flag":0,"nick":"\u79e6\u6b87","qage":12,"rm":0},{"uin":121654011,"role":2,"g":1,"join_time":1591326665,"last_speak_time":1597549705,"lv":{"point":0,"level":1},"card":"\u8e6d\u8bfe-\u66f9\u4e3a\u7f8e","tags":"-1","flag":0,"nick":"\u265dAimee\u00b7Too\u2740","qage":14,"rm":0},{"uin":411732604,"role":2,"g":1,"join_time":1591326665,"last_speak_time":1591584091,"lv":{"point":0,"level":1},"card":"","tags":"-1","flag":0,"nick":"\u4e09\u53f6\u8349\u7684\u624b\u6307","qage":14,"rm":0},{"uin":690763103,"role":2,"g":1,"join_time":1591326665,"last_speak_time":1599960754,"lv":{"point":0,"level":1},"card":"\u674e\u9ad8\u82f1","tags":"-1","flag":0,"nick":"\u4e24\u6b21\u65b9\u7684\u65cb\u5f8b","qage":12,"rm":0},{"uin":1522503760,"role":2,"g":0,"join_time":1591326665,"last_speak_time":1598146137,"lv":{"point":0,"level":1},"card":"\u79b9\u6881","tags":"-1","flag":0,"nick":"\u79b9\u6881","qage":9,"rm":0},{"uin":635763064,"role":2,"g":1,"join_time":1592997221,"last_speak_time":1600596210,"lv":{"point":0,"level":1},"card":"\u970d\u7d2b\u9633","tags":"-1","flag":0,"nick":"\u6f02\u6d41\u6d77\u5cb8","qage":13,"rm":0},{"uin":857566034,"role":2,"g":1,"join_time":1593329449,"last_speak_time":1600587021,"lv":{"point":0,"level":1},"card":"\u4ee3\u723d","tags":"-1","flag":0,"nick":"\u767d\u7fbd\u5f52\u697c","qage":13,"rm":0},{"uin":347158400,"role":2,"g":0,"join_time":1593345739,"last_speak_time":1599385077,"lv":{"point":0,"level":1},"card":"\u6731\u6210","tags":"-1","flag":0,"nick":"\u9ea6\u514b.vod","qage":16,"rm":0},{"uin":704096641,"role":2,"g":1,"join_time":1594023174,"last_speak_time":1600588791,"lv":{"point":0,"level":1},"card":"\u803f\u5a1f","tags":"-1","flag":0,"nick":"704096641","qage":13,"rm":0},{"uin":978502577,"role":2,"g":1,"join_time":1594883618,"last_speak_time":1599992058,"lv":{"point":0,"level":1},"card":"\u5f20\u4e39\u96ea","tags":"-1","flag":0,"nick":"\u3000\u3000Amour\u256e\u66ae\u5ff5","qage":9,"rm":0},{"uin":799614279,"role":2,"g":0,"join_time":1594884719,"last_speak_time":1600575331,"lv":{"point":0,"level":1},"card":"\u9c81\u6d25\u5065","tags":"-1","flag":0,"nick":"\u4e28\u5bd2\u5c10\u6708\u309e","qage":13,"rm":0},{"uin":695254152,"role":2,"g":0,"join_time":1594886366,"last_speak_time":1600596152,"lv":{"point":0,"level":1},"card":"\u738b\u7965\u9f99","tags":"-1","flag":0,"nick":"\u8ffd\u68a6\u8d64\u5b50\u5fc3","qage":13,"rm":0},{"uin":251202767,"role":2,"g":1,"join_time":1594943472,"last_speak_time":1600596158,"lv":{"point":0,"level":1},"card":"\u9ad8\u96ef","tags":"-1","flag":0,"nick":"\u7d2b\u8272\u7cbe\u7075","qage":16,"rm":0},{"uin":120617143,"role":2,"g":1,"join_time":1595481073,"last_speak_time":1596951515,"lv":{"point":0,"level":1},"card":"\u7b71","tags":"-1","flag":0,"nick":"\u6668\u98ce\u5915\u96e8","qage":18,"rm":0},{"uin":357084975,"role":2,"g":1,"join_time":1595817181,"last_speak_time":1600596163,"lv":{"point":0,"level":1},"card":"\u674e\u97e9\u97e9","tags":"-1","flag":0,"nick":"\u2581\u2581\u5e7b\u68a6\u541f\u8ff7\u60d1\u4e0d\u4f4f\u7684\u5fc3","qage":8,"rm":0},{"uin":296915611,"role":2,"g":-1,"join_time":1595927320,"last_speak_time":1600874827,"lv":{"point":0,"level":1},"card":"\u9b4f\u5f3a","tags":"-1","flag":0,"nick":"\u8def\u4eba\u7532@\u63d0\u4e0d\u8d77\u52b2","qage":13,"rm":0}],"count":48,"svr_time":1601129005,"max_count":200,"search_count":48,"extmode":0} import jsonpath # jsonpath取层级比较多,复杂的方式 print(jsonpath.jsonpath(s,'$.max_count')) print(jsonpath.jsonpath(s,'$.mems[0]')) print(jsonpath.jsonpath(s,'$.mems[0].nick')) print(jsonpath.jsonpath(s,'$..level'))
执行结果如图:
(二)写日志:为了方便排查问题
1、Python自带的logging不是很好用,建议安装模块:loguru
# 安装loguru模块 pip install loguru
2、导入loguru模块
# import logging # python自带模块 # import loguru from loguru import logger # debug 打印日志比较详细,调试信息,最低的级别 # info 正常的提示信息 # warning 警告信息 # error 出错了 # exception 程序出异常了 SQL执行出错 import sys def Bob(): logger.remove() # 清除以前它的默认设置 fmt = '[{time}][{level}][{file.path}:line:{line}:function_name:{function}] || msg = {message}' # level file function module time message logger.add(sys.stdout,level='INFO',format=fmt) # print,输出,本地运行的时候,在控制台打印 # logger.add('abc.log',level='INFO',encoding='utf-8',enqueue=True) # 写在日志文件里面
# enqueue=True ,异步写日志
logger.debug('程序开始运行了') logger.debug('开始连接MySQL') logger.info('MySQL配置XXXX') logger.warning('警告,磁盘空间即将不足!') logger.error('程序出错了!') Bob()
执行结果如图:
3、定义写日志函数:
(1)同步写日志
# 同步写日志 def write_log(msg): with open('a.log','a',encoding='utf-8') as fw: fw.write(msg)
(2)异步写日志
from loguru import logger import sys def Bob(): logger.remove() # 清除以前它的默认设置 fmt = '[{time}][{level}][{file.path}:{line}:function_name:{function}] || msg = {message}' # level file function module time message logger.add(sys.stdout,level='INFO',format=fmt) # print,输出 本地运行的时候,在控制台打印 logger.add('abc.log',level='DEBUG',encoding='utf-8',enqueue=True,rotation='1 kb') # 写在日志文件里面 # logger.add("demo1.log",rotation="500 MB") # 文件过大就会重新生成一个文件 # logger.add("demo2.log",rotation="01:00") # 每天1点创建新文件 # logger.add("demo3.log",rotation="2 week") # 文件时间过长就会创建新文件 # logger.add("demo4.log",retention="7 days") # 一个礼拜后会清空 # enqueue=True,异步写日志 for i in range(100): logger.debug('程序开始运行了') logger.debug('开始连接MySQL') logger.info('MySQL配置XXXX') logger.warning('警告,磁盘空间即将不足!') logger.error('程序出错了!') Bob()
(3)rotation和retention
- rotation 可以设置大小,超过多大就产生一个新文件 1 kb,500 m,1 g
- rotation 可以多长时间,1 day 1 hour
- rotation 几点创建新文件,00:00 1:00
- retention 多长时间清空日志
(三)面向对象
1、面向对象:定义类
(1)类(class)、类变量、数据成员、方法重写、局部变量、实例变量、对象、实例、实例化、继承、方法
类:用来描述具有相同的属性和方法的对象的集合。对象是类的实例。一个模板,一个模型。例如:Class Car
类变量:定义在类中且在函数体之外。类变量通常不作为实例变量使用。
数据成员:类变量或者实例变量,用于处理类及其实例对象的相关的数据。
方法:类中定义的函数。
方法重写:从父类继承的方法不能满足子类的需求,可以对其进行改写,这个过程叫方法的覆盖(override),也称方法的重写。
局部变量:定义在方法中的变量,只作用于当前实例的类。
实例变量:在类的声明中,属性是用变量来表示的。这种变量就成为实例变量。
对象:通过类定义的数据结构实例。对象包括两个数据成员(类变量和实例变量)和方法。根据模板创造出来的具体的东西。例如:BMW为对象,实例
实例:根据模板创造出来的具体的东西。例如:BMW=Car()实例化
构造函数:
A:类在实例化的时候,自动执行的函数
B:如果要使用这个类,必须传一些参数写在构造函数里面
实例化:创建一个类的实例,类的具体对象。把模板做成具体东西的过程
继承:一个派生类(derived class)继承基类(base class)的字段和方法。
# 1、经典类 class PersonManager: pass # 2、新式类 class Person2(): pass # 3、新式类 class Person3(object): pass
(2)实例,定义一个汽车类,且再定义方法
# 定义一个汽车类,并且定义方法 class Car: def run(self): print('run..') def baoyang(self): print('payfor') BMW = Car() BMW.run()
BMW.baoyang()
运行结果为:
(3)定义一个类:Person
class Person: def __init__(self,name,sex): # 构造函数,自动执行函数 self.name = name self.sex = sex self.cry() print('我是构造函数') def eat(self): print('%s 在eating..' % self.name) def make(self): print('%s 在making..'% self.name) def say(self): print('my name is %s, sex is %s' %(self.name,self.sex)) def cry(self): print('%s 哇哇哇哇'% self.name) Nancy = Person('Nancy','Female') Bob = Person('Bob','male')
Nancy.eat()
Bob.cry()
(4)基础重载方法
序号 | 方法,描述,简单的调用 |
1 |
_init_(self[,args...]) 构造函数 简单的调用方法:obj = className(args) |
2 |
_del_(self) 析构方法,删除一个对象 简单的调用方法:del obj |
3 |
_repr_(self) 转化为供解释器读取的形式 简单的调用方法:repr(obj) |
4 |
_str_(self) 用于将值转化为适于人阅读的形式 简单的调用方法:str(obj) |
5 |
_cmp_(self,x) 对象比较 简单的调用方法:cmp(obj,x) |
(5)类的私有属性
_private_attrs:两个下划线开头,声明该属性为私有,不能在类的外部被使用或直接访问。类内部的方法中使用时self._private_attrs。
(6)单下划线、双下划线、头尾双下划线说明:
- _foo_:定义的是特殊方法,一般是系统定义名字,类似_init_()之类的
- _foo:以单下划线开头的表示的是protected类型的变量,即保护类型只能允许其本身与子类进行访问,不能用于from module import*
- __foo:双下划线表示的是私有类型(private)的变量,只能是允许这个类本身进行访问了。
2、析构函数和构造函数,以及私有
实例:
import pymysql from loguru import logger import traceback MYSQL_INFO = { 'host':'118.24.3.40', 'user':'jxz', 'password':'123456', 'db':'jxz', 'charset':'utf8', 'autocommit':True } class MySQL: def __init__(self,host,user,password,db,charset='utf8',autocommit=True): self.conn = pymysql.connect(host=host,user=user,password=password,db=db,charset=charset,autocommit=autocommit) self.cursor = self.conn.cursor() def execute(self,sql): try: self.cursor.execute(sql) except Exception: logger.error('sql执行出错,sql语句是{}',sql) logger.error(traceback.format_exc()) def fetchall(self,sql): self.execute(sql) return self.cursor.fetchall() def fetchone(self,sql): self.execute(sql) return self.cursor.fetchone() def bak_db(self): pass def close(self): self.cursor.close() self.conn.close() if __name__ == '__main__': my = MySQL(**MYSQL_INFO) my.fetchall('select * from app_myuser;') my.fetchone('select * from app_myuser where id = 1;') my.execute('delete from user where id = 1;') my.close()
析构函数:实例被销毁的时候自动执行的函数。最后执行析构函数。
# 析构函数 class Person: def __init__(self,name,sex): # 构造函数 self.name = name self.sex = sex self.cry() print('我是构造函数') def __del__(self): print('我是析构函数') def eat(self): print('%s 在eating..' % self.name) def make(self): print('%s 在making..'% self.name) def say(self): print('my name is %s, sex is %s' %(self.name,self.sex)) def cry(self): print('%s 哇哇哇哇'% self.name) Nancy = Person('Nancy','Female') Bob = Person('Bob','male') Nancy.eat() Bob.cry()
私有:只能通过self调用,不能通过实例调用。
只能在类里面用,类外面不能用。
自己定义的方法和变量,如果是__开头,就是私有的
3、各种方法类型
(1)类方法:公共的方法,直接可以通过类名来调用,不需要实例化,通过实例也可以调用(@classmethod)
@classmethod # cls代表类方法 def English(cls): print('会说英语!') Person.English()
(2)类变量:定义在类里面的变量
class Person: country = 'China' def __init__(self,name,sex): self.name=name self.sex=sex def say(self): print('my name is %s, sex is %s' % (self.name, self.sex)) print('我的国籍是 %s' % self.country) Amy = Person('Amy','女') Amy.say()
执行结果为:
(3)实例变量(成员变量):self.XXX
class Person: country = 'China' def __init__(self,name,sex): self.name=name self.sex=sex def say(self): print('my name is %s, sex is %s' % (self.name, self.sex)) print('我的国籍是 %s' % self.country) Amy = Person('Amy','女') Bob = Person('Bob','男') Bob.country='UK' Amy.say() Bob.say()
执行结果为:
(4)静态方法(静态变量):和一个普通方法没有任何区别,和类也没有什么关系,只是定义在类里面(@staticmethod)
@staticmethod def test(): print('小测试')
实例如下:
class Person: country = 'China' def __init__(self,name,sex): self.name=name self.sex=sex def say(self): print('my name is %s, sex is %s' % (self.name, self.sex)) print('我的国籍是 %s' % self.country) @classmethod # cls代表类方法 def English(cls): print('会说英语!') @staticmethod def test(): print('小测试') Person.test() Person.English() Amy = Person('Amy','女') Bob = Person('Bob','男') Bob.country='UK' Amy.say() Bob.say() Bob.test()
执行结果如下:
(5)属性方法:看起来像变量的方法,没有参数(@property)
import time class Person: country = 'China' def __init__(self,name,sex): self.name=name self.sex=sex self.birthday=time.time() @property def age(self): return time.time() - self.birthday Amy = Person('Amy','女') print(Amy.age)
(6)实例方法:必须通过实例化才可以调用,只要参数有self
4、面向对象的3大特性:封装、继承、多态
(1)封装:把零散的代码封装到一起,化零为整
(2)继承:用来节省代码
实例:
class Plant: def trees(self): print('这是一颗大树!') def origion(self): print('来着于云南省') # yangshu继承Plant类 class yangshu(Plant): pass a = yangshu() a.trees() a.origion()
执行结果如下:
(3)多态:Python里面根本不需要多态
(4)继承之重写:
A:重写-直接覆盖
class Plant: def trees(self): print('这是一颗大树!') def origion(self): print('来着于云南省') # yangshu继承Plant类 class yangshu(Plant): # 重写 def origion(self): print('来着于东北省') a = yangshu() a.trees() a.origion()
执行结果为:
B:重写-父用
class Plant: def trees(self): print('这是一颗大树!') def origion(self): print('来着于云南省') # yangshu继承Plant类 class yangshu(Plant): # 重写-覆盖 def origion(self): # super 找到父类,调用再定义 super().origion() print('来着于北京市') a = yangshu() a.trees() a.origion()
5、实例:
import os import requests max_count = 1000 size = 40 bkn = '2140990271' cookies = 'pgv_pvi=328774656; RK=Bqzk3YEWb+; ptcz=4c3263932d8d5a48b929cd2283f1bde3146c1ee5c4f104796f0c8d95df45c3c8; tvfe_boss_uuid=7d887b37fe60ad85; pgv_pvid=3957380448; _qpsvr_localtk=0.9320463335905909; uin=o1270894070; skey=@78hV4b1qu; p_uin=o1270894070; pt4_token=taQ**Bbk0PIr3PB*-*yZeg*Fq8ENsjSXEeEf3Y-9m94_; p_skey=VJFMRPfcF8O-aOhbyYT*g0zfuIhUCzLdT175dWC8CzU_; traceid=f656b26bcb' dir_name = 'qq_pics' class DownLoadQQGroupMemberPic: group_member_url = 'https://qun.qq.com/cgi-bin/qun_mgr/search_group_members' pic_url = 'https://q4.qlogo.cn/g?b=qq&nk=%s&s=140' def __init__(self, gc): self.gc = gc self.mkdir() def mkdir(self): if not os.path.exists(self.dir_name): os.mkdir(self.dir_name) def get_member_info(self, gc, st, end): # 获取群成员的信息 data = {'gc': gc, 'st': st, 'end': end, 'bkn': bkn} headers = {'cookie': cookies} r = requests.post(self.group_member_url, data, headers=headers) members = r.json return members def save_picture(self, name, content): with open(name + '.jpg', 'wb') as fw: fw.write(content) def download_picture(self, qq): r = requests.get(self.pic_url % qq) return r.content def down_qq_pic(self, mems): for m in mems: qq = m.get('uin') name = m.get('card') if m.get('card') else m.get('nick') content = self.download_picture(qq) self.save_picture(name, content) def main(self): count = 0 for j in range(0, self.max_count + 1, self.size): mems = self.get_member_info(self.gc, j + count, j + self.size + count) if 'mems' in mems: self.down_qq_pic(mems)