面向对象补充,进程,数据共享,锁,进程池,模块(requests,bs4(beautifulsoup)),协程
一丶面向对象补充
![](https://images.cnblogs.com/OutliningIndicators/ContractedBlock.gif)
""" class Foo(object): def __init__(self): self.info = {} def __setitem__(self, key, value): self.info[key] = value def __getitem__(self, item): return self.info.get(item) obj = Foo() obj['x'] = 123 print(obj['x']) """ from flask import globals class Foo(object): def __init__(self): object.__setattr__(self, 'info', {}) # 在对象中设置值的本质 def __setattr__(self, key, value): self.info[key] = value def __getattr__(self, item): print(item) return self.info[item] obj = Foo() obj.name = 'alex' print(obj.name) v = [] for i in range(10000): v.append(i) print(v)
二丶进程
进程间数据不共享
![](https://images.cnblogs.com/OutliningIndicators/ContractedBlock.gif)
data_list = [] def task(arg): data_list.append(arg) print(data_list) def run(): for i in range(10): p = multiprocessing.Process(target=task,args=(i,)) # p = threading.Thread(target=task,args=(i,)) p.start() if __name__ == '__main__': run()
常用功能:
- join
- deamon
- name
- multiprocessing.current_process()
- multiprocessing.current_process().ident/pid
类继承方式创建进程
![](https://images.cnblogs.com/OutliningIndicators/ContractedBlock.gif)
class MyProcess(multiprocessing.Process): def run(self): print('当前进程',multiprocessing.current_process()) def run(): p1 = MyProcess() p1.start() p2 = MyProcess() p2.start() if __name__ == '__main__': run()
进程间数据共享
Queue:
![](https://images.cnblogs.com/OutliningIndicators/ContractedBlock.gif)
q = multiprocessing.Queue() def task(arg,q): q.put(arg) def run(): for i in range(10): p = multiprocessing.Process(target=task, args=(i, q,)) p.start() while True: v = q.get() print(v) run()
![](https://images.cnblogs.com/OutliningIndicators/ContractedBlock.gif)
def task(arg,q): q.put(arg) if __name__ == '__main__': q = multiprocessing.Queue() for i in range(10): p = multiprocessing.Process(target=task,args=(i,q,)) p.start() while True: v = q.get() print(v)
Manager:(*)
![](https://images.cnblogs.com/OutliningIndicators/ContractedBlock.gif)
m = multiprocessing.Manager() dic = m.dict() def task(arg): dic[arg] = 100 def run(): for i in range(10): p = multiprocessing.Process(target=task, args=(i,)) p.start() input('>>>') print(dic.values()) if __name__ == '__main__': run()
![](https://images.cnblogs.com/OutliningIndicators/ContractedBlock.gif)
def task(arg,dic): time.sleep(2) dic[arg] = 100 if __name__ == '__main__': m = multiprocessing.Manager() dic = m.dict() process_list = [] for i in range(10): p = multiprocessing.Process(target=task, args=(i,dic,)) p.start() process_list.append(p) while True: count = 0 for p in process_list: if not p.is_alive(): count += 1 if count == len(process_list): break print(dic)
三丶进程锁
四丶进程池
from concurrent.futures import ThreadPoolExecutor,ProcessPoolExecutor (官方推荐方式)
![](https://images.cnblogs.com/OutliningIndicators/ContractedBlock.gif)
import time from concurrent.futures import ThreadPoolExecutor,ProcessPoolExecutor def task(arg): time.sleep(2) print(arg) if __name__ == '__main__': pool = ProcessPoolExecutor(5) for i in range(10): pool.submit(task,i)
五丶初识爬虫
安装:
pip3 install requests
pip3 intall beautifulsoup4
问题:
找不到命令?
方式一:
C:\Users\Administrator\AppData\Local\Programs\Python\Python36\Scripts\pip3 install requests
方式二:
C:\Users\Administrator\AppData\Local\Programs\Python\Python36\Scripts\pip3 install requests
实例:
![](https://images.cnblogs.com/OutliningIndicators/ContractedBlock.gif)
import requests from bs4 import BeautifulSoup from concurrent.futures import ThreadPoolExecutor,ProcessPoolExecutor # 模拟浏览器发送请求 # 内部创建 sk = socket.socket() # 和抽屉进行socket连接 sk.connect(...) # sk.sendall('...') # sk.recv(...) def task(url): print(url) r1 = requests.get( url=url, headers={ 'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.92 Safari/537.36' } ) # 查看下载下来的文本信息 soup = BeautifulSoup(r1.text,'html.parser') print(soup.text) # content_list = soup.find('div',attrs={'id':'content-list'}) # for item in content_list.find_all('div',attrs={'class':'item'}): # title = item.find('a').text.strip() # target_url = item.find('a').get('href') # print(title,target_url) def run(): pool = ThreadPoolExecutor(5) for i in range(1,50): pool.submit(task,'https://dig.chouti.com/all/hot/recent/%s' %i) if __name__ == '__main__': run()
相关:
a. 以上示例进程和线程那个好?
- 线程好
b. requests模块模拟浏览器发送请求
- 本质 requests.get(...):
- 创建socket客户端
- 连接 【阻塞】
- 发送请求
- 接收请求【阻塞】
- 断开连接
c. 线程和进程池