109-python-某视频网站爬取数据
import re from urllib.request import urlopen,Request from multiprocessing import Pool def get_page(url,pattern): headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:23.0) Gecko/20100101 Firefox/23.0'} req = Request(url=url, headers=headers) response = urlopen(req).read().decode('utf-8') return pattern,response # 正则表达式编译结果 网页源代码 def parse_page(info): pattern,page_content=info res=re.findall(pattern,page_content) for item in res: dic={ 'index':item[0].strip(), 'title':item[1].strip(), 'actor':item[2].strip(), 'time':item[3].strip(), 'score': '%s%s'%(item[4].strip(), item[5].strip()) } print(dic) if __name__ == '__main__': # regex = r'<dd>.*?<.*?class="board-index.*?>(\d+)</i>.*?title="(.*?)".*?class="movie-item-info".*?<p class="star">(.*?)</p>.*?<p class="releasetime">(.*?)</p>' # regex = r'<dd>.*?<i class="board-index.*?>(\d+)</i>.*?<p class="name">.*?<a.*?title=".*?".*">.*?(.*?)</a></p>.*?<p class="star">(.*?)</p>.*?<p class="releasetime">(.*?)</p>' # regex = r'<dd>.*?<i class="board-index.*?>(\d+)</i>.*?<p class="name">.*?<a.*?title=".*?".*">.*?(.*?)</a></p>.*?<p class="star">(.*?)</p>.*?<p class="releasetime">(.*?)</p><p class="score">.*?<i class="integer">(\d+\.?)</i><i class="fraction">(\d+)</i></p>.*?' # regex = r'<dd>.*?<i class="board-index.*?>(\d+)</i>.*?<p class="name">.*?<a.*?title=".*?".*">.*?(.*?)</a></p>.*?<p class="star">(.*?)</p>.*?<p class="releasetime">(.*?)</p>' # regex = r'<dd>.*?<i class="board-index.*?>(\d+)</i>.*?<p class="name">.*?<a.*?title="(.*?)".*?<p class="star">(.*?)</p>.*?<p class="releasetime">(.*?)</p>' regex = r'<dd>.*?<i class="board-index.*?>(\d+)</i>.*?<p class="name">.*?<a.*?title="(.*?)".*?<p class="star">(.*?)</p>.*?<p class="releasetime">(.*?)</p>.*?<p class="score"><i class="integer">(.*?)</i>.*?<i class="fraction">(.*?)</i></p>.*?' pattern1=re.compile(regex,re.S) url_dic={'http://maoyan.com/board/7':pattern1} p=Pool() res_l=[] for url,pattern in url_dic.items(): res=p.apply_async(get_page,args=(url,pattern),callback=parse_page) res_l.append(res) for i in res_l: i.get()