爬虫练习
1.
from bs4 import BeautifulSoup info = [] with open('D:\web1111/new_index.html','r') as wb_data: Soup = BeautifulSoup(wb_data,'lxml') images = Soup.select('body > div.main-content > ul > li > img') titles = Soup.select('body > div.main-content > ul > li > div.article-info > h3 > a') descs = Soup.select('body > div.main-content > ul > li > div.article-info > p.description') rates = Soup.select('body > div.main-content > ul > li > div.rate > span') cates = Soup.select('body > div.main-content > ul > li > div.article-info > p.meta-info') # print (images,titles,descs,rates,cates) for title,image,desc,rate,cate in zip(titles,images,descs,rates,cates): data = { 'title' :title.get_text(), 'rate' :rate.get_text(), 'desc' :desc.get_text(), 'cate' :list(cate.stripped_strings), 'image' :image.get('src') } info.append(data) for i in info: if float(i['rate'])>3: print(i['title'],i['cate']) ''' body > div.main-content > ul > li:nth-child(1) > div.article-info > h3 > a body > div.main-content > ul > li:nth-child(1) > div.article-info > p.meta-info > span:nth-child(2) body > div.main-content > ul > li:nth-child(1) > div.rate > span body > div.main-content > ul > li:nth-child(1) > div.article-info > p.description body > div.main-content > ul > li:nth-child(1) > img '''
2.
# !/usr/bin/python #-*- coding: utf-8 -*- from bs4 import BeautifulSoup import requests url = 'http://bj.xiaozhu.com/fangzi/1508951935.html' wb_data = requests.get(url) soup = BeautifulSoup(wb_data.text,'lxml') # 因为是单页面，使用 select 方法获得的元素又是一个列表，那么列表中的第一个元素且也是唯一一个元素即是我们要找的信息 用 “[0]” 索引将其取出 # 后在对其使用处理的方法，因为 beautifulsoup 的些筛选方法并不能针对列表类型的元素使用 ;) title = soup.select('div.pho_info > h4')[0].text address = soup.select('div.pho_info > p')[0].get('title') # 和 get('href') 同理，他们都是标签的一个属性而已，我们只需要的到这个属性的内容即可 price = soup.select('div.day_l > span')[0].text pic = soup.select('#curBigImage')[0].get('src') # “#” 代表 id 这个找元素其实就是找他在页面的唯一 host_name = soup.select('a.lorder_name')[0].text host_gender = soup.select('div.member_pic > div')[0].get('class')[0] # 请在此处打印并观察结果 print(title) print(address) print(price) print(pic) print(host_name) print(host_gender) # 根据结果观察不同性别会用不同的图标样式（class），设计一个函数进行转换 def print_gender(class_name): if class_name == 'member_ico1': return '女' if class_name == 'member_ico': return '男' data = { 'title':title, 'address':address, 'price':price, 'pic':pic, 'host_name':host_name, 'host_gender':print_gender(host_gender) } print(data) # -------------------补充------------------ # 如何批量获取链接 page_link = [] # <- 每个详情页的链接都存在这里，解析详情的时候就遍历这个列表然后访问就好啦~ def get_page_link(page_number): for each_number in range(1,page_number): # 每页24个链接,这里输入的是页码 full_url = 'http://bj.xiaozhu.com/search-duanzufang-p{}-0/'.format(each_number) wb_data = requests.get(full_url) soup = BeautifulSoup(wb_data.text,'lxml') for link in soup.select('a.resule_img_a'): # 找到这个 class 样为resule_img_a 的 a 标签即可 page_link.append(link) # ---------------------
3.
from bs4 import BeautifulSoup data = [] path = './web/new_index.html' with open(path, 'r') as f: Soup = BeautifulSoup(f.read(), 'lxml') titles = Soup.select('ul > li > div.article-info > h3 > a') pics = Soup.select('ul > li > img') descs = Soup.select('ul > li > div.article-info > p.description') rates = Soup.select('ul > li > div.rate > span') cates = Soup.select('ul > li > div.article-info > p.meta-info') for title, pic, desc, rate, cate in zip(titles, pics, descs, rates, cates): info = { 'title': title.get_text(), 'pic': pic.get('src'), 'descs': desc.get_text(), 'rate': rate.get_text(), 'cate': list(cate.stripped_strings) } data.append(info) for i in data: if len(i['rate']) >= 3: print(i['title'], i['cate'])
4.
# !/usr/bin/python #-*- coding: utf-8 -*- import requests from bs4 import BeautifulSoup def fit(url): base_url = url headers = {'User-Agent': "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1"} html = requests.get(base_url,headers=headers) soup = BeautifulSoup(html.text,'lxml') # print soup data = [] # content = soup.select('body > div.zg-wrap.zu-main.clearfix > div.zu-main-content > div > ul > li > div.title > a')[0] # url_last = content.get('href') url_first = 'https://www.zhihu.com' # url_context = url_first+url_last def url_handle(url_last): ##排除掉专栏 if url_last.startswith('/question'): url_context = url_first + url_last return url_context else: url_context = url_last return url_context titles = soup.select('div.zg-wrap.zu-main.clearfix > div.zu-main-content > div > ul > li > div.title > a') writes = soup.select('body > div.zg-wrap.zu-main.clearfix > div.zu-main-content > div > ul > li > div.content > div > div.entry-body > div.entry-meta > div > span.author-link-line > a') prizes = soup.select('body > div.zg-wrap.zu-main.clearfix > div.zu-main-content > div > ul > li > div.content > div > div.entry-left.hidden-phone > a') contents = soup.select('body > div.zg-wrap.zu-main.clearfix > div.zu-main-content > div > ul > li > div.title > a') for title,write,prize,content in zip(titles,writes,prizes,contents): url_last = content.get('href') url_context = url_handle(url_last) # url_context = url_first + url_last html_content = requests.get(url_context,headers=headers) soup = BeautifulSoup(html_content.text,'lxml') if url_context.startswith('https://www.zhihu.com'): first_answer = soup.select('#zh-question-answer-wrap > div:nth-of-type(1) > div.zm-item-rich-text.expandable.js-collapse-body > div.zm-editable-content.clearfix')[0].get_text() else: first_answer = 'no answer' info = { 'title': title.get_text(), 'write': write.get_text(), 'prize': prize.get_text(), 'content': first_answer } data.append(info) for a in data: print 'title:'+a['title'] print 'write:' + a['write'] print 'prize:' + a['prize'] print 'content:' + a['content'] url = 'https://www.zhihu.com/search?type=content&q=健身' fit(url)
