使用python获取房价信息
从贝壳网获取房价信息。
基本的步骤和我的这篇博文一样：https://www.cnblogs.com/mrlayfolk/p/12319414.html。不熟悉的可参考一下。
下面的代码是获取3000个样本的代码。
1 # encoding:utf-8 2 3 ''' 4 目的：从贝壳找房中爬取房价信息。网址：https://cd.ke.com/ershoufang/qingyang/l2/ 5 环境：python 3.7.3 6 所需的库：requests、BeautifulSoup、xlwt 7 ''' 8 9 import logging 10 import xlwt 11 import requests 12 import string 13 from bs4 import BeautifulSoup 14 15 headers = { 16 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36',\ 17 "Host": "cd.ke.com", 18 } 19 20 # 将获取的信息保存到表格中 21 def save_info(content): 22 workbook = xlwt.Workbook(encoding = 'ascii') 23 worksheet = workbook.add_sheet('house info') 24 style = xlwt.XFStyle() # 初始化样式 25 font = xlwt.Font() # 为样式创建字体 26 font.name = 'Times New Roman' 27 font.bold = True # 黑体 28 font.underline = True # 下划线 29 font.italic = True # 斜体字 30 style.font = font # 设定样式 31 worksheet.write(0, 0, '名称') 32 worksheet.write(0, 1, '位置') 33 worksheet.write(0, 2, '房屋信息') 34 worksheet.write(0, 3, '总价（万）') 35 worksheet.write(0, 4, '单价（元/平方米）') 36 37 for i, item in enumerate(content): 38 for j in range(5): #多添加一列（序号） 39 worksheet.write(i+1, j, content[i][j]) 40 workbook.save('./house_info.xls') # 保存文件 41 42 43 # 获取房屋相关的信息 44 # 主要包括：title positon houseinfo totalprice unitprice 45 def get_info(): 46 all_info = [] 47 title_list = [] 48 position_list = [] 49 house_list = [] 50 totalPrice_list = [] 51 unitPrice_list = [] 52 53 for i in range(100): 54 link = 'https://cd.ke.com/ershoufang/qingyang/pg%dl2/' % i 55 r = requests.get(link, headers=headers, timeout=10) 56 print (str(i+1), 'status_code: ', r.status_code) 57 soup = BeautifulSoup(r.text, 'lxml') 58 titleInfo = soup.findAll('div', {'class': 'info clear'}) 59 positionInfo = soup.findAll('div', {'class': 'positionInfo'}) 60 houseInfo = soup.findAll('div', {'class': 'houseInfo'}) 61 totalPrice = soup.findAll('div', {'class': 'totalPrice'}) 62 unitPrice = soup.findAll('div', {'class': 'unitPrice'}) 63 for item in titleInfo: 64 title = item.div.a.text.strip() 65 title_list.append(title) 66 for item in positionInfo: 67 postion = item.a.text.strip() 68 position_list.append(postion) 69 for item in houseInfo: 70 house = item.text.strip().replace('\n', ' ').replace(' ', '') 71 house_list.append(house) 72 for item in totalPrice: 73 total_price = item.span.text.strip() 74 totalPrice_list.append(total_price) 75 for item in unitPrice: 76 unit_price = item.span.text.strip().replace('单价', '').replace('元/平米', '') 77 unitPrice_list.append(unit_price) 78 print (len(title_list)) 79 print (len(position_list)) 80 print (len(house_list)) 81 print (len(totalPrice_list)) 82 print (len(unitPrice_list)) 83 for i in range(len(title_list)): 84 item = [title_list[i], position_list[i], house_list[i], totalPrice_list[i], unitPrice_list[i]] 85 all_info.append(item) 86 87 return all_info 88 89 90 if __name__ == "__main__": 91 all_info = get_info() 92 save_info(all_info)
