python爬虫爬取链家二手房信息
#coding=utf-8 import requests from fake_useragent import UserAgent from bs4 import BeautifulSoup import json import csv import time # 构建请求头 userAgent = UserAgent() headers = { 'user-agent': userAgent .Chrome } # 声明一个列表存储字典 data_list = [] def start_spider(page): #设置重连次数 requests.adapters.DEFAULT_RETRIES = 15 s = requests.session() #设置连接活跃状态为False s.keep_alive = False #爬取的url,默认爬取的南京的链家房产信息 url = 'https://nj.lianjia.com/ershoufang/pg{}/'.format(page) # 请求url resp = requests.get(url, headers=headers,timeout=10) # 讲返回体转换成Beautiful soup = BeautifulSoup(resp.content, 'lxml') # 筛选全部的li标签 sellListContent = soup.select('.sellListContent li.LOGCLICKDATA') # 循环遍历 for sell in sellListContent: try: # 标题 title = sell.select('div.title a')[0].string # 先抓取全部的div信息,再针对每一条进行提取 houseInfo = list(sell.select('div.houseInfo')[0].stripped_strings) # 楼盘名字 loupan = houseInfo[0] #对楼盘的信息进行分割 info = houseInfo[0].split('|') # 房子类型 house_type = info[1].strip() # 面积大小 area = info[2].strip() # 房间朝向 toward = info[3].strip() # 装修类型 renovation = info[4].strip() # 房屋地址 positionInfo = ''.join(list(sell.select('div.positionInfo')[0].stripped_strings)) # 房屋总价 totalPrice = ''.join(list(sell.select('div.totalPrice')[0].stripped_strings)) # 房屋单价 unitPrice = list(sell.select('div.unitPrice')[0].stripped_strings)[0] # 声明一个字典存储数据 data_dict = {} data_dict['title'] = title data_dict['loupan'] = loupan data_dict['house_type'] = house_type data_dict['area'] = area data_dict['toward'] = toward data_dict['renovation'] = renovation data_dict['positionInfo'] = positionInfo data_dict['totalPrice'] = totalPrice data_dict['unitPrice'] = unitPrice data_list.append(data_dict) except Exception as e: print(e) continue def main(): # 只爬取10页 for page in range(1, 10): start_spider(page) time.sleep(3) # 将数据写入json文件 with open('data_json.json', 'a+', encoding='utf-8') as f: json.dump(data_list, f, ensure_ascii=False, indent=4) print('json文件写入完成') # 将数据写入csv文件 with open('./data_csv.csv', 'w', encoding='utf-8', newline='') as f: # 表头 print(data_list) title = data_list[0].keys() # 创建writer对象 writer = csv.DictWriter(f, title) # 写入表头 writer.writeheader() # 批量写入数据 writer.writerows(data_list) print('csv文件写入完成') if __name__ == '__main__': main()
众志成城,共抗疫情