import requests as rq from bs4 import BeautifulSoup import json import time import pandas as pd home_url = 'https://bj.lianjia.com/zufang' headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36' } # 首页 home_rt = rq.get(home_url, headers=headers).text home_soup = BeautifulSoup(home_rt, 'lxml') # 从首页获取到各个区域的入口链接 district_url_rt = home_soup.find_all('li', attrs={'class': 'filter__item--level2', 'data-type': 'district'}) district_urls = [] for i in range(1,len(district_url_rt)): district_name = district_url_rt[i].a.string # 区域名称 dis_url = district_url_rt[i].a.attrs['href'] dis_url = 'https://bj.lianjia.com' + dis_url # 区域链接 district_urls.append([district_name, dis_url]) print(district_urls) print('区域接口获取完毕') finally_house_result = [] # 遍历各个区域链接，分别从每个入口中获取到信息 for dis_url in district_urls: time.sleep(5) district_name = dis_url[0] + '区' district_url = dis_url[1] district_rt = rq.get(district_url, headers=headers) district_rt = district_rt.text district_soup = BeautifulSoup(district_rt, 'lxml') page_num = int(district_soup.find('div', attrs={'class': 'content__pg'}).attrs['data-totalpage']) # 当前区域房屋信息 网页数 # 遍历所有页，获取所有页 房屋标题+url house_titurl = [] for page in range(1, page_num+1): time.sleep(0.8) page_url = district_url + f'/pg{page}' # 当前页面链接 page_results = rq.get(page_url, headers=headers).text page_soup = BeautifulSoup(page_results) current_page_rts = page_soup.find_all('div', attrs={'class': 'content__list--item'}) # 当前页面区域房屋信息列表 # 遍历当前页面，获取 所有房屋 标题+ url for houselist_rt in current_page_rts: house_url = 'https://bj.lianjia.com' + houselist_rt.a['href'] # urs house_title = houselist_rt.a.img['alt'] # 标题 address_list = houselist_rt.div.find('p', attrs={'class': 'content__list--item--des'}).find_all('a') address = address_list[1].string + '.' + address_list[2].string # 地址 house_titurl.append([house_title, address, house_url]) district_num = len(house_titurl) print(f'{district_name}房屋标题&url获取完毕，共{district_num}套租房信息') # 遍历当前区域所有的房屋标题+链接，获取房屋具体信息 for house_page in house_titurl: time.sleep(0.6) house_title = house_page[0] # 房屋标题 address = house_page[1] # 地址 house_url = house_page[2] # 房屋链接 house_rt = rq.get(house_url, headers=headers).text house_soup = BeautifulSoup(house_rt) house_rt1 = house_soup.find_all('li', attrs={'class': 'table_col'}) pay_method = house_rt1[5].string # 支付方式 rent = house_rt1[6].string + house_rt1[1].find('span').string # 房租 deposit = house_rt1[7].string + house_rt1[2].find('span').string # 押金 service_fee = house_rt1[8].string + house_rt1[3].find('span').string # 服务费 agency_fee = house_rt1[9].string + house_rt1[4].find('span').string # 中介费 house_rt2 = house_soup.find_all('li', attrs={'class': 'fl oneline'}) size = house_rt2[1].string[3:] # 面积 toward = house_rt2[2].string[3:] # 朝向 in_time = house_rt2[5].string[3:] # 入住时间 rent_term = house_rt2[7].string[3:] # 租期 storey = house_rt2[10].string[3:] # 楼层 elevator = house_rt2[11].string[3:] # 电梯 gas = house_rt2[17].string[3:] # 燃气 # 配套设施 supporting_facilities = [] for faci in range(21, len(house_rt2)): supporting_facilities.append(house_soup.find_all('li', attrs={'class': 'fl oneline'})[faci].text.strip()) supporting_facilities = json.dumps(supporting_facilities, ensure_ascii=False) # 中介信息 agency_names = house_soup.find_all('a', attrs={'class': 'name'}) agency_phones = house_soup.find_all('div', attrs={'class': 'phone'}) agency_scores = house_soup.find_all('div', attrs={'class': 'rate'}) agency_list = [] for name, phone, score in zip(agency_names, agency_phones, agency_scores): agency_list.append({'中介姓名': name.string, '电话': phone.string, '评分': score.text.strip()}) agency_list = json.dumps(agency_list, ensure_ascii=False) finally_house_result.append([district_name, address, house_title, size, toward, storey, elevator, gas, supporting_facilities, rent_term, in_time, rent, deposit, service_fee, agency_fee, agency_list]) print(f'{district_name}房屋信息获取完毕，共{district_num}套') data_num = len(finally_house_result) columns = ['区域', '地址', '标题', '面积', '朝向', '楼层', '电梯', '燃气', '配套设施', '租期', '入住时间', '房租', '押金', '服务费', '中介费', '中介联系方式'] house_finally_dfdata = pd.DataFrame(finally_house_result, columns=columns) house_finally_dfdata.to_excel('d:\\Desktop\\20191124链家北京各城区租房信息.xlsx') print(f'北京市各城区租房信息获取完毕，共{data_num}套')
