爬取链家二手房信息【爬虫模板】

爬虫模板几乎一个样儿
提前安装pandas用于存储数据、requests处理URL资源、Bs4(BeautifulSoup4)提取数据、lxml读取网页

爬取链家二手房信息

# -*- coding: utf-8 -*-
# @Author  : LEHOSO
# @FileName: Lianjia2.py
# @Time    : 2021/10/11 16:55

import time

import pandas as pd
import requests
from bs4 import BeautifulSoup

# 表头
header = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
                  'AppleWebKit/537.36 (KHTML, like Gecko) '
                  'Chrome/95.0.4638.54 Safari/537.36 Edg/95.0.1020.30'
}
a = []


def get_info(url):
    wb_data = requests.get(url, headers=header)
    # 爬取整个网页
    soup = BeautifulSoup(wb_data.text, 'lxml')
    # 网页单个元素
    ranks = soup.select('div.totalPrice.totalPrice2 > span')
    title = soup.select('div.title > a')
    location = soup.select('div.flood > div > a')
    area = soup.select('div.flood > div > a:nth-child(3)')
    fllowInfo = soup.select('div.followInfo')
    # 存入进列表
    for ranks, titles, locaitons, areas, fllowInfos in zip(ranks, title, location, area, fllowInfo):
        data = {
            '价格': ranks.get_text().strip(),
            '标题': titles.get_text().strip(),
            '位置': locaitons.get_text().strip() + '-' + areas.get_text().strip(),
            '关注': fllowInfos.get_text().strip().split('/')[0],
            '距今发布日期': fllowInfos.get_text().strip().split('/')[1]
        }
        a.append(data)
        print(data)


if __name__ == '__main__':
    # 网址路径
    urls = [
        'https://cq.lianjia.com/ershoufang/'
    ]
    for url in urls:
        get_info(url)
        time.sleep(2)
    # pandas存入数据
    df_out = pd.DataFrame(a, columns=['价格', '标题', '位置', '关注', '距今发布日期'])
    #导出为xlsx格式
    df_out.to_excel('aaa.xlsx')