python抓取链家房源信息

闲着没事就抓取了下链家网的房源信息，抓取的是北京二手房的信息情况，然后通过网址进行分析，有100页，并且每页的url都是类似的

url = 'https://bj.lianjia.com/ershoufang/pg' + 页数，然后请求是get 请求，所以静态页面，然后依次来进行分析，并且存储在mongodb中，每次插入的时候还是要字符串装换成json格式在进行插入，页面的解析用的是bs，解析很方便，代码用的是单进程，耗时是大致66s，因为怕ip被封，所以在每次页面请求之后都要sleep 1秒。

#-*-coding:utf-8-*-
import urllib
import urllib2
import re
import requests
import json
import lxml
from bs4 import BeautifulSoup
import time
from pymongo import MongoClient

from lxml import etree
client = MongoClient('localhost',27017)
db = client.test
House = db.House
headers = {
    'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
    'Accept-Encoding':'gzip, deflate, br',
    'Accept-Language':'zh-CN,zh;q=0.9',
    'Cache-Control':'max-age=0',
    'Connection':'keep-alive',
    'Cookie':'......',
    'Host':'bj.lianjia.com',
    'Upgrade-Insecure-Requests':'1',
    'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36'
}
URL = 'https://bj.lianjia.com/ershoufang/pg'
def download(url):
    num_try = 2
    while num_try > 0:
        num_try -= 1
        try:
            content = requests.get(url,headers = headers)
            return content.text
        except urllib2.URLError as e:
            print 'Download error',e.reason

    return None


def get_message(url):
    html = download(url)

    soup = BeautifulSoup(html,'html.parser')
    prices = soup.find_all('div','priceInfo')
    total_price = []
    for each in prices:
        total_price.append(each.span.string)
    address = []
    house_types = []
    areas = []
    towards = []
    decorates = []
    elevates = []
    message = soup.find_all('div',attrs={'class':'houseInfo'})
    for each in message:
        List = each.get_text().split('|')
        address.append(List[0].strip())
        house_types.append(List[1].strip())
        areas.append(List[2].strip())
        towards.append(List[3].strip())
        decorates.append(List[4].strip())
        if len(List) == 5:
            elevates.append("None")
        else:
            elevates.append(List[5].strip())
    for addres,house_type,area,price,toward,decorate,elevate in zip(address,house_types,areas,total_price,towards,decorates,elevates):
        mess = "{\"Address\":\"%s\",\"House_type\":\"%s\",\"Area\":\"%s\",\"Price\":\"%s\",\"Toward\":\"%s\",\"Decorate\":\"%s\",\"Elevete\":\"%s\"}"%(addres,house_type,area,price,toward,decorate,elevate)
        print mess
        message = json.loads(mess)
        House.insert(message)

if __name__ == '__main__':
    t = time.time()
    print t
    for num in xrange(1,101):

        url = URL + str(num)
        print url
        get_message(url)
        time.sleep(1)
    t1 = time.time()
    print 'Total time:'
    print t1 - t - 100