Python 爬58同城 城市租房信息

爬取完会自动生成csv电子表格文件，含有房价、押付、链接等信息

环境

py2.7

pip install lxml

pip install cssselect

 

#coding:utf-8
import csv
import urllib2
import lxml.html
import time
import sys
from lxml.cssselect import CSSSelector
import threading
reload(sys)
sys.setdefaultencoding('utf8')

print "请输入要爬取得城市简称例如bj（北京）："
CITY=str(raw_input(">>>"))
def download(url, user_agent='Google', num_retries=2):

    headers = {'User-agent': user_agent}
    request = urllib2.Request(url, headers=headers)
    try:
        html = urllib2.urlopen(request).read()
    except urllib2.URLError as e:
        html = None
        if num_retries > 0:
            if hasattr(e, 'code') and 500 <= e.code < 600:
                return download(url, num_retries-1)
    return html


def get_data(url):
    html_text_detail = download(url)
    try:
        tree = lxml.html.fromstring(html_text_detail)
        house_ext = CSSSelector('div.house-pay-way > span:nth-child(3)')
        house_title = CSSSelector('div.main-wrap > div.house-title > h1')
        house_pay_way1 = CSSSelector('div.house-pay-way > span:nth-child(1)')
        house_pay_way2 = CSSSelector('div.house-pay-way > span:nth-child(2)')
        print house_title(tree)[0].text_content()
        print '%s|%s' % (house_pay_way1(tree)[0].text_content(), house_pay_way2(tree)[0].text_content())

        for i in range(7):
            for j in range(2):
                css = 'div.house-desc-item > ul.f14 > li:nth-child(%s) > span:nth-child(%s)' % (i+1, j+1)
                house_info = CSSSelector(css)
                data = [
                ('标题 ： ',house_title(tree)[0].text_content(), '#',url),
                ('价格： ',house_pay_way1(tree)[0].text_content(), '#'),
                ('压付： ',house_pay_way2(tree)[0].text_content(), '#'),
                ('详情： ',house_info(tree)[0].text_content().replace(' ', ''), '#')]
                with open('%s_houses.csv'%CITY,'ab+') as csvfile:
                    writer = csv.writer(csvfile,lineterminator='\n')
                    writer.writerows(data)

    except TypeError as e:
        pass
    except IndexError as e:
        pass

def get_url(html):
    tree = lxml.html.fromstring(html)
    sel = CSSSelector('div.mainbox > div.main > div.content > div.listBox > ul.listUl > li > div.des > h2 > a')
    url_list = []
    for i in sel(tree):
        if i.get('href') not in url_list:
            url_list.append(i.get('href'))
    return url_list


if __name__ == '__main__':
    url_index = 'http://%s.58.com/chuzu/'%CITY
    html_text_list = download(url_index)
    url_list = get_url(html_text_list)

    for url_detail in url_list:
        thr = threading.Thread(target=get_data, args=(url_detail,))
        thr.start()

        time.sleep(0.001)