#coding:utf8
#爬取网上车市[http://www.cheshi.com/]的数据
import requests, json, time, re, os, sys, time,urllib2,shutil,string
import threading
import MySQLdb
import redis
from pyquery import PyQuery as pq
from urlparse import urljoin
from selenium import webdriver
#设置utf-8编码格式
reload(sys)
sys.setdefaultencoding( "utf-8" )
#读取文件内容
def getLines(filename):
file_object = open(filename,'rb')
lines = file_object.readlines()
return lines
#根据url_name获取url_type_id
def get_url_type_id(v_url_name):
#3000 品牌 奥迪
#4000 奥迪 奥迪A6
url_type_id = ''
for line in getLines('/home/shutong/crawl/car/script/brand.ini'):
line = line.strip()
url_cate = line.split(',')[1]
url_name = line.split(',')[2]
if v_url_name.strip() == url_name.strip():
url_type_id = line.split(',')[0]
return url_type_id
break
else :
continue
return url_type_id
class ResultData():
'''数据文件类'''
def __init__(self,industry_id,url_type_id,url_name,url_value,web_type_id,web_name,date_id):
self.industry_id = industry_id
self.url_type_id = url_type_id
self.url_name = url_name
self.url_value = url_value
self.web_type_id = web_type_id
self.web_name = web_name
self.date_id = date_id
def __str__(self):
return self.industry_id,self.url_type_id,self.url_name,self.url_value,self.web_type_id,self.self.web_name,ResultData.date_id
class Base(object):
'''文件保存的基类'''
def __init__(self,dev_prd_flag):
self.dev_prd_flag = dev_prd_flag
pass
#print "This is init function"
#保存数据到文件文件
def _saveContext(self,filename,*name):
format = '^'
context = name[0]
for i in name[1:]:
context = context + format + str(i)
context = str(context).replace('（','(').replace('）',')').replace('，',',').replace('：',':')
if self.dev_prd_flag != 'prd':
print context
else:
#去除文件路径名首位空格
filename = filename.strip()
#读取目录名称
path = os.path.dirname(filename)
#如果目录不存在则创建目录
if not os.path.exists(path):
os.makedirs(path)
#读取文件名称,以追加的方式写文件
name = os.path.basename(filename)
fp = open(filename,'a')
fp.write(context+'\n')
fp.close()
def saveData(self,filename,ResultData):
if ResultData.url_type_id:
self._saveContext(filename,ResultData.industry_id,ResultData.url_type_id,ResultData.url_name,ResultData.url_value,ResultData.web_type_id,ResultData.web_name,ResultData.date_id)
else:
#将数据进行保存在redis中
r = redis.Redis(host='192.168.122.140',port=6379,db=0)
r.sadd('errorList',ResultData.industry_id+'^'+ResultData.url_name+'^'+ResultData.url_value)
def __str__(self):
return '保存文件的基类'
class Crawl(Base):
'''爬虫基础类'''
driver = None
#构造方法
def __init__(self,name,dev_prd_flag):
super(Crawl,self).__init__(dev_prd_flag='dev')
self.dev_prd_flag = dev_prd_flag
self.name = name
#self.driver = init_driver()
'''初始化启动浏览器'''
def init_driver(self):
ua = "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.3 Safari/537.36"
cap = webdriver.DesiredCapabilities.PHANTOMJS
cap["phantomjs.page.settings.resourceTimeout"] = 20000
cap["phantomjs.page.settings.loadImages"] = True
cap["phantomjs.page.settings.disk-cache"] = True
cap["phantomjs.page.settings.userAgent"] = ua
cap["phantomjs.page.customHeaders.User-Agent"] =ua
cap["phantomjs.page.customHeaders.Referer"] = "http://tj.ac.10086.cn/login/"
driver = webdriver.PhantomJS(executable_path='/home/shutong/phantomjs/bin/phantomjs',desired_capabilities=cap, service_args=['--ignore-ssl-errors=true'])
driver.set_page_load_timeout(60)
driver.set_script_timeout(60)
#return driver
self.driver = driver
#获取网页文本
def getHtml(self,url,code='utf-8'):
html = ''
try:
if self.driver:
self.driver.get(url)
html = self.driver.page_source
else :
headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:23.0) Gecko/20100101 Firefox/23.0'}
request = urllib2.Request(url,headers=headers)
response = urllib2.urlopen(request,data=None,timeout=60)
if code:
if code == 'gbk':
html = unicode(response.read(),'gbk')
else:
html = unicode(response.read(),str(code))
except:
pass
finally:
return html
'''析构方法'''
def __del__(self):
if self.driver:
self.driver.quit()
print "浏览器成功关闭"
else:
print "浏览器未打开使用"
def __str__(self):
return "爬虫基础类"
def start_crawl(url):
#连接redis数据库
r = redis.Redis(host='192.168.122.140',port=6379,db=0)
urllist = []
html = crawl.getHtml(url,'gbk')
d = pq(html)
for a in d('a'):
a = pq(a)
try:
url_value = urljoin(url,a.attr('href'))
name = a.text()
#if re.compile(r'([a-z]+) ([a-z]+)', re.I)
#http://newcar.xcar.com.cn/162/
if re.match( r'http://newcar.xcar.com.cn/[0-9]{1,10}/$', url_value, re.M|re.I):
#print url_value,name
#urllist.append(url_value)
#将数据存储在redis中
#r.sadd('urllist',url_value)
pass
elif re.match(r'http://newcar.xcar.com.cn/m[0-9]{1,10}/$',url_value,re.M|re.I):
r.sadd('urllist',url_value)
except:
pass
#for index in urllist:
for index in list(set(urllist)):
print index
try:
#return start_crawl(index)
pass
except:
pass
def start_wscs_crawl(url):
#生产或者测试标志 dev为测试 prd为生产
flag = 'prd'
#汽车行业ID
industry_id = '004004'
#移动PC端web_type_id
web_type_id = '0'
#网站名称
web_name = '网上车市'
crawl = Crawl('网上车市',flag)
#加载浏览器
#crawl.init_driver()
html = crawl.getHtml(url)
d = pq(html)
for div in d('div').filter('.list-box'):
div = pq(div)
#品牌
brand = div('div').filter('.lb').find('span').text()
#品牌url
brand_url = urljoin(url,div('div').filter('.lb')('a').attr('href'))
#print brand,brand_url
url_type_id = '3000'
url_name = brand
url_value = brand_url
#保存品牌数据
#crawl._saveContext(filename,industry_id,url_type_id,url_name,url_value,web_type_id,web_name)
resultData = ResultData(industry_id,url_type_id,url_name,url_value,web_type_id,web_name,date_id)
crawl.saveData(filename,resultData)
brand = div('div').filter('.rb')('dl')('dt')('a').text().replace('>>','')
brand_url = urljoin(url,div('div').filter('.rb')('dl')('dt')('a').attr('href'))
#print brand,brand_url
url_type_id = '3000'
url_name = brand
url_value = brand_url
#保存品牌数据
resultData = ResultData(industry_id,url_type_id,url_name,url_value,web_type_id,web_name,date_id)
crawl.saveData(filename,resultData)
#crawl._saveContext(filename,industry_id,url_type_id,url_name,url_value,web_type_id,web_name)
for dd in div('div').filter('.rb')('dl')('dd'):
dd = pq(dd)
car_name = dd('div').filter('.con')('h4').text()
car_url = urljoin(url,dd('div').filter('.con')('h4')('a').attr('href'))
#print car_name,car_url
url_type_id = get_url_type_id(car_name)
url_name = car_name
url_value = car_url
#保存车系数据
#crawl._saveContext(filename,industry_id,url_type_id,url_name,url_value,web_type_id,web_name)
resultData = ResultData(industry_id,url_type_id,url_name,url_value,web_type_id,web_name,date_id)
crawl.saveData(filename,resultData)
#制作汽车实体信息
#品牌 子品牌 车系名称 价位 图片url 网站名称
#多线程启动
def start_mutli_crawl():
list = []
for word in string.uppercase:
#url = 'http://www.autohome.com.cn/grade/carhtml/%s.html' %(word)
url = 'http://product.cheshi.com/static/selectcar/%s.html?t=1519713137030' % (word)
list.append(url)
#定义线程数组
threads = []
#创建线程
for i in range(len(list)):
t = threading.Thread(target=start_wscs_crawl,args=(list[i],))
threads.append(t)
#开启线程
for i in range(len(list)):
threads[i].start()
for i in range(len(list)):
threads[i].join()
#filename = '/home/shutong/crawl/car/script/wscs.csv'
#date_id = '20180227'
date_id = sys.argv[1]
filename = sys.argv[2]
#url = 'http://product.cheshi.com/static/selectcar/B.html?t=1519713137030'
#start_wscs_crawl(url)
#多线程启动
start_mutli_crawl()
