#rr=httppool[getdomain(url)[0]].urlopen("get",getdomain(url)[1],retries=1,headers=headers).data
def do(url):
try:
return httppool[getdomain(url)[0]].get_url(getdomain(url)[1],retries=1,headers=headers).data
except urllib3.HTTPError,e:
print "在获取url=%s的源码时出现了异常:%s"%(url,e)
return ""
rr = with_timeout(5, do, url, timeout_value="")
ee=datetime.now()
print "cost time--------------------------:%s"%(ee-ss).seconds
#rr=httppool[getdomain(url)[0]].get_url(getdomain(url)[1],retries=1,headers=headers)
pdata = StringIO.StringIO(rr)#下面6行是实现解压缩
gzipper = gzip.GzipFile(fileobj = pdata)
isUseGzip=True
try:
data = gzipper.read()
except(IOError):
isUseGzip=False
print 'unused gzip'
data = rr#当有的服务器不支持gzip格
#print data
print "isUseGzip:%s"%isUseGzip
return data
