用python和BeautifulSoup抓取百度搜索结果10-20页面中的网站链接

#-*-coding=utf-8-*-
#author:zhangle

import urllib2
import urllib
import chardet
import thread
import threading
from bs4 import BeautifulSoup as BS

myLock = threading.RLock()
class GetUrls(object):
	pageCount = 1
	search_url = 'http://www.baidu.com/s?wd=key'
	req_header =  {'User-Agent':'Mozilla/5.0(Windows;U;Windows NT 6.1;en-US; rv:1.9.1.6)Gecko/20091201 Firefox/3.5.6'}
	def __init__(self,inputInfo):
		self.inputInfo = inputInfo
		GetUrls.search_url = GetUrls.search_url.replace('key',self.inputInfo)

	#detect the coding of the html page
	def __detectCode(self,url):
		htmlInfo = urllib.urlopen(url).info()
		coding = htmlInfo.getparam('charset')
		if coding is None:
			htmlInfo = urllib.urlopen(url).read()
			coding = chardet.detect(htmlInfo)['encoding']
			if coding is None:
				coding = 'utf-8'
		coding = coding.lower()
		return coding

	#get the html title 
	def __getTitle(self,url):
		coding = self.__detectCode(url)
		try:
			titleReq = urllib2.Request(url,None,GetUrls.req_header)
			titleRes = urllib2.urlopen(titleReq)
			html = titleRes.read()
			titleSoup = BS(html.decode(coding,'ignore'))
			title = titleSoup.title.string
			return title
		except urllib2.HTTPError:
			return None
		except urllib2.URLError:
			return None
	
	#get the information inside the html page
	def __getInfo(self,redUrl):
		#myLock.acquire()
		with open('info.txt',mode='a') as a_file:
			title = self.__getTitle(redUrl)
			if title: 
				#print title.encode('gbk','ignore')
				a_file.write(title.encode('gbk','ignore') + '\n')
				a_file.write(redUrl + '\n\n')
		#myLock.release()

	def __searchUrls(self,url):
		if GetUrls.pageCount > 20:
			return
		else:
			req = urllib2.Request(url,None,GetUrls.req_header)
			res = urllib2.urlopen(req)
			html = res.read()
			soup = BS(html.decode('utf-8','ignore'))

			#get the 100 urls from page 10 to page 20
			if GetUrls.pageCount > 10:
				htmlList = soup.find_all('h3')
				for hh in htmlList:
					#get url in the html page
					urlInPage = hh.a.get('href')

					#get url after redirecting
					try:
						req = urllib2.Request(urlInPage,None,GetUrls.req_header)
						redUrl = urllib2.urlopen(req).geturl()
					except urllib2.HTTPError:
						redUrl = urlInPage
					thread.start_new_thread(self.__getInfo,(redUrl,))
						
			GetUrls.pageCount += 1
			pNode = soup.find_all('span',text=GetUrls.pageCount)
			nextUrl = 'http://www.baidu.com' + pNode[0].parent.get('href')

			self.__searchUrls(nextUrl)

	def UrlParse(self):
		self.__searchUrls(GetUrls.search_url)

if __name__ == '__main__':
	getUrlInfo = GetUrls('挖掘机')
	getUrlInfo.UrlParse()