Python抓取百度搜索结果

Python 用来做一些简单的工作还是不错的，一个练手的代码，抓取百度关键字搜索的结果

 

# coding=utf-8
import urllib2 as url
import string
import urllib
import re

def baidu_search(keyword):
    p= {'wd': keyword}
    res=url.urlopen("http://www.baidu.com/s?"+urllib.urlencode(p))
    html=res.read()
    return html
def getList(regex,text):
    arr = []
    res = re.findall(regex, text)
    if res:
        for r in res:
            arr.append(r)
    return arr
def getMatch(regex,text):
    res = re.findall(regex, text)
    if res:
        return res[0]
    return ""
def clearTag(text):
    p = re.compile(u'<[^>]+>')
    retval = p.sub("",text)
    return retval

html = baidu_search('天下无贼')
content = unicode(html, 'utf-8','ignore')

arrList = getList(u"<table.*?class=\"result\".*?>.*?<\/a>", content)
for item in arrList:
    regex = u"<h3.*?class=\"t\".*?><a.*?href=\"(.*?)\".*?>(.*?)<\/a>"
    link = getMatch(regex,item)
    url = link[0]
    title = clearTag(link[1]).encode('utf8')
    print url
    print title