爬取百度贴吧

这个爬虫代码结构已经比较清晰了，以后的爬虫都可以套用这个模板

from urllib import request,parse
import time
import random
from useragents import ua_list

class BaiduSpider(object):
  def __init__(self):
    self.url = 'http://tieba.baidu.com/f?kw={}&pn={}'

  # 获取响应内容
  def get_html(self,url):
    headers = {
      'User-Agent': random.choice(ua_list)
    }
    req = request.Request(url=url,headers=headers)
    res = request.urlopen(req)
    html = res.read().decode('utf-8')

    return html

  # 提取所需数据
  def parse_html(self):
    pass

  # 保存
  def write_html(self,filename,html):
    with open(filename,'w',encoding='utf-8') as f:
      f.write(html)

  # 主函数
  def main(self):
    name = input('请输入贴吧名:')
    begin = int(input('请输入起始页:'))
    end = int(input('请输入终止页:'))
    # url缺2个数据: 贴吧名 pn
    params = parse.quote(name)
    for page in range(begin,end+1):
      pn = (page-1)*50
      url = self.url.format(params,pn)
      filename = '{}-第{}页.html'.format(name,page)
      # 调用类内函数
      html = self.get_html(url)
      self.write_html(filename,html)
      # 每爬取1个页面随机休眠1-3秒钟
      time.sleep(random.randint(1,3))

      print('第%d页爬取完成' % page)


if __name__ == '__main__':
  start = time.time()
  spider = BaiduSpider()
  spider.main()
  end = time.time()
  print('执行时间:%.2f' % (end-start))

ua_list里的User-Agent只写了几个，如果常用可以再加：

ua_list = [
  'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.163 Safari/535.1',
  'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:6.0) Gecko/20100101 Firefox/6.0',
  'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; InfoPath.3)',
]

 

效果：