Python爬虫(三)--百度贴吧 

  1. #coding=utf-8
  3. import urllib2
  4. import urllib
  5. import re
  7. # 处理页面标签类
  8. class Tool:
  9.     # 去除img标签，7位长空格
  10.     removeImg = re.compile(r'<img.*?>| {7}|')
  11.     # 删除超链接标签
  12.     removeAddr = re.compile('<a.*?>|</a>')
  13.     # 把换行标签换位\n
  14.     replaceLine = re.compile('<tr>|<div>|</div|</p>')
  15.     # 将制表<td>换位\t
  16.     replaceTD = re.compile('<td>')
  17.     # 将段落开头换为\n加两空格
  18.     replacePara = re.compile('<p.*?>')
  19.     # 将换行符或双换行符换为\n
  20.     replaceBR = re.compile('<br><br>|<br>')
  21.     # 删除其他标签
  22.     removeExtraTag = re.compile('<.*?>')
  23.     def replace(self,x):
  24.         x = re.sub(self.removeImg,"",x)
  25.         x = re.sub(self.removeAddr,"",x)
  26.         x = re.sub(self.replaceLine,"\n",x)
  27.         x = re.sub(self.replaceTD,"\t",x)
  28.         x = re.sub(self.replacePara,"\n",x)
  29.         x = re.sub(self.replaceBR,"\n",x)
  30.         x = re.sub(self.removeExtraTag,"",x)
  31.         # 将前后多余的内容删除
  32.         return x.strip()
  35. # 百度贴吧爬虫类
  36. class BDTB:
  38.     # 初始化，传入基地址，传入是否“只看楼主”参数
  39.     def __init__(self,baseUrl,seeLz,floorTag):
  40.         # base连接地址
  41.         self.basURL = baseUrl
  42.         # 是否只看楼主
  43.         self.seeLZ = '?see_lz='+str(seeLz)
  44.         # 工具类
  45.         self.tool = Tool()
  46.         # 全局file变量，文件写入操作对象
  47.         self.file = None
  48.         # 楼层标号,初始为1
  49.         self.floor = 1
  50.         # 默认的标题
  51.         self.defaultTitle = u"百度贴吧"
  52.         # 是否写入楼分隔符的标记
  53.         self.floorTag = floorTag
  56.     # 获取该页(pageNum)的代码
  57.     def getPage(self,pageNum):
  58.         try:
  59.             url = self.basURL + self.seeLZ + '&pn=' + str(pageNum)
  60.             request = urllib2.Request(url)
  61.             response = urllib2.urlopen(request)
  62.             # 返回utf-8格式编码内容
  63.             return response.read().decode('utf-8')
  64.         except urllib2.URLError,e:
  65.             if hasattr(e,"reason"):
  66.                 print u"连接百度贴吧失败，错误原因",e.reason
  67.                 return None
  69.     # 获得页面标题
  70.     def getTitle(self,page):
  71.         # 获取标题的正则表达式
  72.         pattern = re.compile(r'<h3 class="core_title_txt.*?>(.*?)</h3>',re.S)
  73.         result = re.search(pattern, page)
  74.         if result:
  75.             return result.group(1).strip()
  76.         else:
  77.             return None
  79.     # 提取页数
  80.     def getPageNum(self,page):
  81.         # 获取页数的正则表达式
  82.         pattern = re.compile(r'reply_num.*?</span.*?class="red">(.*?)</span>',re.S)
  83.         result = re.search(pattern,page)
  84.         if result:
  85.             return result.group(1).strip()
  86.         else:
  87.             return None
  89.     # 提取内容
  90.     def getContent(self,page):
  91.         pattern = re.compile(r'post_content.*?>(.*?)</div>',re.S)
  92.         items = re.findall(pattern,page)
  93.         contents = []
  94.         for item in items:
  95.             #将文本内容进行处理，同时在前后加上换行符
  96.             content = "\n" + self.tool.replace(item)+"\n"
  97.             contents.append(content.encode('utf-8'))
  98.         return contents
  100.     def setFileTitle(self,title):
  101.         if title is not None:
  102.             self.file = open(title+".txt","w+")
  103.         else:
  104.             self.file = open(self.defaultTitle+".txt","w+")
  106.     # 写文件
  107.     def writeData(self,contents):
  108.         for item in contents:
  109.             if self.floorTag == "1":
  110.                 floorLine = "\n"+str(self.floor)+u"--------------------------------------\n"
  111.                 self.file.write(floorLine)
  112.             self.file.write(item)
  113.             self.floor += 1
  115.     def start(self):
  116.         indexPage = self.getPage(1)
  117.         pageNum = self.getPageNum(indexPage)
  118.         title = self.getTitle(indexPage)
  119.         self.setFileTitle(title)
  120.         if pageNum == None:
  121.             print  u"URL已失效，请重试"
  122.             return
  123.         try:
  124.             print "该帖子共有"+str(pageNum)+"页。"
  125.             for i in range(1,int(pageNum)+1):
  126.                 print "正在写入第"+str(i)+"页数据"
  127.                 page = self.getPage(i)
  128.                 contents = self.getContent(page)
  129.                 self.writeData(contents)
  130.         except IOError,e:
  131.             print "写入异常，原因:"+e.message
  132.         finally:
  133.             print "写入完成"
  136. baseURL = 'http://tieba.baidu.com/p/3138733512'
  137. seeLz = raw_input("是否只获取楼主发言，是输入1，否输入0\n")
  138. floorTag = raw_input('是否写入楼层信息，是输入1，否输入0\n')
  139. bdtb = BDTB(baseURL,seeLz,floorTag)
  140. bdtb.start()




posted @ 2016-10-17 20:44  ldbmcs  阅读(323)  评论(0)  编辑  收藏  举报