无法解决的问题

学习python时做了一个爬虫爬取百度贴吧的内容，但是用BeautifulSoup得到的结果使用find_all函数却无法获取。

 

getCommentInfo.py:

from urllib import request
import requests
from bs4 import BeautifulSoup
from mylog import MyLog as mylog
import random

class Item(object):
    title = None    #帖子标题
    firstAuthor = None  #创建者
    firstTime = None    #创建时间
    reNum = None    #总回复数
    content = None  #最后回复内容
    lastAuthor = None   #最后回复者
    lastTime = None     #最后回复时间

class GetTiebaInfo(object):
    def __init__(self,url):
        self.url = url
        self.log = mylog()
        self.pageSum = 5
        self.urls = self.getUrls(self.pageSum)
        self.items = self.spider(self.urls)
        self.pipelines(self.items)

    def getUrls(self,pageSum):
        urls = []
        pns = [str(i*50) for i in range(pageSum)]
        ul = self.url.split('=')
        for pn in pns:
            ul[-1] = pn
            url = '='.join(ul)
            urls.append(url)
        self.log.info(u"获取URLS成功 ")
        return urls

    def spider(self,urls):
        items = []
        for url in urls:
            htmlContent = self.getResponseContent(url)
            with open("content.html","w",encoding='utf-8') as f:
                f.write(htmlContent)
            soup = BeautifulSoup(htmlContent,'lxml')
            with open('soup.txt','w',encoding='utf-8') as fp:
                fp.write(soup.text)

            tagsli = soup.find_all('li',attrs={'class':'j_thread_list clearfix'})
            for tag in tagsli:
                item = Item()
                item.title = tag.find('a',attrs={'class':'j_th_tit '}).get_text().strip()
                item.firstAuthor = tag.find('span',attrs={'class':'frs-author-name-wrap'}).a.get_text().strip()
                item.firstTime = tag.find('span',attrs={'title':u'创建时间'.encode('utf-8')}).get_text().strip()
                item.reNum = tag.find('span',attrs={'title':u'回复'.encode('utf-8')}).get_text().strip()
                item.content = tag.find('div',attrs={'class':'threadlist_abs threadlist_abs_onlyline '}).get_text().strip()
                item.lastAuthor = tag.find('span',attrs={'class':'tb_icon_author_rely j_replyer'}).a.get_text().strip()
                item.lastTime = tag.find('span',attrs={'title':u'最后回复时间'.encode('utf-8')}).get_text().strip()
                items.append(item)
                self.log.info(u'获取标题为<<%s>>的项成功 ...' %item.title)
        return items

    def pipelines(self,items):
        fileName = u'百度贴吧_权力的游戏.txt'.encode('utf-8')
        with open(fileName,'w') as fp:
            for item in items:
                fp.write('title:%s \t author:%s \t firstTime:%s \n content:%s \n return:%s \n lastAuthor:%s \t lastTime:%s \n\n\n\n'
                         %(item.title.encode('utf-8'),item.firstAuthor.encode('utf-8'),item.firstTime.encode('utf-8'),item.content.encode('utf-8'),item.lastTime.encode('utf-8')))
                self.log.info(u'标题为<<%s>>的项输入到"%s"成功' %(item.title,fileName.decode('utf-8')))

    def getResponseContent(self,url):
        header = {
            'Accept': 'text/heml,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
            'Accept-Encoding': 'gzip,deflate,sdch', 'Accept-Language': 'zh_CN,zh;q=0.8',
            'Connect': 'keep-alive',
            'User-Agent': 'Mozilla/5.0(Windows NT 6.3;WOW64) AppleWebKit/537.36 (KHTML,like Gecko) Chrome/43.0.235'
        }
        timeout = random.choice(range(80, 180))
        try:
            response = requests.get(url,headers = header,timeout = timeout)
            response.encoding = 'utf-8'
        except:
            self.log.error(u'Python 返回 URL:%s 数据失败' %url)
        else:
            self.log.info(u'Python 返回URL:%s 数据成功' %url)
            return response.content.decode('utf-8')
if __name__=='__main__':
    url = u'http://tieba.baidu.com/f?kw=权力的游戏&ie=utf-8&pn=50'
    GTI = GetTiebaInfo(url)

 

mylog.py

import logging
import getpass
import sys

#定义MyLog类
class MyLog(object):
    def __init__(self):
        self.user = getpass.getuser()
        self.logger = logging.getLogger(self.user)
        self.logger.setLevel(logging.DEBUG)

        #日志文件名
        self.logFile = sys.argv[0][0:-3] + '.log'
        self.formatter = logging.Formatter('%(asctime)-12s %(levelname)-8s %(name)-10s %(message)-12s\r\n')

        #文件显示到屏幕并输出到日志文件
        self.logHand = logging.FileHandler(self.logFile,encoding='utf-8')
        self.logHand.setFormatter(self.formatter)
        self.logHand.setLevel(logging.DEBUG)

        self.logHandSt = logging.StreamHandler()
        self.logHand.setFormatter(self.formatter)
        self.logHandSt.setLevel(logging.DEBUG)

        self.logger.addHandler(self.logHand)
        self.logger.addHandler(self.logHandSt)

    def debug(self,msg):
        self.logger.debug(msg)

    def info(self,msg):
        self.logger.info(msg)

    def warn(self,msg):
        self.logger.warning(msg)

    def error(self,msg):
        self.logger.error(msg)

    def critical(self,msg):
        self.logger.critical(msg)

# if __name__=='__main__':
# #     mylog = MyLog()
# #     mylog.debug(u"I'm debug 测试中文")
# #     mylog.info("I'm info")
# #     mylog.warn("I'm warn")
# #     mylog.error(u"I'm error 测试中文")
# #     mylog.critical("I'm critical")

 

错误：

　　在getCommentInfo.py中40行左右的htmlContent可得到原html的正确内容，但经BeautifulSoup后，返回的soup内容变化，导致无法爬取结果。可从两个调式文件content.html和soup.txt得知。