爬虫——三个小实战

贴吧爬取

写代码前，构思需要的功能块；写代码时，把各个功能模块名提前写好

初始化

初始化必要参数，完成基础设置 爬取百度贴吧lol吧：爬取地址中的get参数须传递(可以指定不同主题的贴吧和页码)

• 主题名
• 初始网址
• 请求头

生成网址

生成每一页的路由地址

• 根据列表生成式生成多个页面的地址

下载

get请求给每一页的地址，爬取页面

保存

保存爬取结果到文件中，把每一页爬取结果写入到对应名字的文件中

控制流程

将以上爬取操作封装到run函数中，方便外部对象调用，以后会在此添加多线程

• 生成要爬取的每一页的路由地址
• 通过for循环遍历每一个路由地址
• 对每个路由地址进行爬取和获取页码操作，并进行保存

源码

import requests

class TiebaSpider:
    def __init__(self, tieba_name_crawl):
        """
        初始化必要参数，完成基础设置
        爬取百度贴吧lol吧：爬取地址中的get参数须传递(可以指定不同主题的贴吧和页码)
        """
        self.tieba_name = tieba_name_crawl
        self.url_base = 'https://tieba.baidu.com/f?kw=' + tieba_name_crawl + '&ie=utf-8&pn={}'
        self.headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:61.0) Gecko/20100101 Firefox/61.0'}

    def make_url(self):
        """
        生成每一页的路由地址
        :return:(列表生成式)
        """
        return [self.url_base.format(i) for i in range(4)]

    def download_url(self, url_str):
        """
        get请求给每一页的地址，爬取页面
        :param url_str: 每一页的路由地址
        :return: 爬取的结果
        """
        result = requests.get(url_str, headers=self.headers)
        return result.text

    def save_result(self, result, page_num):
        """
        保存爬取结果到文件中
        :param result: 每一页的爬取结果
        :param page_num: 页码，方便分类保存
        :return: 把每一页爬取结果写入到对应名字的文件中
        """
        # with open('./download/lol' + str(page_num) + '.html', 'ab') as f:
        #     f.write(result.encode('utf-8'))
        file_path = './download/{}~第{}页.html'.format(self.tieba_name,page_num)
        with open(file_path,'wb') as f:
            f.write(result.encode('utf-8'))

    def run(self):
        """
        将以上爬取操作封装到run函数中，方便外部对象调用，以后会在此添加多线程
        · 生成要爬取的每一页的路由地址
        · 通过for循环遍历每一个路由地址
        · 对每个路由地址进行爬取和获取页码操作，并进行保存
        :return:
        """
        url_lists = self.make_url()
        for url_str in url_lists:
            result_str = self.download_url(url_str)
            p_num = url_lists.index(url_str) + 1
            self.save_result(result=result_str,page_num=p_num)

if __name__ == '__main__':
    tieba_spider = TiebaSpider('lol')
    tieba_spider.run()

 

爬取糗事百科

import requests
from bs4 import BeautifulSoup
import lxml.html

class QiushiSpider:
    def __init__(self):
        """
        初始化必要参数，完成基础设置
        """
        # self.tieba_name = qiushi_name_crawl
        # https: // www.qiushibaike.com / 8
        # hr / page / 2 /
        self.url_base = 'https://www.qiushibaike.com/8hr/page/{}/'
        # self.url_base = 'https://tieba.baidu.com/f?kw=' + qiushi_name_crawl + '&ie=utf-8&pn={}'
        self.headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:61.0) Gecko/20100101 Firefox/61.0'}

    def make_url(self):
        return [self.url_base.format(i) for i in range(4)]

    def download_url(self, url_str):
        result = requests.get(url_str, headers=self.headers)
        #----------
        # html = lxml.html.fromstring(result.text)
        # html_data = html.xpath('//div[@class="content"]/span[1]/text()')
        # data_all = []
        # # for h in html_data:
        # #     data_all.append(h)
        # return html_data
        #-----------
        return result.text

    def save_result(self, result, page_num):
        with open('./download/qiushi' + str(page_num) + '.html', 'ab') as f:
            f.write(result.encode('utf-8'))


# qiushi = QiushiSpider()
# qiushi_url = qiushi.make_url()
# j = 1
# for i in qiushi_url:
#     qiushi_text = qiushi.download_url(url_str=i)
#     qiushi.save_result(result=qiushi_text, page_num=j)
#     j += 1

 

爬取国家信息

BeautifulSoup方式

import requests
from bs4 import BeautifulSoup
class CountrySoup:
    def __init__(self,country_name):
        self.country_name = country_name
        self.url_base = 'http://example.webscraping.com/places/default/view/{}'.format(self.country_name)
        self.headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:61.0) Gecko/20100101 Firefox/61.0',}

    def download_url(self):
        result = requests.get(self.url_base,headers=self.headers)
        soup = BeautifulSoup(result.text,'lxml')
        tr = soup.find(attrs={'id':"places_country__row"})
        print(tr,type(tr))
        td = tr.find(attrs={'class':"w2p_fw"})
        print(td,type(td))

        print(td.text)

lxml方式

class CountrySpider:
    def __init__(self,country_name):
        self.country_name = country_name
        self.url_base = 'http://example.webscraping.com/places/default/view/{}'.format(self.country_name)
        self.headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:61.0) Gecko/20100101 Firefox/61.0',}

    def download_url(self,url_str):
        result = requests.get(url_str,headers=self.headers)
        html = lxml.html.fromstring(result.text)
        data_country = html.xpath('//tr[@id="places_country__row"]/td[@class="w2p_fw"]/text()')
        data_capital = html.xpath('//tr[@id="places_capital__row"]/td[@class="w2p_fw"]/text()')
        data_area = html.xpath('//tr[@id="places_area__row"]/td[@class="w2p_fw"]/text()')
        data_all = ['国家：'+data_country[0],'首都：'+data_capital[0],'国土面积：'+data_area[0]]
        return data_all
        # print(html_data)

    def save_result(self,result):
        print(type(result),result)
        for r in result:
            r = r + '\n'
            with open('./country.txt','ab') as f:
                f.write(r.encode('utf-8'))
        # with open('./country.txt','ab') as f:
        #     f.writelines(result)
    def run(self):
        result = self.download_url(self.url_base)
        self.save_result(result)


if __name__ == '__main__':
    # c = CountrySpider('Bolivia-27')
    # c.run()
    s = CountrySoup('Bolivia-27')
    s.download_url()