数据采集实践作业三

作业①

1）

要求：指定一个网站，爬取这个网站中的所有的所有图片，例如中国气象网（http://www.weather.com.cn）。分别使用单线程和多线程的方式爬取。(限定爬取图片数量为学号后3位)

输出信息:

将下载的Url信息在控制台输出，并将下载的图片存储在images子文件夹中，并给出截图。

 单线程：

from bs4 import BeautifulSoup
from bs4 import UnicodeDammit
import urllib.request


def getUrl(start_url):
    try:
        urls=[]
        req=urllib.request.Request(start_url,headers=headers)
        data=urllib.request.urlopen(req)
        data=data.read()
        dammit=UnicodeDammit(data,["utf-8","gbk"])
        data=dammit.unicode_markup
        soup=BeautifulSoup(data,"lxml")
        imagesUrl = soup.select('div[class="oi"] div[class="bt"] a')
        imgsUrl = []#单个图片集的链接
        for x in imagesUrl:
            imgsUrl.append(x['href'])
        return imgsUrl
    except Exception as err:
        print(err)


def imageSpider(start_url,count):
    try:
        urls=[]
        req=urllib.request.Request(start_url,headers=headers)
        data=urllib.request.urlopen(req)
        data=data.read()
        dammit=UnicodeDammit(data,["utf-8","gbk"])
        data=dammit.unicode_markup
        soup=BeautifulSoup(data,"lxml")
        imgsUrl = soup.select('div[class="buttons"] span img')
        for x in imgsUrl:
            urls.append(x['src'])
        for url in urls:
            try:
                    download(url,count)
                    count+=1
                    if count>124:
                        break
            except Exception as err:
                print(err)
        return count
    except Exception as err:
        print(err)

def download(url,count):
    try:
    # 提取文件后缀扩展名
        if (url[len(url) - 4] == "."):
            ext = url[len(url) - 4:]
        else:
            ext = ""
        req = urllib.request.Request(url, headers=headers)
        data = urllib.request.urlopen(req, timeout=100)
        data = data.read()
        fobj = open(r"C:\Users\wcl\Desktop\data collected\test3\3.1.1\\" + str(count) + ext, "wb")
        fobj.write(data)
        fobj.close()
        print("downloaded " + str(count) + ext + ' sucessfully' + ':' + url)
    except Exception as err:
        print(err)


start_url = "http://p.weather.com.cn/zrds/index.shtml"

headers = {"User-Agent": "Mozilla/5.0 (Windows; U; Windows NT 6.0 x64;"
              " en-US; rv:1.9pre) Gecko/2008072421 Minefield/3.0.2pre"}

count = 1
imgsUrl = getUrl(start_url)
for imgUrl in imgsUrl:
    count = imageSpider(imgUrl,count)
    if count>124:
        break

多线程：

import random
import time

from bs4 import BeautifulSoup
from bs4 import UnicodeDammit
import urllib.request
import threading

def getUrl(start_url):
    try:
        urls=[]
        req=urllib.request.Request(start_url,headers=headers)
        data=urllib.request.urlopen(req)
        data=data.read()
        dammit=UnicodeDammit(data,["utf-8","gbk"])
        data=dammit.unicode_markup
        soup=BeautifulSoup(data,"lxml")
        imagesUrl = soup.select('div[class="oi"] div[class="bt"] a')
        imgsUrl = []#单个图片集的链接
        for x in imagesUrl:
            imgsUrl.append(x['href'])
        return imgsUrl
    except Exception as err:
        print(err)


def imageSpider(start_url,count):
    global threads
    try:
        urls=[]
        req=urllib.request.Request(start_url,headers=headers)
        data=urllib.request.urlopen(req)
        data=data.read()
        dammit=UnicodeDammit(data,["utf-8","gbk"])
        data=dammit.unicode_markup
        soup=BeautifulSoup(data,"lxml")
        imgsUrl = soup.select('div[class="buttons"] span img')
        for x in imgsUrl:
            urls.append(x['src'])
        for url in urls:
                count += 1
                T = threading.Thread(target=download, args=(url, count))
                T.setDaemon(False)
                T.start()
                threads.append(T)
                time.sleep(random.uniform(0.03, 0.06))
                if count > 123:
                    break
        return count
    except Exception as err:
        return

def download(url,count):
    try:
    # 提取文件后缀扩展名
        if (url[len(url) - 4] == "."):
            ext = url[len(url) - 4:]
        else:
            ext = ""
        req = urllib.request.Request(url, headers=headers)
        data = urllib.request.urlopen(req, timeout=100)
        data = data.read()
        fobj = open(r"C:\Users\wcl\Desktop\data collected\test3\3.1.2\\" + str(count) + ext, "wb")
        fobj.write(data)
        fobj.close()
        print("downloaded " + str(count) + ext + ' sucessfully' + ':' + url)
    except Exception as err:
        print(err)


start_url = "http://p.weather.com.cn/zrds/index.shtml"

headers = {"User-Agent": "Mozilla/5.0 (Windows; U; Windows NT 6.0 x64;"
              " en-US; rv:1.9pre) Gecko/2008072421 Minefield/3.0.2pre"}

imgsUrl = getUrl(start_url)
count = 0
threads = []
for imgUrl in imgsUrl:
    count = imageSpider(imgUrl, count)
    for t in threads:
        t.join()
    if count > 123:
        break
print("The End")

 

结果截图：

单线程：

 

 多线程：

 

2）心得体会

多线程比单线程快好多，setDaemon(False)。

作业②

1）

要求：使用scrapy框架复现作业①。

输出信息：

同作业①

 

 spider：

import scrapy
from demo2.items import Demo2Item

class MySpider(scrapy.Spider):
    name = "MySpider"

    def start_requests(self):
        imagesUrl = 'http://p.weather.com.cn/zrds/index.shtml'
        yield scrapy.Request(url=imagesUrl, callback=self.parse)

    def parse(self, response, **kwargs):
        data = response.body.decode(response.encoding)
        selector = scrapy.Selector(text=data)
        imgsUrl = selector.xpath('//div[@class="oi"]/div[@class="bt"]/a/@href').extract()
        for x in imgsUrl:
                yield scrapy.Request(url=x, callback=self.parse1)

    def parse1(self, response):
        item = Demo2Item()
        data = response.body.decode(response.encoding)
        selector = scrapy.Selector(text=data)
        item['url'] = selector.xpath('//div[@class="buttons"]/span/img/@src').extract()
        yield item

pipeline：

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html


# useful for handling different item types with a single interface
import urllib.request

# headers = {"User-Agent": "Mozilla/5.0 (Windows; U; Windows NT 6.0 x64;"
#               " en-US; rv:1.9pre) Gecko/2008072421 Minefield/3.0.2pre"}
count = 1
class Demo2Pipeline:
    def process_item(self, item, spider):
        global count
        for url in item['url']:
            if count <= 124:
                # req = urllib.request.Request(url, headers=headers)
                # data = urllib.request.urlopen(req, timeout=100)
                # data = data.read()
                # path = "../imgs" + str(item['count']) + ".jpg"
                # with open(path, 'wb') as f:
                #     f.write(data)
                # f.close()
                path = '../imgs/' + str(count) + '.jpg'
                urllib.request.urlretrieve(url, path)
                print("downloaded " + str(count) + ".jpg" + ' sucessfully' + ':' + url)
            count+=1
        return item

 

结果截图：

 

2）心得体会

熟悉了spider，pipeline的编写。

作业③

1）

要求：爬取豆瓣电影数据使用scrapy和xpath，并将内容存储到数据库，同时将图片存储在

imgs路径下。

候选网站： https://movie.douban.com/top250

• 输出信息：

  序号	电影名称	导演	演员	简介	电影评分	电影封面
  1	肖申克的救赎	弗兰克·德拉邦特	蒂姆·罗宾斯	希望让人自由	9.7	./imgs/xsk.jpg
  2....

 

spider：

import scrapy
from doubanMovie.items import DoubanmovieItem
from bs4 import UnicodeDammit


class MoviespiderSpider(scrapy.Spider):
    name = 'MovieSpider'
    start_urls = 'https://movie.douban.com/top250/'

    def start_requests(self):
        for i in range(0, 250, 25):
            url = MoviespiderSpider.start_urls+ "?start=" + str(i) + "&filter="
            # print(url)
            # url = MoviespiderSpider.start_urls
            yield scrapy.Request(url=url, callback=self.parse)

    def parse(self, response):
        try:
            dammit = UnicodeDammit(response.body, ["utf-8", "utf-16", "gbk"])
            data = dammit.unicode_markup
            selector = scrapy.Selector(text=data)
            lis = selector.xpath('//*[@id="content"]/div/div[1]/ol/li')
            for li in lis:
                item = DoubanmovieItem()
                name = li.xpath('./div/div[2]/div[1]/a/span[1]/text()').extract_first()
                director_actor = li.xpath('./div/div[2]/div[2]/p[1]/text()[1]').extract_first().split("    ")[7]
                state = li.xpath('./div/div[2]/div[2]/p[2]/span/text()[1]').extract_first()
                rank = li.xpath('./div/div[2]/div[2]/div/span[2]/text()').extract_first()
                image_url = li.xpath('./div/div[1]/a/img/@src').extract_first()
                director_actor = director_actor.split(":")
                director = director_actor[1].split("\xa0")[0]
                if len(director_actor) > 2:
                    actor = director_actor[2]
                else:
                    actor = 'None'
                item['name'] = name.strip() if name else "" #电影名称
                item["director"] = director.strip() if director else ""  # 导演
                item["actor"] = actor.strip() if actor else ""  # 演员
                item["state"] = state.strip() if state else "" #简介
                item["rank"] = rank.strip() if rank else "" #排名
                item["image_url"] = image_url.strip() if image_url else "" #封面图片链接
                yield item

        except Exception as err:
            print(err)

pipeline：

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html


# useful for handling different item types with a single interface
import os
import urllib.request
from itemadapter import ItemAdapter
import pymysql
import time

class DoubanmoviePipeline(object):
    def open_spider(self,spider):
        print("opened")
        try:
            self.con=pymysql.connect(host="localhost", port=3306, user="root", password="123456",
                                     db="mydb", charset="utf8")
            self.cursor = self.con.cursor(pymysql.cursors.DictCursor)
            self.cursor.execute("delete from movies")
            self.opened=True
            self.count = 0
        except Exception as err:
            print(err)
            self.opened=False


    def close_spider(self,spider):
        if self.opened:
            self.con.commit()
            self.con.close()
            self.opened=False
        print("closed")
        print("总共爬取",self.count,"条数据")
    def process_item(self, item, spider):
        try:
            # print(item['name'])
            # print(item['director'])
            # print(item['actor'])
            # print(item['state'])
            # print(item['rank'])
            # print(item['image_url'])
            # print()

            if self.opened:
                self.cursor.execute("insert into movies(电影名称,导演,演员,简介,电影评分,电影封面) "
                                    "values(%s,%s,%s,%s,%s,%s)",
                                    (item['name'], item["director"], item["actor"], item["state"], item["rank"],
                                     item["image_url"]))
                path = '../MovieCovers/' +str(self.count) + '.' + item['name'] + '.jpg'
                urllib.request.urlretrieve(item['image_url'], path)
                time.sleep(0.05)
                self.count += 1
                print("爬取第" + str(self.count) + "条数据成功")
        except Exception as err:
            print(err)
        return item

 

 

结果截图：

 

 

 

 

2）心得体会

爬半天图片忘加.jpg后缀，记得加，可能就我会忘记(～￣(OO)￣)ブ。