爬取猫眼电影榜单TOP100榜-以csv文件保存

csv文件作用

将爬取的数据存放到本地的csv文件中

使用流程

# 1、导入模块
# 2、打开csv文件
# 3、初始化写入对象
# 4、写入数据(参数为列表)

import csv 

with open('film.csv','w') as f:
    writer = csv.writer(f)
    writer.writerow([])

 

示例代码

创建 test.csv 文件，在文件中写入数据

# 单行写入（writerow([]))
import csv
with open('test.csv','w',newline='') as f:
    writer = csv.writer(f)
    writer.writerow(['步惊云','36'])
    writer.writerow(['超哥哥','25'])

# 多行写入(writerows([(),(),()]
import csv
with open('test.csv','w',newline='') as f:
    writer = csv.writer(f)
    writer.writerows([('聂风','36'),('秦霜','25'),('孔慈','30')])

 

爬虫代码

from urllib import request
import re
import time
import random
from useragents import ua_list
import csv

class MaoyanSpider(object):
  def __init__(self):
    self.url = 'https://maoyan.com/board/4?offset={}'
    # 计数
    self.num = 0

  def get_html(self,url):
    headers = {
      'User-Agent' : random.choice(ua_list)
    }
    req = request.Request(url=url,headers=headers)
    res = request.urlopen(req)
    html = res.read().decode('utf-8')
    # 直接调用解析函数
    self.parse_html(html)

  def parse_html(self,html):
    re_bds = r'<div class="movie-item-info">.*?title="(.*?)".*?class="star">(.*?)</p>.*?releasetime">(.*?)</p>'
    pattern = re.compile(re_bds,re.S)
    # film_list: [('霸王别姬','张国荣','1993'),()]
    film_list = pattern.findall(html)
    # 直接调用写入函数
    self.write_html(film_list)

  # 存入csv文件 - writerow()
  # def write_html(self,film_list):
  #   with open('film.csv','a') as f:
  #     # 初始化写入对象,注意参数f别忘了
  #     writer = csv.writer(f)
  #     for film in film_list:
  #       L = [
  #         film[0].strip(),
  #         film[1].strip(),
  #         film[2].strip()[5:15]
  #       ]
  #       # writerow()参数为列表
  #       writer.writerow(L)

  # 存入csv文件 - writerows()
  def write_html(self,film_list):
    L = []
    with open('film.csv','a') as f:
      # 初始化写入对象,注意参数f别忘了
      writer = csv.writer(f)
      for film in film_list:
        t = (
          film[0].strip(),
          film[1].strip(),
          film[2].strip()[5:15]
        )
        L.append(t)
        self.num += 1
      # writerows()参数为列表
      writer.writerows(L)

  def main(self):
    for offset in range(0,31,10):
      url = self.url.format(offset)
      self.get_html(url)
      time.sleep(random.randint(1,2))
    print('共抓取数据:',self.num)

if __name__ == '__main__':
  start = time.time()
  spider = MaoyanSpider()
  spider.main()
  end = time.time()
  print('执行时间:%.2f' % (end-start))