爬虫:豆瓣电影Top250

感觉爬虫就是一种自动在网上收集信息的程序
对豆瓣Top250的爬取，就是写一个爬虫程序，让他模仿人类行为，打开网站，找到信息，然后提取数据
这段代码是使用lxml库来解析HTML，并通过XPath选择器提取数据

import requests  #用于发起网络请求。
from lxml import etree  #用于解析HTML文档,这里使用的是lxml库
import csv  #用于写入CSV文件
import time  #用于在请求之间添加延迟
import codecs  
headers = {  
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36'  
}  # 模拟浏览器访问，防止被豆瓣网站识别为爬虫而遭到封锁
def get_first_text(lst):  
    try:  
        return lst[0].strip()  
    except IndexError:  
        return ""  
  
def scrape_douban_movie(url, headers):  
    try:  
        response = requests.get(url, headers=headers)  
        response.raise_for_status()  
        return response.text  
    except requests.RequestException as e:  
        print(f"Error fetching URL: {url} - {e}")  
        return None  
  
def main():  
    urls = [f"https://movie.douban.com/top250?start={i * 25}&filter=" for i in range(10)]  
    count = 1  #用于计数
    movie_data = []  
    for url in urls:  
        html = scrape_douban_movie(url, headers)  
        if html is None:  
            continue  
  
        parsed_html = etree.HTML(html)      ## 将返回的文本加工为可以解析的html  
        movie_list = parsed_html.xpath('//*[@id="content"]/div/div[1]/ol/li')  
  
        for movie in movie_list:  
            title = get_first_text(movie.xpath('./div/div[2]/div[1]/a/span[1]/text()'))       #电影标题 
            director_actor = get_first_text(movie.xpath('./div/div[2]/div[2]/p[1]/text()'))   #电影导演 演员
            score = get_first_text(movie.xpath('./div/div[2]/div[2]/div/span[2]/text()'))     #电影评分
            introduction = get_first_text(movie.xpath('./div/div[2]/div[2]/p[2]/span/text()'))#电影引言
  
            print(count, title, director_actor, score, introduction)  
            movie_data.append([title, director_actor, score, introduction])  
  
            count += 1  
            time.sleep(1)    
    with codecs.open('movies.csv', 'w', 'utf-8-sig') as file:  
        writer = csv.writer(file)  
        writer.writerow(["Title", "Director/Actor", "Score", "Introduction"])  
        writer.writerows(movie_data)  
  
if __name__ == '__main__':  
    main()

下边是使用bs4解析

import requests  
from bs4 import BeautifulSoup  
  
headers = {  
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'  
}  
  
def get_movie_info(soup):  
    movies = []  
    movie_list = soup.find('ol', class_='grid_view').find_all('li')  #在这个标签下找到所有的li标签(包含每部电影的信息)

    for movie_li in movie_list:  
        title = movie_li.find('span', class_='title').get_text()  
        info = movie_li.find('p', class_='').get_text().strip()  
        director_actor = info.split('\xa0\xa0\xa0')[0]   
        other_info = info.split('\xa0\xa0\xa0')[1:]  
        rating_info = movie_li.find('span', class_='rating_num').get_text()  
        introduction = movie_li.find('span', class_='inq').get_text() if movie_li.find('span', class_='inq') else '' 
        #将提取出的电影信息作为一个字典添加到movies列表中 
        movies.append({  
            'title': title,  
            'director_actor': director_actor,  
            'rating': rating_info,  
            'introduction': introduction  
        })  
    return movies  

def main():
    urls = ["https://movie.douban.com/top250?start={}".format(str(i * 25)) for i in range(10)]
    
    with open('movies.txt', 'w', encoding='utf-8') as file:
        for url in urls:
            response = requests.get(url, headers=headers)
            response.raise_for_status()
            soup = BeautifulSoup(response.text, 'html.parser')
            movies = get_movie_info(soup)
            
            for movie in movies:
                file.write(f"Title: {movie['title']}\n")
                file.write(f"Director/Actor: {movie['director_actor']}\n")
                file.write(f"Rating: {movie['rating']}\n")
                file.write(f"Introduction: {movie['introduction']}\n")
                file.write('\n')

if __name__ == '__main__':
    main()

下面是使用正则表达式解析HTML
借助了两个网站:
https://curlconverter.com/
https://regex101.com/

import requests
import re
import pandas as pd

listss = []
for i in range(0, 250, 25):
    cookies = {
        'bid': 'T4QMlOS21eo',
        '_pk_id.100001.4cf6': '5e25c0c864dca561.1706688370.',
        '__yadk_uid': '8F9Q5P1dXq6TkE7QP5FEJIEhvxsTIfd4',
        '_pk_ref.100001.4cf6': '%5B%22%22%2C%22%22%2C1707383866%2C%22https%3A%2F%2Fcn.bing.com%2F%22%5D',
        '_pk_ses.100001.4cf6': '1',
        'ap_v': '0,6.0',
        '__utma': '30149280.175749301.1706688370.1707269654.1707383866.6',
        '__utmb': '30149280.0.10.1707383866',
        '__utmc': '30149280',
        '__utmz': '30149280.1707383866.6.6.utmcsr=cn.bing.com|utmccn=(referral)|utmcmd=referral|utmcct=/',
        '__utma': '223695111.96802592.1706688370.1707269654.1707383866.6',
        '__utmb': '223695111.0.10.1707383866',
        '__utmc': '223695111',
        '__utmz': '223695111.1707383866.6.6.utmcsr=cn.bing.com|utmccn=(referral)|utmcmd=referral|utmcct=/',
    }

    headers = {
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
        'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
        'Connection': 'keep-alive',
        'Referer': 'https://movie.douban.com/top250?start=25&filter=',
        'Sec-Fetch-Dest': 'document',
        'Sec-Fetch-Mode': 'navigate',
        'Sec-Fetch-Site': 'same-origin',
        'Sec-Fetch-User': '?1',
        'Upgrade-Insecure-Requests': '1',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36 Edg/121.0.0.0',
        'sec-ch-ua': '"Not A(Brand";v="99", "Microsoft Edge";v="121", "Chromium";v="121"',
        'sec-ch-ua-mobile': '?0',
        'sec-ch-ua-platform': '"Windows"',
    }

    params = {
        'start': i,
        'filter': '',
    }

    response = requests.get('https://movie.douban.com/top250', params=params, cookies=cookies, headers=headers).text
    titles = re.findall('alt="(.*?)"', response)[:25]  # 电影名称
    directors = re.findall('<p class="pl">(.*?)&nbsp;', response)  # 导演
    actors = re.findall('<p class="pl">(.*?)&nbsp;/&nbsp;', response)  # 演员
    scores = re.findall('<span class="rating_num" property="v:average">(.*?)</span>', response)  # 评分
    quotes = re.findall('<span class="inq">(.*?)</span>', response)  # 引言

    for j in range(len(titles)):
        data = {
            'title': titles[j],
            'director': directors[j].strip(),
            'actor': actors[j].strip(),
            'score': scores[j],
            'quote': quotes[j]
        }
        listss.append(data)
        print(data)
df = pd.DataFrame(listss)
print(df)