Python3爬取影片入库

Python3爬取影片入库

 

1、服务器说明

[root@openshift maoyan]# cat /etc/redhat-release

CentOS Linux release 7.4.1708 (Core)

[root@openshift maoyan]# python -V

Python 3.6.3 :: Anaconda, Inc.

 

2、爬取电影入库

首页页面分地址分析

 

子页面数据获取，四个字段的数据写入，

 

 

3、mysql数据库连接

import pymysql

pymysql.install_as_MySQLdb()

 

class Sql(object):

    conn = pymysql.connect(

        host="127.0.0.1",

        port=3306,

        user='root',

        passwd='123456',

        db="movies",

        charset="utf8"

)

  

4、源代码编写

[root@openshift maoyan]# cat maoyan2.py

# coding:utf-8

import requests,os,sys,django

from bs4 import BeautifulSoup

import re,urllib

import pymysql

pymysql.install_as_MySQLdb()

import datetime

 

headers = {

'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',

'Accept-Encoding':'gzip, deflate, sdch',

'Accept-Language':'zh-CN,zh;q=0.8',

'Cache-Control':'max-age=0',

'Connection':'keep-alive',

'Host':'maoyan.com',

'Upgrade-Insecure-Requests':'1',

'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36'}

class Sql(object):

    conn = pymysql.connect(

        host="127.0.0.1",

        port=3306,

        user='root',

        passwd='123456',

        db="movies",

        charset="utf8"

    )

    

    url = 'http://maoyan.com/films?showType=3'

    #url = 'https://maoyan.com/films?showType=3&offset=30'

    wbdata = requests.get(url,headers=headers)

    soup = BeautifulSoup(wbdata.content,'html5lib')

    movie_list = soup.select('div.movie-item > a')

    for movie in movie_list:

        m_url = 'http://maoyan.com' + movie.get('href')

        m_data = requests.get(m_url,headers=headers)

        m_soup = BeautifulSoup(m_data.content,'html5lib')

        name = m_soup.select_one('div.movie-brief-container > h3.name').get_text()

        movie_cate = m_soup.select("div.movie-brief-container > ul > li")[0].get_text()

        release_date = m_soup.select("div.movie-brief-container > ul > li")[2].get_text()[0:10]

        movie_img = m_soup.select_one('div.avatar-shadow > img').get('src')

 

        created = datetime.datetime.now()

        viewd = 1

 

        cur = conn.cursor()

        cur.execute("insert into userscore_movie(name,movie_cate,viewed,created,release_date,movie_img) VALUES('%s','%s','%d','%s','%s','%s')" %(name,movie_cate,viewd,created,release_date,movie_img))

        print('正在爬取电影: '+name)

        cur.close()

        conn.commit()

Sql()

 

5、执行脚本，爬取数据过程

 

 

6、数据库查看

 

自此，完成了Python3爬取影片入库过程。