Python实现爬取需要登录的网站完整示例

from selenium import webdriver
dirver = webdriver.Firefox()
dirver.get('https://music.douban.com/')
for i in dirver.find_elements_by_css_selector('.new-albums .album-title'):
    print(i.text)

 

import requests
from lxml import html
# 创建 session 对象。这个对象会保存所有的登录会话请求。
session_requests = requests.session()
# 提取在登录时所使用的 csrf 标记
login_url = "https://bitbucket.org/account/signin/?next=/"
result = session_requests.get(login_url)
tree = html.fromstring(result.text)
authenticity_token = list(set(tree.xpath("//input[@name='csrfmiddlewaretoken']/@value")))[0]
payload = {
  "username": "<你的用户名>",
  "password": "<你的密码>",
  "csrfmiddlewaretoken": authenticity_token # 在源代码中，有一个名为 “csrfmiddlewaretoken” 的隐藏输入标签。
}
# 执行登录
result = session_requests.post(
  login_url,
  data = payload,
  headers = dict(referer=login_url)
)
# 已经登录成功了，然后从 bitbucket dashboard 页面上爬取内容。
url = 'https://bitbucket.org/dashboard/overview'
result = session_requests.get(
  url,
  headers = dict(referer = url)
)
# 测试爬取的内容
tree = html.fromstring(result.content)
bucket_elems = tree.findall(".//span[@class='repo-name']/")
bucket_names = [bucket.text_content.replace("n", "").strip() for bucket in bucket_elems]
print(bucket_names)

 

from bs4 import BeautifulSoup
import requests

class CSDN(object):
    def __init__(self, headers):
        self.session = requests.Session()
        self.headers = headers
    def get_webflow(self):
        url = 'http://passport.csdn.net/account/login'
        response = self.session.get(url=url, headers=self.headers)
        soup = BeautifulSoup(response.text, 'html.parser')
        lt = soup.find('input', {'name': 'lt'})['value']
        execution = soup.find('input', {'name': 'execution'})['value']
        soup.clear()
        return (lt, execution)
    def login(self, account, password):
        self.username = account
        self.password = password
        lt, execution = self.get_webflow()
        data = {
            'username': account,
            'password': password,
            'lt': lt,
            'execution': execution,
            '_eventId': 'submit'
        }
        url = 'http://passport.csdn.net/account/login'
        response = self.session.post(url=url, headers=self.headers, data=data)
        if (response.status_code == 200):
            print('正常')
        else:
            print('异常')
    def func(self):
        headers1={
            'Host':'write.blog.csdn.net',
            'Upgrade-Insecure-Requests':'1',
            'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36'
        }
        response=self.session.get(url='http://write.blog.csdn.net/postlist',headers=headers1,allow_redirects=False)
        print(response.text)
if __name__ == '__main__':
    headers = {
        'Host': 'passport.csdn.net',
        'Origin': 'http://passport.csdn.net',
        'Referer':'http://passport.csdn.net/account/login',
        'Upgrade-Insecure-Requests':'1',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.87 Safari/537.36',
    }
    csdn = CSDN(headers=headers)
    account = ''
    password = ''
    csdn.login(account=account, password=password)
    csdn.func()

#coding=utf-8  
import requests  
import re  
import time  
import json  
from bs4 import BeautifulSoup as BS  
import sys 
  
headers = {  
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36',  
}  
  
def Get_Movie_URL():  
    urls = []  
    for i in range(1,11):  
        # 第一页的URL是不一样的，需要另外进行处理  
        if i != 1:  
            url = "http://www.mtime.com/top/movie/top100/index-%d.html" % i  
        else:  
            url = "http://www.mtime.com/top/movie/top100/"  
        r = requests.get(url=url,headers=headers)  
        soup = BS(r.text,'lxml')  
        movies = soup.find_all(name='a',attrs={'target':'_blank','href':re.compile('http://movie.mtime.com/(\d+)/'),'class':not None})  
        for m in movies:  
            urls.append(m.get('href'))  
    return urls  
  
def Create_Ajax_URL(url):  
    movie_id = url.split('/')[-2]  
    t = time.strftime("%Y%m%d%H%M%S0368", time.localtime())  
    ajax_url = "http://service.library.mtime.com/Movie.api?Ajax_CallBack=true&Ajax_CallBackType=Mtime.Library.Services&Ajax_CallBackMethod=GetMovieOverviewRating&Ajax_CrossDomain=1&Ajax_RequestUrl=%s&t=%s&Ajax_CallBackArgument0=%s" % (url,t,movie_id)  
    return ajax_url  
  
def Crawl(ajax_url):  
    r = requests.get(url=ajax_url,headers=headers)  
    if r.status_code == 200:  
        r.encoding = 'utf-8'  
        result = re.findall(r'=(.*?);',r.text)[0]  
        if result is not None:  
            value = json.loads(result)  
  
            movieTitle = value.get('value').get('movieTitle')  
            TopListName = value.get('value').get('topList').get('TopListName')  
            Ranking = value.get('value').get('topList').get('Ranking')  
            movieRating = value.get('value').get('movieRating')  
            RatingFinal = movieRating.get('RatingFinal')  
            RDirectorFinal = movieRating.get('RDirectorFinal')  
            ROtherFinal = movieRating.get('ROtherFinal')  
            RPictureFinal = movieRating.get('RPictureFinal')  
            RStoryFinal = movieRating.get('RStoryFinal')  
            print(movieTitle)  
            if value.get('value').get('boxOffice'):  
                TotalBoxOffice = value.get('value').get('boxOffice').get('TotalBoxOffice')  
                TotalBoxOfficeUnit = value.get('value').get('boxOffice').get('TotalBoxOfficeUnit')  
                print('票房：%s%s' % (TotalBoxOffice,TotalBoxOfficeUnit))  
            print('%s——No.%s' % (TopListName,Ranking))  
            print('综合评分：%s 导演评分：%s 画面评分：%s 故事评分：%s 音乐评分：%s' %(RatingFinal,RDirectorFinal,RPictureFinal,RStoryFinal,ROtherFinal))  
            print('****' * 20)  
  
def main():  
    urls = Get_Movie_URL()  
    for u in urls:  
        Crawl(Create_Ajax_URL(u))  
  
    # 问题所在，请求如下单个电影链接时时不时会爬取不到数据  
    # Crawl(Create_Ajax_URL('http://movie.mtime.com/98604/'))  
  
if __name__ == '__main__':  
    main() 

 

 相关工具

链接: https://pan.baidu.com/s/1oEw_MsaAWcMx7NQII6jXYg 密码: e6b6

 

链接: https://pan.baidu.com/s/1fSppM-hK2x9Jk9RGqvRMqg 密码: 4q43