爬虫中获取cookie的方式

 

为什么要获取cookie？

因为有的页面爬取的时候，需要登录后才能爬，比如知乎，如何判断一个页面是否已经登录，通过判断是否含有cookies就可以，我们获取到cookie后就可以携带cookie来访问需要登录后的页面了。

方式一使用session

这里的session并不是django中的session，而是requests中的session

import requests

url = 'https://www.processon.com/login'
login_email = '283867@qq.com'
login_password = 'ZZZ0'
# 创建一个session,作用会自动保存cookie
session = requests.session()
data = {
    'login_email': login_email,
    'login_password': login_password
}
# 使用session发起post请求来获取登录后的cookie,cookie已经存在session中
response = session.post(url = url,data=data)

# 用session给个人主页发送请求，因为session中已经有cookie了
index_url = 'https://www.processon.com/diagrams'
index_page = session.get(url=index_url).text
print(index_page)

 

把cookie保存在本地，并判断用户是否已经登录

import requests
from http import cookiejar

# 创建一个session,作用会自动保存cookie
session = requests.session()

# 指定cookie保存的路径
session.cookies = cookiejar.LWPCookieJar(filename="cookies.txt")
try:
    session.cookies.load(ignore_discard=True)  # 加载cookie文件，ignore_discard = True,即使cookie被抛弃，也要保存下来
except:
    print('cookie未能加载')

def login_save_cookie():
    """
    登录并保存cookie到本地
    :return: 
    """
    url = 'https://www.processon.com/login'
    login_email = '*****@qq.com'
    login_password = '****1391'
    data = {
        'login_email': login_email,
        'login_password': login_password
    }
    # 使用session发起post请求来获取登录后的cookie,cookie已经存在session中
    response = session.post(url=url, data=data)
    # 把cookie保存到文件中
    session.cookies.save()


def read_cookie():
    """
    读取cookie进入登录后的页面
    :return: 
    """
    index_url = 'https://www.processon.com/diagrams'
    index_page = session.get(url=index_url).text
    print(index_page)

def login_y_n():
    """
    判断用户是否已经登录，我们这里使用的方法是：随便找一个登陆后页面的url，如果我们访问它时不发生重定向，我们就可以
    判断该用户应经登录了
    :return:
    """
    url = 'https://www.processon.com/diagrams/new#template'
    response = session.get(url = url,allow_redirects=False) # allow_redirects =False不允许重定向到登录页面
    if response != 200:
        return False
    else:
        return True

read_cookie()

 

方法二 使用selenium获取cookies

from selenium import webdriver
import json
browser = webdriver.Chrome(executable_path=r"E:\爬虫视频\day04\chromedriver_win32_2.46\chromedriver.exe")

def get_cookies():
    """
    通过selenium获取cookie保存在文件中
    :return:
    """
    url = 'https://www.processon.com/login'
    browser.get(url=url)
    browser.find_element_by_id('login_email').send_keys('286867@qq.com')
    browser.find_element_by_id('login_password').send_keys('ZZZ0391')
    browser.find_element_by_id('signin_btn').click()
    # 获取cookie，这里得到的是一个列表
    cookies_list = browser.get_cookies()
    browser.close()
    with open("cookies.txt", "w") as fp:
        json.dump(cookies_list, fp)  # 这里切记，如果我们要使用json.load读取数据，那么一定要使用json.dump来写入数据，
        # 不能使用str(cookies)直接转为字符串进行保存，因为其存储格式不同。这样我们就将cookies保存在文件中了。


def read_cookie():
    """
    读取cookie,添加到browser中
    :return:
    """
    url = 'https://www.processon.com/diagrams'
    browser.get(url=url) # 这里必须先访问一次否则页面不能打开
    with open('./cookies.txt','r') as fp:
        cookies_list = json.load(fp)
        for cookies in cookies_list:
            browser.add_cookie(cookies)
    browser.get(url)

read_cookie()

注意用selenium来获取的cookie是一个列表，列表中有很多字典，字典中有domain、expiry、name、value、path等key，但是在我们真正的浏览器中就只有一个字典，字典中只有name 键对应的值和value对应的值，所以在使用的时候

还需要转换一下：

[{"domain": ".processon.com", "expiry": 1560351255.689168, "httpOnly": false, "name": "_sid", "path": "/", "secure": false, "value": "796afe66ce2a6002828ab3ca281f96fb"},
{"domain": ".processon.com", "httpOnly": true, "name": "JSESSIONID", "path": "/", "secure": false, "value": "EBDACE1BDAB1464A2CCBBFFB7048A238.jvm1"},
{"domain": ".processon.com", "expiry": 1586703257, "httpOnly": false, "name": "zg_did", "path": "/", "secure": false, "value": "%7B%22did%22%3A%20%2216a173113351bb-062c441b2e33b7-7a1437-144000-16a173113362e%22%7D"},
{"domain": ".processon.com", "expiry": 1560351255.689117, "httpOnly": false, "name": "processon_userKey", "path": "/", "secure": false, "value": "59f7fba9e4b0edf0e25cd413"},
 {"domain": ".processon.com", "expiry": 1555167313, "httpOnly": false, "name": "_gat", "path": "/", "secure": false, "value": "1"},
 {"domain": ".processon.com", "expiry": 1555253657, "httpOnly": false, "name": "_gid", "path": "/", "secure": false, "value": "GA1.2.1345294219.1555167253"},
{"domain": ".processon.com", "expiry": 1618239257, "httpOnly": false, "name": "_ga", "path": "/", "secure": false, "value": "GA1.2.555498451.1555167253"},
 {"domain": ".processon.com", "expiry": 1586703257, "httpOnly": false, "name": "zg_3f37ba50e54f4374b9af5be6d12b208f", "path": "/", "secure": false, 
 "value": "%7B%22sid%22%3A%201555167253312%2C%22updated%22%3A%201555167257424%2C%22info%22%3A%201555167253326%2C%22superProperty%22%3A%20%22%7B%7D%22%2C%22platform
 %22%3A%20%22%7B%7D%22%2C%22utm%22%3A%20%22%7B%7D%22%2C%22referrerDomain%22%3A%20%22%22%2C%22cuid%22%3A%20%2259f7fba9e4b0edf0e25cd413%22%7D"}]

 

Cookie: zg_did=%7B%22did%22%3A%20%2216a16fc24ab1e7-08f589794c6e4d-7a1437-144000-16a16fc24ac76a%22%7D; _ga=GA1.2.1095343087.1555163784; _gid=GA1.2.545489346.1555163784; processon_userKey=59f7fba9e4b0edf0e25cd413; _sid=796afe66ce2a6002828ab3ca281f96fb; zg_3f37ba50e54f4374b9af5be6d12b208f=%7B%22sid%22%3A%201555163784372%2C%22updated%22%3A%201555165807015%2C%22info%22%3A%201555163784376%2C%22superProperty%22%3A%20%22%7B%7D%22%2C%22platform%22%3A%20%22%7B%7D%22%2C%22utm%22%3A%20%22%7B%7D%22%2C%22referrerDomain%22%3A%20%22%22%2C%22cuid%22%3A%20%2259f7fba9e4b0edf0e25cd413%22%7D; JSESSIONID=685AABAF6B8D70AF25E501C7E9E67A32.jvm1