引入

  • 相关的门户网站在进行登录的时候,如果用户连续登录的次数超过3次或者5次的时候,就会在登录页中动态生成验证码。通过验证码达到分流和反爬的效果。

目录:

  • 使用超级鹰打码平台识别验证码

知识点回顾

  • session的创建方式
  • session的作用
  • proxies参数的作用
  • 高匿,透明代理的区别

 

超级鹰打码平台处理验证码的实现流程:

- 1.对携带验证码的页面数据进行抓取
- 2.可以将页面数据中验证码进行解析,验证码图片下载到本地
- 3.可以将验证码图片提交给三方平台进行识别,返回验证码图片上的数据值
    - 超级鹰打码平台:
        - 1.在官网中进行注册(普通用户和开发者用户)
        - 2.登录普通用户(查看余额),退出;
      登录开发者用户: - 1.实例代码的下载(开发文档-》各语言SDK例子下载-》Python实例下载http://www.chaojiying.com/api-14.html)
import requests
from hashlib import md5

class Chaojiying_Client(object):

    def __init__(self, username, password, soft_id):
        self.username = username
        password = password.encode('utf8')
        self.password = md5(password).hexdigest()
        self.soft_id = soft_id
        self.base_params = {
            'user': self.username,
            'pass2': self.password,
            'softid': self.soft_id,
        }
        self.headers = {
            'Connection': 'Keep-Alive',
            'User-Agent': 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)',
        }

    def PostPic(self, im, codetype):
        """
        im: 图片字节
        codetype: 题目类型 参考 http://www.chaojiying.com/price.html
        """
        params = {
            'codetype': codetype,
        }
        params.update(self.base_params)
        files = {'userfile': ('ccc.jpg', im)}
        r = requests.post('http://upload.chaojiying.net/Upload/Processing.php', data=params, files=files, headers=self.headers)
        return r.json()

    def ReportError(self, im_id):
        """
        im_id:报错题目的图片ID
        """
        params = {
            'id': im_id,
        }
        params.update(self.base_params)
        r = requests.post('http://upload.chaojiying.net/Upload/ReportError.php', data=params, headers=self.headers)
        return r.json()


chaojiying = Chaojiying_Client('超级鹰用户名', '超级鹰用户名的密码', '911742')    #用户中心>>软件ID 生成一个替换 911742
im = open(r'D:\others\下载.jpg', 'rb').read()    #本地图片文件路径 来替换 a.jpg 有时WIN系统须要//
code = chaojiying.PostPic(im, 9004)['pic_str'].replace('|', ',') #9004 验证码类型  官方网站>>价格体系
print('code:',code)
示例代码
            - 2.创建一个软件:软件ID http://www.chaojiying.com/apiuser/mysoft/-》添加新的软件
        -3.使用示例代码中的源码文件中的代码进行修改,让其识别验证码图片中的数据值

代码展示:

import requests
from hashlib import md5
#调用了打码平台的相关的接口对指定的验证码图片进行识别,返回图片上的数据值
class Chaojiying_Client(object):

    def __init__(self, username, password, soft_id):
        self.username = username
        password = password.encode('utf8')
        self.password = md5(password).hexdigest()
        self.soft_id = soft_id
        self.base_params = {
            'user': self.username,
            'pass2': self.password,
            'softid': self.soft_id,
        }
        self.headers = {
            'Connection': 'Keep-Alive',
            'User-Agent': 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)',
        }

    def PostPic(self, im, codetype):
        """
        im: 图片字节
        codetype: 题目类型 参考 http://www.chaojiying.com/price.html
        """
        params = {
            'codetype': codetype,
        }
        params.update(self.base_params)
        files = {'userfile': ('ccc.jpg', im)}
        r = requests.post('http://upload.chaojiying.net/Upload/Processing.php', data=params, files=files, headers=self.headers)
        return r.json()

    def ReportError(self, im_id):
        """
        im_id:报错题目的图片ID
        """
        params = {
            'id': im_id,
        }
        params.update(self.base_params)
        r = requests.post('http://upload.chaojiying.net/Upload/ReportError.php', data=params, headers=self.headers)
        return r.json()


import requests
from lxml import etree
import json
import time
import re
#1.对携带验证码的页面数据进行抓取
url = 'https://www.douban.com/accounts/login?source=movie'
headers = {
    'User-Agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Mobile Safari/537.36'
}
page_text = requests.get(url=url,headers=headers).text

#2.可以将页面数据中验证码进行解析,验证码图片下载到本地
tree = etree.HTML(page_text)
codeImg_url = tree.xpath('//*[@id="captcha_image"]/@src')[0]
#获取了验证码图片对应的二进制数据值
code_img = requests.get(url=codeImg_url,headers=headers).content


#获取capture_id
'<img id="captcha_image" src="https://www.douban.com/misc/captcha?id=AdC4WXGyiRuVJrP9q15mqIrt:en&size=s" alt="captcha" class="captcha_image">'
c_id = re.findall('<img id="captcha_image".*?id=(.*?)&amp.*?>',page_text,re.S)[0]
with open('./code.png','wb') as fp:
    fp.write(code_img)

#获得了验证码图片上面的数据值
chaojiying = Chaojiying_Client('超级鹰用户名', '超级鹰用户名的密码', '911742')    #用户中心>>软件ID 生成一个替换 911742
im = open('code.png', 'rb').read()                                                    #本地图片文件路径 来替换 code.png 有时WIN系统须要//
codeText = chaojiying.PostPic(im, 9101)['pic_str']                                        #9101 验证码类型  官方网站>>价格体系
#进行登录操作
post = 'https://accounts.douban.com/login'
data = {
    "source": "movie",
    "redir": "https://movie.douban.com/",
    "form_email": "15027900535",
    "form_password": "bobo@15027900535",
    "captcha-solution":codeText,
    "captcha-id":c_id,
    "login": "登录",
}
print(c_id)
login_text = requests.post(url=post,data=data,headers=headers).text
with open('./login.html','w',encoding='utf-8') as fp:
    fp.write(login_text)
超级鹰嵌入爬虫使用示例

 

 
模拟登录:爬取一些当前用户的相关数据信息
import requests
from hashlib import md5

class Chaojiying_Client(object):

    def __init__(self, username, password, soft_id):
        self.username = username
        password = password.encode('utf8')
        self.password = md5(password).hexdigest()
        self.soft_id = soft_id
        self.base_params = {
            'user': self.username,
            'pass2': self.password,
            'softid': self.soft_id,
        }
        self.headers = {
            'Connection': 'Keep-Alive',
            'User-Agent': 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)',
        }

    def PostPic(self, im, codetype):
        """
        im: 图片字节
        codetype: 题目类型 参考 http://www.chaojiying.com/price.html
        """
        params = {
            'codetype': codetype,
        }
        params.update(self.base_params)
        files = {'userfile': ('ccc.jpg', im)}
        r = requests.post('http://upload.chaojiying.net/Upload/Processing.php', data=params, files=files, headers=self.headers)
        return r.json()

    def ReportError(self, im_id):
        """
        im_id:报错题目的图片ID
        """
        params = {
            'id': im_id,
        }
        params.update(self.base_params)
        r = requests.post('http://upload.chaojiying.net/Upload/ReportError.php', data=params, headers=self.headers)
        return r.json()


#人人网的模拟登录
import requests
import urllib
from lxml import etree

#获取session对象
session = requests.Session()
#将验证码图片进行下载
headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36'
}
url = 'http://www.renren.com/'
page_text = requests.get(url=url,headers=headers).text

tree = etree.HTML(page_text)
code_img_url_list = tree.xpath('//*[@id="verifyPic_login"]/@src')
if len(code_img_url_list) == 0:
    data = {
        "email":"www.zhangbowudi@qq.com",
        "origURL":"http://www.renren.com/home",
        "domain":"renren.com",
        "key_id":"1",
        "captcha_type":"web_login",
        "password":"4f0350f09aeffeef86307747218b214b0960bdf35e30811c0d611fe39db96ec1",
        "rkey":"9e75e8dc3457b14c55a74627fa64fb43",
        "f":"http%3A%2F%2Fwww.renren.com%2F289676607",
    }
else:
    #识别验证码图片中的数据值
    chaojiying = Chaojiying_Client('超级鹰用户名', '超级鹰用户名的密码', '911742')    #用户中心>>软件ID 生成一个替换 911742
    im = open('code.jpg', 'rb').read()      #本地图片文件路径 来替换 code.jpg 有时WIN系统须要//
    code_data = chaojiying.PostPic(im, 1902)['pic_str']      #1902 验证码类型  官方网站>>价格体系
    data = {
        "email":"www.zhangbowudi@qq.com",
        "icode":code_data,
        "origURL":"http://www.renren.com/home",
        "domain":"renren.com",
        "key_id":"1",
        "captcha_type":"web_login",
        "password":"4f0350f09aeffeef86307747218b214b0960bdf35e30811c0d611fe39db96ec1",
        "rkey":"9e75e8dc3457b14c55a74627fa64fb43",
        "f":"http%3A%2F%2Fwww.renren.com%2F289676607",
    }
    code_img_url = tree.xpath('//*[@id="verifyPic_login"]/@src')[0]
    
    img_content = session.get(url=code_img_url,headers=headers).content
    with open('code.jpg', 'wb') as fp:
        fp.write(img_content)

    

#模拟登录
login_url = 'http://www.renren.com/ajaxLogin/login?1=1&uniqueTimestamp=201914927558'

#该次请求产生的cookie会被自动存储到session对象中
session.post(url=login_url,data=data,headers=headers)

url = 'http://www.renren.com/289676607/profile'
main_text = session.get(url=url,headers=headers).text

with open('renren.html','w',encoding='utf-8') as fp:
    fp.write(main_text)
人人网的模拟登录(包含验证码处理及cookie)
import requests
from hashlib import md5

class Chaojiying_Client(object):

    def __init__(self, username, password, soft_id):
        self.username = username
        password = password.encode('utf8')
        self.password = md5(password).hexdigest()
        self.soft_id = soft_id
        self.base_params = {
            'user': self.username,
            'pass2': self.password,
            'softid': self.soft_id,
        }
        self.headers = {
            'Connection': 'Keep-Alive',
            'User-Agent': 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)',
        }

    def PostPic(self, im, codetype):
        """
        im: 图片字节
        codetype: 题目类型 参考 http://www.chaojiying.com/price.html
        """
        params = {
            'codetype': codetype,
        }
        params.update(self.base_params)
        files = {'userfile': ('ccc.jpg', im)}
        r = requests.post('http://upload.chaojiying.net/Upload/Processing.php', data=params, files=files, headers=self.headers)
        return r.json()

    def ReportError(self, im_id):
        """
        im_id:报错题目的图片ID
        """
        params = {
            'id': im_id,
        }
        params.update(self.base_params)
        r = requests.post('http://upload.chaojiying.net/Upload/ReportError.php', data=params, headers=self.headers)
        return r.json()


#模拟登录古诗文网
s = requests.Session()
headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36'
}
login_url = 'https://so.gushiwen.org/user/login.aspx?from=http://so.gushiwen.org/user/collect.aspx'
page_text = requests.get(url=login_url,headers=headers).text
tree = etree.HTML(page_text)
img_src = 'https://so.gushiwen.org'+tree.xpath('//*[@id="imgCode"]/@src')[0]
img_data = s.get(url=img_src,headers=headers).content
with open('./img.jpg','wb') as fp:
    fp.write(img_data)
chaojiying = Chaojiying_Client('超级鹰用户名', '超级鹰用户名的密码', '911742')    #用户中心>>软件ID 生成一个替换 911742
im = open('img.jpg', 'rb').read()     #本地图片文件路径 来替换 a.jpg 有时WIN系统须要//
img_text = chaojiying.PostPic(im, 1902)['pic_str']     #1902 验证码类型  官方网站>>价格体系

#模拟登录
url = 'https://so.gushiwen.org/user/login.aspx?from=http%3a%2f%2fso.gushiwen.org%2fuser%2fcollect.aspx'
data = {
    "__VIEWSTATE":"9AsGvh3Je/0pfxId7DYRUi258ayuEG4rrQ1Z3abBgLoDSOeAUatOZOrAIxudqiOauXpR9Zq+dmKJ28+AGjXYHaCZJTTtGgrEemBWI1ed7oS7kpB7Rm/4yma/+9Q=",
    "__VIEWSTATEGENERATOR":"C93BE1AE",
    "from":"http://so.gushiwen.org/user/collect.aspx",
    "email":"www.zhangbowudi@qq.com",
    "pwd":"bobo328410948",
    "code":img_text,
    "denglu":"登录",
}
page_text = s.post(url=url,headers=headers,data=data).text
with open('./gushiwen.html','w',encoding='utf-8') as fp:
    fp.write(page_text)
古诗文网的模拟登录(包含验证码处理及cookie)

 

 

posted on 2020-01-17 17:06  始终不够啊  阅读(1239)  评论(0编辑  收藏  举报