Python + Selenium + Firefox 使用代理 auth 的用户名密码授权

Python + Firefox + 插件（closeproxy.xpi）

其中，closeproxy.xpi文件，需要Google、Bing搜下都能搜到下载地址

完整的测试代码如下：

 

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
from selenium import webdriver
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
from selenium.webdriver.common.proxy import *
from pyvirtualdisplay import Display
from base64 import b64encode
  
  
proxy = {
    "host": "123.57.78.100",
    "port": "12345",
    "user": "username",
    "pass": "password"
}
  
profile = webdriver.FirefoxProfile()
  
# add new header
profile.add_extension("modify_headers-0.7.1.1-fx.xpi")
profile.set_preference("extensions.modify_headers.currentVersion", "0.7.1.1-fx")
profile.set_preference("modifyheaders.config.active", True)
profile.set_preference("modifyheaders.headers.count", 1)
profile.set_preference("modifyheaders.headers.action0", "Add")
profile.set_preference("modifyheaders.headers.name0", "Proxy-Switch-Ip")
profile.set_preference("modifyheaders.headers.value0", "yes")
profile.set_preference("modifyheaders.headers.enabled0", True)
  
# add proxy
profile.set_preference('network.proxy.type', 1)
profile.set_preference('network.proxy.http', proxy['host'])
profile.set_preference('network.proxy.http_port', int(proxy['port']))
profile.set_preference('network.proxy.no_proxies_on', 'localhost, 127.0.0.1')
#profile.set_preference("network.proxy.username", 'aaaaa')
#profile.set_preference("network.proxy.password", 'bbbbb')
  
# Proxy auto login
profile.add_extension('closeproxy.xpi')
credentials = '{user}:{pass}'.format(**proxy)
credentials = b64encode(credentials.encode('ascii')).decode('utf-8')
profile.set_preference('extensions.closeproxyauth.authtoken', credentials)
  
profile.update_preferences()
  
driver = webdriver.Firefox(profile)
driver.get("https://proxy.mimvp.com/ip.php")
print driver.page_source
  
driver.quit()

　　

 

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
#!/usr/bin/env python
# -*- coding:utf-8 -*-
#
# Selenium + Firefox 支持 http、https
#
# 米扑代理示例：
# https://proxy.mimvp.com/demo2.php
#
# 米扑代理购买：
# https://proxy.mimvp.com
#
# mimvp.com
# 2017-01-08
  
# Python + Selenium + Firefox 设置密码时，需要使用到两个插件：
# 插件1： modify_headers-0.7.1.1-fx.xpi
# 下载地址：https://github.com/mimvp/mimvp-proxy-demo
#
# 方式2： close_proxy_authentication-1.1.xpi
# 下载地址：https://github.com/mimvp/mimvp-proxy-demo
#      
# 本示例由米扑代理原创，测试代理来自于米扑代理
# 密码授权和白名单ip设置，请见米扑代理 - 会员中心：https://proxy.mimvp.com/usercenter/userinfo.php?p=whiteip
  
  
from selenium import webdriver
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
from selenium.webdriver.common.proxy import *
from pyvirtualdisplay import Display
# from xvfbwrapper import Xvfb
  
import bs4, os
from base64 import b64encode
  
import sys
reload(sys)
sys.setdefaultencoding('utf8')
  
  
## webdriver + firefox （不使用代理，爬取网页）
def spider_url_firefox(url):
    browser = None
    display = None
    try:
        display = Display(visible=0, size=(800, 600))
        display.start()
        browser = webdriver.Firefox()       # 打开 FireFox 浏览器
        browser.get(url)   
        content = browser.page_source
        print("content: " + str(content))
    finally:
        if browser: browser.quit()
        if display: display.stop()
  
  
## webdriver + firefox + proxy + whiteip （无密码，或白名单ip授权）
## 米扑代理：https://proxy.mimvp.com
def spider_url_firefox_by_whiteip(url):
    browser = None
    display = None
      
    ## 白名单ip，请见米扑代理会员中心： https://proxy.mimvp.com/usercenter/userinfo.php?p=whiteip
    mimvp_proxy = {
                    'ip'            : '140.143.62.84',      # ip
                    'port_https'    : 19480,                # http, https
                    'port_socks'    : 19481,                # socks5
                    'username'      : 'mimvp-user',
                    'password'      : 'mimvp-pass'
                  }
      
    try:
        display = Display(visible=0, size=(800, 600))
        display.start()
          
        profile = webdriver.FirefoxProfile()
          
        # add proxy
        profile.set_preference('network.proxy.type', 1)     # ProxyType.MANUAL = 1
        if url.startswith("http://"):
            profile.set_preference('network.proxy.http', mimvp_proxy['ip'])
            profile.set_preference('network.proxy.http_port', mimvp_proxy['port_https'])    # 访问http网站
        elif url.startswith("https://"):
            profile.set_preference('network.proxy.ssl', mimvp_proxy['ip'])
            profile.set_preference('network.proxy.ssl_port', mimvp_proxy['port_https'])     # 访问https网站
        else:
            profile.set_preference('network.proxy.socks', mimvp_proxy['ip'])
            profile.set_preference('network.proxy.socks_port', mimvp_proxy['port_socks'])
            profile.set_preference('network.proxy.ftp', mimvp_proxy['ip'])
            profile.set_preference('network.proxy.ftp_port', mimvp_proxy['port_https'])
            profile.set_preference('network.proxy.no_proxies_on', 'localhost,127.0.0.1')
          
        ## 不存在此用法，不能这么设置用户名密码 （舍弃）
#         profile.set_preference("network.proxy.username", 'mimvp-guest')
#         profile.set_preference("network.proxy.password", 'welcome2mimvp')
      
        profile.update_preferences()
          
        browser = webdriver.Firefox(profile)       # 打开 FireFox 浏览器
        browser.get(url)   
        content = browser.page_source
        print("content: " + str(content))
    finally:
        if browser: browser.quit()
        if display: display.stop()
  
  
## webdriver + firefox + proxy + https （https密码授权）
## 米扑代理：https://proxy.mimvp.com
def spider_url_firefox_by_proxy(url):
    browser = None
    display = None
      
    ## 授权密码，请见米扑代理会员中心： https://proxy.mimvp.com/usercenter/userinfo.php?p=whiteip
    mimvp_proxy = {
                    'ip'            : '140.143.62.84',      # ip
                    'port_https'    : 19480,                # http, https
                    'port_socks'    : 19481,                # socks5
                    'username'      : 'mimvp-user',
                    'password'      : 'mimvp-pass'
                  }
  
    try:
        display = Display(visible=0, size=(800, 600))
        display.start()
          
        profile = webdriver.FirefoxProfile()
          
        # add new header
        profile.add_extension("modify_headers-0.7.1.1-fx.xpi")
        profile.set_preference("extensions.modify_headers.currentVersion", "0.7.1.1-fx")
        profile.set_preference("modifyheaders.config.active", True)
        profile.set_preference("modifyheaders.headers.count", 1)
        profile.set_preference("modifyheaders.headers.action0", "Add")
        profile.set_preference("modifyheaders.headers.name0", "Proxy-Switch-Ip")
        profile.set_preference("modifyheaders.headers.value0", "yes")
        profile.set_preference("modifyheaders.headers.enabled0", True)
  
        # add proxy
        profile.set_preference('network.proxy.type', 1)     # ProxyType.MANUAL = 1
        if url.startswith("http://"):
            profile.set_preference('network.proxy.http', mimvp_proxy['ip'])
            profile.set_preference('network.proxy.http_port', mimvp_proxy['port_https'])    # 访问http网站
        elif url.startswith("https://"):
            profile.set_preference('network.proxy.ssl', mimvp_proxy['ip'])
            profile.set_preference('network.proxy.ssl_port', mimvp_proxy['port_https'])     # 访问https网站
   
        # Proxy auto login （自动填写密码，进行代理授权）
        profile.add_extension('close_proxy_authentication-1.1.xpi')
        credentials = '{username}:{password}'.format(username=mimvp_proxy['username'], password=mimvp_proxy['password'])    # auth
        credentials = b64encode(credentials.encode('ascii')).decode('utf-8')
        profile.set_preference('extensions.closeproxyauth.authtoken', credentials)
  
        profile.update_preferences()
          
        browser = webdriver.Firefox(profile)       # 打开 FireFox 浏览器
        browser.get(url)   
        content = browser.page_source
        print("content: " + str(content))
    finally:
        if browser: browser.quit()
        if display: display.stop()
  
  
## webdriver + firefox + proxy + socks （socks密码授权）
## 米扑代理：https://proxy.mimvp.com
def spider_url_firefox_by_socks(url):
    browser = None
    display = None
      
    ## 授权密码，请见米扑代理会员中心： https://proxy.mimvp.com/usercenter/userinfo.php?p=whiteip
    mimvp_proxy = {
                    'ip'            : '140.143.62.84',      # ip
                    'port_https'    : 19480,                # http, https
                    'port_socks'    : 19481,                # socks5
                    'username'      : 'mimvp-user',
                    'password'      : 'mimvp-pass'
                  }
  
    proxy_config = Proxy({
                    'proxyType'     : ProxyType.MANUAL,         # 1
                    'httpProxy'     : mimvp_proxy['ip'] + ":" + str(mimvp_proxy['port_https']),
                    'sslProxy'      : mimvp_proxy['ip'] + ":" + str(mimvp_proxy['port_https']),
                    'socksProxy'    : mimvp_proxy['ip'] + ":" + str(mimvp_proxy['port_socks']),
                    'ftpProxy'      : mimvp_proxy['ip'] + ":" + str(mimvp_proxy['port_https']),
                    'noProxy'       : 'localhost,127.0.0.1',
                    'socksUsername' : mimvp_proxy['username'],
                    'socksPassword' : mimvp_proxy['password'],
                  })
      
    try:
        display = Display(visible=0, size=(800, 600))
        display.start()
          
        profile = webdriver.FirefoxProfile()
          
        # add new header
        profile.add_extension("modify_headers-0.7.1.1-fx.xpi")
        profile.set_preference("extensions.modify_headers.currentVersion", "0.7.1.1-fx")
        profile.set_preference("modifyheaders.config.active", True)
        profile.set_preference("modifyheaders.headers.count", 1)
        profile.set_preference("modifyheaders.headers.action0", "Add")
        profile.set_preference("modifyheaders.headers.name0", "Proxy-Switch-Ip")
        profile.set_preference("modifyheaders.headers.value0", "yes")
        profile.set_preference("modifyheaders.headers.enabled0", True)
          
        # auto save auth
        profile.set_preference("signon.autologin.proxy", 'true')
        profile.set_preference("network.websocket.enabled", 'false')
        profile.set_preference('network.proxy.share_proxy_settings', 'false')
        profile.set_preference('network.automatic-ntlm-auth.allow-proxies', 'false')
        profile.set_preference('network.auth.use-sspi', 'false')
        profile.update_preferences()
          
        browser = webdriver.Firefox(proxy=proxy_config, firefox_profile=profile)       # 打开 FireFox 浏览器
        browser.get(url)   
        content = browser.page_source
        print("content: " + str(content))
    finally:
        if browser: browser.quit()
        if display: display.stop()
  
  
if __name__ == '__main__':
    url = 'https://ip.cn'
    url = 'https://mimvp.com'
    url = 'https://proxy.mimvp.com/ip.php'
      
    # 不使用代理，爬取网页，成功
    spider_url_firefox(url)
      
    # 代理无密码，或设置白名单ip，成功
    spider_url_firefox_by_whiteip(url)
      
    # http, https 密码授权，成功
    spider_url_firefox_by_proxy(url)
  
    # socks5 密码授权，失败 （仍然是本机ip请求的，不是代理ip请求）
    spider_url_firefox_by_socks(url)

　　

 

posted @   CrossPython  阅读(458)  评论(0)  编辑  收藏  举报
相关博文：
·  Python3+selenium3+Firefox 设置浏览器headless模式运行+下载文件
·  selenium无头浏览器，禁用图片，禁用js，切换UA，反爬
·  代理的设置
·  selenium+python配置chrome浏览器的选项的实现
·  08-selenium
阅读排行：
· 分享一个免费、快速、无限量使用的满血 DeepSeek R1 模型，支持深度思考和联网搜索！
· 使用C#创建一个MCP客户端
· 基于 Docker 搭建 FRP 内网穿透开源项目（很简单哒）
· ollama系列1：轻松3步本地部署deepseek，普通电脑可用
· 按钮权限的设计及实现
历史上的今天：
2018-10-20 VIM编码检查
2018-10-20 mysql 远程连接
2018-10-20 CentOS7下部署Django项目详细操作步骤
2018-10-20 Django之404、500、400错误处理
点击右上角即可分享
微信分享提示
SQL AI 助手