python爬虫学习（3）：使用User-Agent和代理ip

使用User-Agent
方法一，先建立head，作为参数传进去

import urllib.request
import json

content=input("请输入需要翻译的内容:")
url='http://fanyi.youdao.com/translate?smartresult=dict&smartresult=rule'

data={}

data['i']=content
data['from']='AUTO'
data['to']='AUTO'
data['smartresult']='dict'
data['client']='fanyideskweb'
data['salt']='1520575049536'
data['sign']='4514c46c320493ba8c034eaa8d9decaf'
data['doctype']='json'
data['version']='2.1'
data['keyfrom']='fanyi.web'
data['action']='FY_BY_CLICKBUTTION'
data['typoResult']='false'
data['ue']='utf-8'
data=urllib.parse.urlencode(data).encode('utf-8')

head={}
head['User-Agent']='Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3355.4 Safari/537.36'
#urllib.request.Request的可以有三个参数，head（必须是字典)可以作为第三个参数
req=urllib.request.Request(url,data,head)
response=urllib.request.urlopen(req)
html=response.read().decode('utf-8')
target=json.loads(html)
print("翻译结果：%s"%target['translateResult'][0][0]['tgt'])

################################################################################

方法二使用建立request后add_hander

import urllib.request
import json

content=input("请输入需要翻译的内容:")
url='http://fanyi.youdao.com/translate?smartresult=dict&smartresult=rule'

data={}
data['i']=content
data['from']='AUTO'
data['to']='AUTO'
data['smartresult']='dict'
data['client']='fanyideskweb'
data['salt']='1520575049536'
data['sign']='4514c46c320493ba8c034eaa8d9decaf'
data['doctype']='json'
data['version']='2.1'
data['keyfrom']='fanyi.web'
data['action']='FY_BY_CLICKBUTTION'
data['typoResult']='false'
data['ue']='utf-8'
data=urllib.parse.urlencode(data).encode('utf-8')

req=urllib.request.Request(url,data)
req.add_header('User-Agent','Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3355.4 Safari/537.36')
response=urllib.request.urlopen(req)
html=response.read().decode('utf-8')
target=json.loads(html)
print("翻译结果：%s"%target['translateResult'][0][0]['tgt'])

###########################################################################################

#使用代理ip

 

import urllib

def main():
#要访问的网址
url='http://www.whatismyip.com.tw/'
#代理ip
proxy={'http':'106.46.136.112:808'}
#创建ProxyHandler
proxy_support=urllib.request.ProxyHandler(proxy)
#创建opener
opener=urllib.request.build_opener(proxy_support)
#添加User-Agent
opener.addheaders=[('User-Agent','Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36')]
#安装opener
urllib.request.install_opener(opener)
#使用自己安装好的opener
response=urllib.request.urlopen(url)
#读取相应信息并解码
html=response.read().decode('utf-8')
print(html)
if __name__=='__name__':
main()