爬虫相关

data = urllib.request.urlopen(url='http://www.sina.com') #
print(data.info()) # 获取网页的信息
print(data.getcode()) # 获取爬去过后是否成功的状态码
print(data.geturl()) #  获取爬取的url
print(data.read()) # 获取爬取网页的内容

data2 = urllib.request.quote('http://www.baidu.com') # 对url进行编码，去除特殊的字符，讲中文进行编码
>>>http%3A//www.baidu.com
data3 = urllib.request.unquote(data2)  # 对网页url进行解码
>>>http://www.baidu.com

# 注意以下三种添加UA的方式, headers的方式，是字典还是元组
#《一》
headers = ('User-Agent', 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Mobile Safari/537.36')

opener = urllib.request.build_opener()  #构建opener对象
opener.addheaders = [headers]  # 给opener对象添加UA
data = opener.open(url) # 带UA进行爬取url
print(data.read())

#《二》
headers = {'User-Agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Mobile Safari/537.36'}


request= urllib.request.Request(url, headers=headers)
data = urllib.request.urlopen(request)
print(data.read())

#《三》
request= urllib.request.Request(url)
request.add_header('User-Agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Mobile Safari/537.36')
data = urllib.request.urlopen(request)
print(data.read())

#get
key = '周星驰电影全集'
new_key = urllib.request.quote(key)  #对中文进行编码
url = 'http://www.baidu.com/ssid=e1156d6c/from=844b/s?word='+ new_key
request = urllib.request.Request(url)
data = urllib.request.urlopen(request )

#post 传参是字典形式，需要进行两次编码
key = {}
new_key = urllib.request.urlencode(key).encode('utf-8')
request = urllib.request.Request(url, data=new_key)

# 代理ip  （http://www.xicidaili.com/）

proxy_addr = '118.114.77.47:8080'
url = 'http://www.baidu.com'

proxy = urllib.request.ProxyHandler({'http':proxy_addr}) # 使用ProxyHandler类构建对象，注意传参方式
opener = urllib.request.build_opener(proxy,
urllib.request.HTTPHandler)  # 使用代理地址构建opener对象，这里需要添加HTTPHandler类
urllib.request.install_opener(opener) # opener设置为全局对象
# data = opener.open(url)
data = urllib.request.urlopen(url) # 有了全局对象，使用urlopen就会使用opener对象来打开
print(data.read().decode('utf8'))

# DebugLog 打印调试日志设置如下
httphd = urllib.request.HTTPHandler(debuglevel=1)  debuglevel设置为1
httpshd = urllib.request.HTTPSHandler(debuglevel=1)
opener = urllib.request.build_opener(httphd,httpshd)
urllib.request.install_opener(opener)
data = urllib.request.urlopen(url)

# 异常处理，两个类
urllib.error.URLError
urllib.error.HTTPError # URLError的子类
# 一般来说，产生URLError的原因有如下几种可能：1）连接不上服务器2）远程URL不存在3）无网络4）触发了HTTPError