Python爬虫学习第一章
#第四章
#4.2 快速使用Urllib扒取网页
#(1)
>>> import urllib.request
#(2)
>>> file=urllib.request.urlopen("http://www.baidu.com")
#(3)
>>> data=file.read()
>>> dataline=file.readline()
#(4)
>>> print(dataline)
#(5)
>>> print(data)
#(6)
>>> fhandle=open("D:/Python35/myweb/part4/1.html","wb")
>>> fhandle.write(data)
99437
>>> fhandle.close()
#(7)
>>>filename=urllib.request.urlretrieve("http://edu.51cto.com",filename="D:/Python35/myweb/part4/2.html")
#(8)
>>> urllib.request.urlcleanup()
#(9)
>>> file.info()
<http.client.HTTPMessage object at 0x0000000003623D68>
#(10)
>>> file.getcode()
200
#(11)
>>> file.geturl()
'http://www.baidu.com'
#(12)
>>> urllib.request.quote("http://www.sina.com.cn")
'http%3A//www.sina.com.cn'
#(13)
>>> urllib.request.unquote("http%3A//www.sina.com.cn")
'http://www.sina.com.cn'
#4.3 浏览器的完全模拟--Headers属性
#(1)
>>> import urllib.request
>>> url= "http://blog.csdn.net/weiwei_pig/article/details/51178226"
>>> file=urllib.request.urlopen(url)
#(2)
mport urllib.request
url= "http://blog.csdn.net/weiwei_pig/article/details/51178226"
headers=("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 Safari/537.36 SE 2.X MetaSr 1.0")
opener = urllib.request.build_opener()
opener.addheaders = [headers]
data=opener.open(url).read()
#(3)
>>> fhandle=open("D:/Python35/myweb/part4/3.html","wb")
>>> fhandle.write(data)
47630
>>> fhandle.close()
#(4)
import urllib.request
url= "http://blog.csdn.net/weiwei_pig/article/details/51178226"
req=urllib.request.Request(url)
req.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 Safari/537.36 SE 2.X MetaSr 1.0')
data=urllib.request.urlopen(req).read()
#4.4 超时设置
#(1)
import urllib.request
for i in range(1,100):
try:
file=urllib.request.urlopen("http://yum.iqianyue.com",timeout=1)
data=file.read()
print(len(data))
except Exception as e:
print("出现异常-->"+str(e))
#(2)
import urllib.request
for i in range(1,100):
try:
file=urllib.request.urlopen("http://yum.iqianyue.com",timeout=30)
data=file.read()
print(len(data))
except Exception as e:
print("出现异常-->"+str(e))
#4.5 HTTP协议请求实战
#(1)
mport urllib.request
keywd="hello"
url="http://www.baidu.com/s?wd="+keywd
req=urllib.request.Request(url)
data=urllib.request.urlopen(req).read()
fhandle=open("D:/Python35/myweb/part4/4.html","wb")
fhandle.write(data)
fhandle.close()
#(2)
import urllib.request
url="http://www.baidu.com/s?wd="
key="韦玮老师"
key_code=urllib.request.quote(key)
url_all=url+key_code
req=urllib.request.Request(url_all)
data=urllib.request.urlopen(req).read()
fh=open("D:/Python35/myweb/part4/5.html","wb")
fh.write(data)
fh.close()
#(3)
import urllib.request
import urllib.parse
url = "http://www.iqianyue.com/mypost/"
postdata =urllib.parse.urlencode({
"name":"ceo@iqianyue.com",
"pass":"aA123456"
}).encode('utf-8') #将数据使用urlencode编码处理后，使用encode()设置为utf-8编码
req = urllib.request.Request(url,postdata)
req.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 Safari/537.36 SE 2.X MetaSr 1.0')
data=urllib.request.urlopen(req).read()
fhandle=open("D:/Python35/myweb/part4/6.html","wb")
fhandle.write(data)
fhandle.close()
#4.6 瞒天过海之代理服务器的设置
#(1)
def use_proxy(proxy_addr,url):
import urllib.request
proxy= urllib.request.ProxyHandler({'http':proxy_addr})
opener = urllib.request.build_opener(proxy, urllib.request.HTTPHandler)
urllib.request.install_opener(opener)
data = urllib.request.urlopen(url).read().decode('utf-8')
return data
proxy_addr="202.75.210.45:7777"
data=use_proxy(proxy_addr,"http://www.baidu.com")
print(len(data))
#4.7 DebugLog实战
#(1)
import urllib.request
httphd=urllib.request.HTTPHandler(debuglevel=1)
httpshd=urllib.request.HTTPSHandler(debuglevel=1)
opener=urllib.request.build_opener(httphd,httpshd)
urllib.request.install_opener(opener)
data=urllib.request.urlopen("http://edu.51cto.com")
#4.8 异常处理神器--URLError实战
#(1)
import urllib.request
import urllib.error
try:
urllib.request.urlopen("http://blog.csdn.net")
except urllib.error.URLError as e:
print(e.code)
print(e.reason)
#(2)
import urllib.request
import urllib.error
try:
urllib.request.urlopen("http://blog.csdn.net")
except urllib.error.HTTPError as e:
print(e.code)
print(e.reason)
#(3)
import urllib.request
import urllib.error
try:
urllib.request.urlopen("http://blog.baidusss.net")
except urllib.error.HTTPError as e:
print(e.reason)
#(4)
import urllib.request
import urllib.error
try:
urllib.request.urlopen("http://blog.baidusss.net")
except urllib.error.URLError as e:
print(e.reason)
#(5)
import urllib.request
import urllib.error
try:
urllib.request.urlopen("http://blog.baidusss.net")
except urllib.error.HTTPError as e:
print(e.code)
print(e.reason)
except urllib.error.URLError as e:
print(e.reason)
#(6)
import urllib.request
import urllib.error
try:
urllib.request.urlopen("http://www.baidussssss.net")
except urllib.error.URLError as e:
print(e.code)
print(e.reason)
#(7)
import urllib.request
import urllib.error
try:
urllib.request.urlopen("http://blog.csdn.net")
except urllib.error.URLError as e:
if hasattr(e,"code"):
print(e.code)
if hasattr(e,"reason"):
print(e.reason)
