Some in urllib2 - python2.7

1. urlopen可以给一个Request Object返回一个response object，read()读取相应对象的内容，这时候的print(the_page)可以输出网页的html内容

import urllib2

req = urllib2.Request('http://www.voidspace.org.uk')
response = urllib2.urlopen(req)
the_page = response.read()

print(the_page)

 

2. Request对象可以给server传输数据，还可以传输一些额外信息(metadata)，如HTTP"headers"

 

3.如我们所知request可以用POST方式给server传输数据，这些数据可以通过标准方式进行编码之后进行传输，这里用了urlencode函数进行编码

import urllib2
import urllib

url = 'http://www.someserver.com/cgi-bin/register.cgi'

values = {'name':'Michael Foord',
          'location': 'Northampton',
          'language': 'Python'
          }

data = urllib.urlencode(values)
req = urllib2.Request(url, data)
response = urllib2.urlopen(req)

the_page = response.read()

 

 

　　当然也可以用GET模式来传输数据，默认没有加data参数的时候就是使用GET模式，实际上我们知道POST是将数据编码后打包发送，GET类似与将数据加在url的末尾进行传输

import urllib2
import urllib


values = {'name':'Michael Foord',
          'location': 'Northampton',
          'language': 'Python'
          }

data = urllib.urlencode(values)
print(data) # encoded data

url = 'http://www.example.com/example.cgi'
full_url = url + '?' + data #use '?' to add data at the end
req = urllib2.Request(full_url)
response = urllib2.urlopen(req)

the_page = response.read()
print(the_page)

 

４．Headers

　　一些服务器只提供给浏览器访问，而上面的方式默认以名字python-urllib/2.7进行访问，所以需要将自己“伪装”成浏览器的名字

 

import urllib
import urllib2

url = 'http://www.someserver.com/cgi-bin/register.cgi'
user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'

values = {'name' : 'Michael Foord',
'location' : 'Northampton',
'language' : 'Python' }

headers = { 'User-Agent' : user_agent }
data = urllib.urlencode(values)

req = urllib2.Request(url, data, headers)
response = urllib2.urlopen(req)
the_page = response.read()

 

 

５.　URLError with a "reason" attribute

import urllib
import urllib2
from urllib2 import URLError

req = urllib2.Request('http://www.pretend_server.org')

try:
    urllib2.urlopen(req)
except URLError as e:
    print e.reason

 

 

6. HTTPError with a "code" attribute, codes in the 100-299 range indicatesuccess, you will usually only see error codes in the 400-599 range.

import urllib
import urllib2
from urllib2 import URLError

req = urllib2.Request('http://www.python.org/fish.html')

try:
    urllib2.urlopen(req)
except urllib2.HTTPError as e:
    print e.code
    print e.read()

 

 

7. Two basic approaches

#1
from urllib2 import Request, urlopen, URLError, HTTPError

req = Request(someurl)

try:
    response = urlopen(req)
except HTTPError as e:
    print 'The server couldn\'t fulfill the request.'
    print 'Error code: ', e.code
except URLError as e:
    print 'We failed to reach a server.'
    print 'Reason: ', e.reason
else:
    print('everything is fine')

#2
from urllib2 import Request, urlopen, URLError

req = Request(someurl)
try:
    response = urlopen(req)
except URLError as e:
    if hasattr(e, 'reason'):
        print 'We failed to reach a server.'
        print 'Reason: ', e.reason
    elif hasattr(e, 'code'):
        print 'The server couldn\'t fulfill the request.'
        print 'Error code: ', e.code
else:
    # everything is fine

 

8. Basic Authentication
　　当需要认证的时候，服务器会发出一个header来请求认证，如WWW-Authenticate: Basic realm="cPanel Users"，然后用户可以把用户名和密码作为一个header加在requese中再次请求.
一般不需要考虑格式范围的话可以直接用HTTPPasswordMgrWithDefaultRealm来设定某个URL的用户和密码

from urllib2 import Request, urlopen, URLError
import urllib2

#create a password manager
password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()

username = 'Prime'
password = 'Bee'

top_level_url = "http://example.com/foo/"
password_mgr.add_password(None, top_level_url, username, password)

handler = urllib2.HTTPBasicAuthHandler(password_mgr)

opener = urllib2.build_opener(handler)
opener.open(someurl)

# Install the opener, not necessarily
urllib2.install_opener(opener)

 

 

9. 设置socket的默认等待时间

import socket

timeout = 10
socket.setdefaulttimeout(timeout)