Internet
0x01 URL的解析/反解析/连接
解析
urlparse()--分解URL
# -*- coding: UTF-8 -*- from urlparse import urlparse url = 'http://user:pwd@NetLoc:80/p1;param/p2?query=arg#frag' parsed = urlparse(url) print parsed print parsed.scheme print parsed.netloc print parsed.path print parsed.params print parsed.query print parsed.fragment print parsed.username print parsed.password print parsed.hostname,'(netloc in lowercase)' print parsed.port
urlsplit()--替换urlparse()，但不会分解参数。(没有params属性)
# -*- coding: UTF-8 -*- from urlparse import urlsplit url = 'http://user:pwd@NetLoc:80/p1;param/p2?query=arg#frag' parsed = urlsplit(url) print parsed print parsed.scheme print parsed.netloc print parsed.path print parsed.params print parsed.query print parsed.fragment print parsed.username print parsed.password print parsed.hostname,'(netloc in lowercase)' print parsed.port
urldefrag()--从URL中剥离出片段标识符
# -*- coding: UTF-8 -*- from urlparse import urldefrag url = 'http://NetLoc/path;param?query=arg#frag' print 'original :',url url,fragment = urldefrag(url) print 'url :',url print 'fragment :',fragment
反解析
geturl()--只适用于urlparse()或urlsplit()返回的对象
# -*- coding: UTF-8 -*- from urlparse import urlparse url = 'http://NetLoc/path;param?query=arg#frag' print 'original :',url parsed = urlparse(url) print 'after geturl() :',parsed.geturl()
urlunparse()--将包含串的普通元组拼接成一个URL（如果输入URL包含多余部分，重新构造的URL可能会将其去除）
# -*- coding: UTF-8 -*- from urlparse import urlparse,urlunparse url = 'http://NetLoc/path;param?query=arg#frag' print 'ORIGINAL URL:',url parsed = urlparse(url) print 'PARSED :',type(parsed),parsed t = parsed[:] print 'TUPLE :',type(t),t print 'NEW :',urlunparse(t)
连接
urljoin()--由相对片段构造绝对URL
# -*- coding: UTF-8 -*- from urlparse import urljoin print urljoin('http://www.example.com/path/file.html','anotherfile.html') print urljoin('http://www.example.com/path/file.html','../anotherfile.html')
# -*- coding: UTF-8 -*- from urlparse import urljoin print urljoin('http://www.example.com/path/','/subpath/file.html') print urljoin('http://www.example.com/path/','subpath/file.html')
注:如果连接到URL的路径以斜线开头(/)，这会将URL的路径重置为顶级路径。如果不是以一个斜线开头，则追加到当前URL路径的末尾。
0x02 BaseHTTPServer--实现web服务器的基类
HTTP GET
下面一个示例展示了一个请求处理器如何向客户返回一个响应
1 # -*- coding: UTF-8 -*- 2 3 from BaseHTTPServer import BaseHTTPRequestHandler 4 import urlparse 5 6 class GetHandler (BaseHTTPRequestHandler): 7 def do_GET(self): 8 parsed_path = urlparse.urlparse(self.path) 9 message_parts = [ 10 'CLIENT VALUES:', 11 'client_address=%s (%s)' % (self.client_address, 12 self.address_string()), 13 'command=%s' % self.command, 14 'path=%s' % self.path, 15 'real_path=%s' % parsed_path.path, 16 'query=%s' % self.request_version, 17 '', 18 'SERVER VALUES:', 19 'server_version=%s' % self.server_version, 20 'sys_version=%s' % self.sys_version, 21 'protocol_version=%s' % self.protocol_version, 22 '', 23 'HEADERS RECEIVED:', 24 ] 25 for name,value in sorted(self.headers.items()): 26 message_parts.append('%s=%s' % (name,value.rstrip())) 27 message_parts.append('') 28 message = '\r\n'.join(message_parts) 29 self.send_response(200) 30 self.end_headers() 31 self.wfile.write(message) 32 return 33 34 if __name__ == '__main__': 35 from BaseHTTPServer import HTTPServer 36 server = HTTPServer(('localhost',8080),GetHandler) 37 print 'Starting server,use <Ctrl+C> to stop' 38 server.serve_forever()
HTTP POST
支持POST请求需要多做一些工作，因为基类不会自动解析表单数据。cgi模块提供了FieldStorage类，如果给定了正确的输入，它知道如何解析表单。
1 # -*- coding: UTF-8 -*- 2 3 from BaseHTTPServer import BaseHTTPRequestHandler 4 import cgi 5 6 7 class PostHandler(BaseHTTPRequestHandler): 8 def do_POST(self): 9 # parse the form data posted 10 form = cgi.FieldStorage( 11 fp=self.rfile, 12 headers=self.headers, 13 environ={'REQUEST_METHOD': 'POST', 14 'CONTENT_TYPE': self.headers['Content-Type'], 15 }) 16 17 # begin the response 18 self.send_response(200) 19 self.end_headers() 20 self.wfile.write('Client:%s\n' % str(self.client_address)) 21 self.wfile.write('User-agent:%s\n' % str(self.headers['user-agent'])) 22 self.wfile.write('Path:%s\n' % self.path) 23 self.wfile.write('Form data:\n') 24 25 # Echo back information about what was posted in the form 26 for field in form.keys(): 27 field_item = form[field] 28 if field_item.filename: 29 # the field contains an uploaded file 30 file_data = field_item.file.read() 31 file_len = len(file_data) 32 del file_data 33 self.wfile.write( 34 '\tUpload %s as "%s" (%d bytes)\n' % (field, field_item.filename, file_len)) 35 else: 36 # regular form values 37 self.wfile.write('\t%s=%s\n' % (field, form[field].value)) 38 39 return 40 41 42 if __name__ == '__main__': 43 from BaseHTTPServer import HTTPServer 44 server = HTTPServer(('localhost', 8080), PostHandler) 45 print 'starting sever,use <Ctrl+Z> to stop' 46 server.serve_forever()
线程与进程
HTTPServer是SocketServer.TCPServer的一个子类，不使用多线程或者多进程来处理请求。要增加线程或进程，需要使用相应的mix-in技术从SocketServer创建一个新类。
1 # -*- coding: UTF-8 -*- 2 3 from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler 4 from SocketServer import ThreadingMixIn 5 import threading 6 7 class Handler(BaseHTTPRequestHandler): 8 def do_GET(self): 9 self.send_response(200) 10 self.end_headers() 11 message=threading.currentThread().getName() 12 self.wfile.write(message) 13 self.wfile.write('\n') 14 return 15 16 class ThreadedHTTPServer(ThreadingMixIn,HTTPServer): 17 """Handler requests in a separate thread.""" 18 19 if __name__ == '__main__': 20 server = ThreadedHTTPServer(('localhost',8080),Handler) 21 print 'Starting server, use <Ctrl+C> to stop' 22 server.serve_forever()
处理错误
1 # -*- coding: UTF-8 -*- 2 3 from BaseHTTPServer import BaseHTTPRequestHandler 4 5 class ErrorHandler(BaseHTTPRequestHandler): 6 def do_GET(self): 7 self.send_error(404) 8 return 9 10 if __name__ == '__main__' 11 from BaseHTTPServer import HTTPServer 12 server = HTTPServer(('localhost',8080),ErrorHandler) 13 print 'Starting server, use <Ctrl+C> to stop' 14 server.serve_forever()
设置首部
send_header()方法将向HTTP响应添加首部数据。
1 # -*- coding: UTF-8 -*- 2 3 from BaseHTTPServer import BaseHTTPRequestHandler 4 import urlparse 5 import time 6 7 class GetHandler(BaseHTTPRequestHandler): 8 def do_GET(self): 9 self.send_response(200) 10 self.send_header('Last-Modified', 11 self.date_time_string(time.time())) 12 self.end_headers() 13 self.wfile.write('Response body \n') 14 return 15 16 if __name__ == '__main__': 17 from BaseHTTPServer import HTTPServer 18 server = HTTPServer(('localhost',8080),GetHandler) 19 print 'Starting server, use <Ctrl+C> to stop' 20 server.serve_forever()
0x03 urllib--网络资源访问
作用：访问不需要验证的远程资源/coocie等等。
利用缓存实现简单获取
urllib提供的urlretrieve()函数提供下载数据的功能。参数:1.URL 2.存放数据的一个临时文件和一个报告下载进度的函数。另外如果UTL指示一个表单，要求提交数据，那么urlretrieve()还有有一个参数表示要传递的数据。调用程序可以直接删除这个文件，或者将这个文件作为一个缓存，使用urlcleanup()将其删除。
使用一个HTTP GET请求从一个web服务器获取数据的例子:
1 # -*- coding: UTF-8 -*- 2 3 import urllib 4 import os 5 6 def reporthook(blocks_read, block_size, total_size): 7 """total_size is reported in bytes, 8 block_size is the amount read each time. 9 blocks_read is the number of blocks successfully read. 10 """ 11 if not blocks_read: 12 print 'Connection opened' 13 return 14 if total_size < 0: 15 #unknown size 16 print 'Read %d blocks (%d bytes)' % (blocks_read,blocks_read* block_size) 17 else: 18 amount_read = blocks_read * block_size 19 print 'Read %d blocks, or %d/%d' % (blocks_read,amount_read,total_size) 20 return 21 22 try: 23 filename,msg = urllib.urlretrieve('http://blog.doughellmann.com/', reporthook=reporthook) 24 25 print 26 print 'File:',filename 27 print 'Headers:' 28 print msg 29 print 'File exists before cleanup:', os.path.exists(filename) 30 31 finally: 32 urllib.urlcleanup() 33 print 'File still exists:', os.path.exists(filename)
参数编码
对参数编码并追加到URL，从而将它们传递到服务器。(error)
1 # -*- coding: UTF-8 -*- 2 3 import urllib 4 5 query_args = {'q':'query string','foo':'bar'} 6 encoded_args = urllib.urlencode(query_args) 7 print 'Encoded:', encoded_args 8 9 url = 'http://localhost:8080/?' + encoded_args 10 print urllib.urlopen(url).read()
要使用变量的不同出现向查询串传入一个值序列，需要在调用urlencode()时将doseq设置为True。
1 # -*- coding: UTF-8 -*- 2 3 import urllib 4 5 query_args = {'foo':['foo1','foo2']} 6 print 'Single :',urllib.urlencode(query_args) 7 print 'Sequence:',urllib.urlencode(query_args,doseq=True)
结果时一个查询串，同一个名称与多个值关联。
查询参数中可能有一些特殊字符，在服务器端对URL解析时这些字符会带来问题，所以在传递到urlencode()时要对这些特殊字符"加引号"。要在本地对特殊字符加引号从而得到字符串的“安全”版本。
直接使用quote()或quote_plus()函数。
1 # -*- coding: UTF-8 -*- 2 3 import urllib 4 5 url = 'http://localhost:8080/~dhellmann/' 6 print 'urlencode() :',urllib.urlencode({'url':url}) 7 print 'quote() :',urllib.quote(url) 8 print 'quote_plus():',urllib.quote_plus(url)
加引号的逆过程
相应的使用unquote()或unquote_plus()函数。
1 # -*- coding: UTF-8 -*- 2 3 import urllib 4 5 print urllib.unquote('http%3A//localhost%3A8080/%7Edhellmann/') 6 print urllib.unquote_plus('http%3A%2F%2Flocalhost%3A8080%2F%7Edhellmann%2F')
路径与URL
有些操作系统在本地文件和URL中使用不同的值分隔路径的不同部分。为了保证代码可移植，可以使用函数pathname2url()和url2pathname()来回转换。
1 # -*- coding: UTF-8 -*- 2 3 import os 4 from urllib import pathname2url,url2pathname 5 6 print '== Default ==' 7 path = '/a/b/c' 8 print 'Original:',path 9 print 'URL:',pathname2url(path) 10 print 'Path:',url2pathname('/d/e/f') 11 12 13 print '== Windows,without drive letter ==' 14 path = r'\a\b\c' 15 print 'Original:',path 16 print 'URL:',pathname2url(path) 17 print 'Path:',url2pathname('/d/e/f') 18 print 19 20 print '== Windows, with drive letter ==' 21 path = r'C:\a\b\c' 22 print 'Original:',path 23 print 'URL:',pathname2url(path) 24 print 'Path:',url2pathname('/d/e/f') 25 print
0x04 urllib2--网络资源访问
作用：用于打开扩展URL的库，这些URL可以通过定义定制协议处理器来扩展。
urllib2模块提供了一个更新的API来使用URL标识的Internet资源。
HTTP GET
...临场error
0x05 Base64--用ASCLL编码二进制数据
base64编码
1 # -*- coding: UTF-8 -*- 2 3 import base64 4 import textwrap 5 6 #load this sourse file and strip the header. 7 with open(__file__,'rt') as input: 8 raw = input.read() 9 initial_data = raw.split('#end_pymotw_headers')[1] 10 11 encoded_data = base64.b64encode(initial_data) 12 13 num_initial = len(initial_data) 14 15 #there will never be more than 2 padding bytes. 16 padding = 3 - (num_initial %3) 17 18 print '%d bytes before encoding' % num_initial 19 print 'Expect %d padding bytes' % padding 20 print '%d bytes after encoding' % len(encoded_data) 21 print 22 print encoded_data
base64解码
1 # -*- coding: UTF-8 -*- 2 3 import base64 4 5 original_string = 'this is the data, in the clear.' 6 print 'Original:' , original_string 7 encoded_string = base64.b64encode(original_string) 8 9 print 'Encoded:',encoded_string 10 11 decoded_string = base64.b64decode(encoded_string) 12 print 'Decoded:',decoded_string
URL安全的变种
因为默认的Base64字母表可能使用+和/，这两个字符在URL中会用到，所以通常很必要使用一个候选编码来替换这些字符。+替换成-，/替换成下划线_
1 # -*- coding: UTF-8 -*- 2 3 import base64 4 5 encodes_with_pluses = chr(251) + chr(239) 6 encodes_with_slashes = chr(255) * 2 7 8 for original in [encodes_with_pluses,encodes_with_slashes]: 9 print 'Original :',repr(original) 10 print 'Standard encodingL:',base64.standard_b64encode(original) 11 print 'UTL-safe encoding :',base64.urlsafe_b64encode(original) 12 print
其他编码
1 # -*- coding: UTF-8 -*- 2 3 import base64 4 5 original_string = 'This is the data,in the clear.' 6 print 'Original:', original_string 7 8 #Base32字母表包括ASCLL集中的26个大写字母以及数字2~7 9 encoded_string = base64.b32encode(original_string) 10 print 'Base32Encoded :', encoded_string 11 decoded_string = base64.b32decode(encoded_string) 12 print 'Base32Decoded :', decoded_string 13 14 #Base16函数处理十六进制字母表 15 encoded_string = base64.b16encode(original_string) 16 print 'Base16Encoded :',encoded_string 17 encoded_string = base64.b16decode(encoded_string) 18 print 'Base16Decoded :',encoded_string
0x06 robotparser--网络蜘蛛访问控制
作用：解析用于控制网络蜘蛛的robots.txt文件
1 # -*- coding: UTF-8 -*- 2 3 import robotparser 4 import urlparse 5 6 AFENT_NAME = 'PyMOTW' 7 URL_BASE = 'http://www.doughellmann.com/' 8 parser = robotparser.RobotFileParser() 9 parser.set_url(urlparse.urljoin(URL_BASE,'robots.txt')) 10 parser.read() 11 12 PATHS = [ 13 '/', 14 '/PyMOTW/', 15 '/admin/', 16 '/downloads/PyMOTW-1.92.tar.gz', 17 ] 18 19 for path in PATHS: 20 print '%6s : %s' % (parser.can_fetch(AFENT_NAME,path),path) 21 url = urlparse.urljoin(URL_BASE,path) 22 print '%6s : %s' % (parser.can_fetch(AFENT_NAME,url),url) 23 print
can_fetch()的URL参数可以是一个相对于网站根目录的相对路径，也可以是一个完全URL。
长久蜘蛛
如果一个应用需要花很长时间来处理它下载的资源，或者受到抑制，需要在很多次下载之间暂停，这样的移动应当以其下载内容的寿命为根据，定期检查新的robots.txt文件。这个寿命并不是自动管理的，不过模块提供了一些简便方法，利用这些方法可以更容易地跟踪文件的寿命。
1 # -*- coding: UTF-8 -*- 2 3 import robotparser 4 import urlparse 5 import time 6 7 AGENT_NAME = 'PyMOTW' 8 URL_BASE = 'http://www.doughellmann.com/' 9 parser = robotparser.RobotFileParser() 10 parser.set_url(urlparse.urljoin(URL_BASE,'robots.txt')) 11 parser.read() 12 parser.modified() 13 14 PATHS = [ 15 '/', 16 '/PyMOTW/', 17 '/admin/', 18 '/downloads/PyMOTW-1.92.tar.gz', 19 ] 20 21 for path in PATHS: 22 age = int(time.time() - parser.mtime()) 23 print 'age:',age, 24 if age>1: 25 print 'rereading robots.txt' 26 parser.read() 27 parser.modified() 28 else: 29 print 30 print '%6s : %s' % (parser.can_fetch(AGENT_NAME,path),path) 31 #Simulate delay in processing 32 time.sleep(1) 33 print
如果已下载的文件寿命超过了1秒，这个极端例子就会下载一个新的robots.txt文件。作为一个更好的长久应用，在下载整个文件之前可能会请求文件的修改世界。
0x07 Cookie--HTTP Cookie
创建和设置Cookie
1 # -*- coding: UTF-8 -*- 2 3 import Cookie 4 5 c = Cookie.SimpleCookie() 6 c['name'] = 'p0pl4r' 7 print c
输出是一个合法的Set-Cookie首部，可以作为HTTP响应的一部分传递到客户。
Morsel
cookie的所有RFC属性都可以通过表示cookie值的Morsel对象来管理，如到期时间/路径/域。
1 # -*- coding: UTF-8 -*- 2 3 import Cookie 4 import datetime 5 6 def show_cookie(c): 7 print c 8 for key,morsel in c.iteritems(): 9 print 10 print 'key=',morsel.key 11 print 'value=',morsel.value 12 print 'coded_value=',morsel.coded_value 13 for name in morsel.keys(): 14 if morsel[name]: 15 print '%s = %s' % (name,morsel[name]) 16 17 c = Cookie.SimpleCookie() 18 19 #A cookie with a value that has to be encoded to fit into the headers 20 c['encoded_value_cookie'] = '"cookie_value"' 21 c['encoded_value_cookie']['comment'] = 'this is cookie\'s comment' 22 23 #A cookie that only applies to part of a site 24 c['restricted_cookie'] = 'cookie_value' 25 c['restricted_cookie']['path'] = '/sub/path' 26 c['restricted_cookie']['domain'] = 'PyMOTW' 27 c['restricted_cookie']['secure'] = 'True' 28 29 #A cookie that expires in 5 minutes 30 c['with_max_age'] = 'expires in 5 minutes' 31 c['with_max_age']['max-age'] = 300 # seconds 32 33 #A cookie that expires at a specific time 34 c['expires_at_time'] = 'cookie_value' 35 time_to_live = datetime.timedelta(hours = 1) 36 expires = datetime.datetime(2018,9,19,18,30,14)+time_to_live 37 38 #Date format:Wdy,DD-Mon-YY HH:MM:SS: GMT 39 expires_at_time = expires.strftime('%a,%d %b %Y %H:%M:%S') 40 c['expires_at_time']['expires'] = expires_at_time 41 show_cookie(c)
