使用python网络库下载
下载1000次网页资源
1，普通循环方式下载1000次，非常慢
#!/usr/bin/python # -*- coding: utf-8 -*- import sys import os import time import urllib import urllib2 total_times = 1000 def worker(url): try: f = urllib2.urlopen(url,timeout=10800) body = f.read() except: print sys.exc_info() return 0 return 1 if __name__ == "__main__": for i in range(total_times): url = "http://web.kuaipan.cn/static/images/pc.png" worker(url) #root:~/test # time ./c.py #real 4m6.700s #user 0m1.192s #sys 0m1.736s
2，使用进程池下载，有点慢
#!/usr/bin/python # -*- coding: utf-8 -*- import sys import os import time import urllib import urllib2 import multiprocessing total_times = 1000 def worker(url): try: f = urllib2.urlopen(url,timeout=10800) body = f.read() except: print sys.exc_info() return 0 return 1 if __name__ == "__main__": pool_size = multiprocessing.cpu_count() * 2 pool = multiprocessing.Pool(processes=pool_size) for i in range(total_times): url = "http://web.kuaipan.cn/static/images/pc.png" pool.apply_async(worker, (url,)) pool.close() pool.join() #root:~/test # time ./pc.py #real 1m43.668s #user 0m1.480s #sys 0m1.628s
3，使用twisted网络库，同样发起1000次请求，耗时减少为15s左右，性能提升很多，很快
#!/usr/bin/python from sys import argv from pprint import pformat #from twisted.internet.task import react from twisted.internet import reactor from twisted.web.client import Agent, readBody from twisted.web.http_headers import Headers total_times = 1000 times = 0 def cbRequest(response): #print 'Response version:', response.version #print 'Response code:', response.code #print 'Response phrase:', response.phrase #print 'Response headers:' #print pformat(list(response.headers.getAllRawHeaders())) d = readBody(response) d.addCallback(cbBody) return d def cbBody(body): #print 'Response body:' #print body data = body def cbShutdown(ignored): global times times = times + 1 if total_times - 1 < times: reactor.stop() def curl(url): agent = Agent(reactor) d = agent.request( 'GET', url, Headers({'User-Agent': ['Twisted Web Client Example']}), None) d.addCallback(cbRequest) d.addBoth(cbShutdown) return d if __name__ == '__main__': for i in range(total_times): curl("http://web.kuaipan.cn/static/images/pc.png") reactor.run() #root:~/test # time ./tc.py #real 0m15.480s #user 0m3.596s #sys 0m0.720s
4，使用twisted网络库长连接，耗时也是很少，很快
#!/usr/bin/python from sys import argv from pprint import pformat #from twisted.internet.task import react from twisted.internet import reactor from twisted.web.http_headers import Headers from twisted.internet import reactor from twisted.internet.defer import Deferred, DeferredList from twisted.internet.protocol import Protocol from twisted.web.client import Agent, HTTPConnectionPool total_times = 1000 times = 0 class IgnoreBody(Protocol): def __init__(self, deferred): self.deferred = deferred def dataReceived(self, bytes): pass def connectionLost(self, reason): self.deferred.callback(None) def cbRequest(response): #print 'Response code:', response.code finished = Deferred() response.deliverBody(IgnoreBody(finished)) return finished pool = HTTPConnectionPool(reactor) agent = Agent(reactor, pool=pool) def requestGet(url): d = agent.request('GET', url) d.addCallback(cbRequest) return d def cbShutdown(ignored): global times times = times + 1 if total_times - 1 < times: reactor.stop() def curl(url): agent = Agent(reactor) d = agent.request( 'GET', url, Headers({'User-Agent': ['Twisted Web Client Example']}), None) d.addCallback(cbRequest) d.addBoth(cbShutdown) return d for i in range(total_times): curl("http://web.kuaipan.cn/static/images/pc.png") reactor.run() #root:~/test # time ./tpc.py #real 0m12.817s #user 0m3.508s #sys 0m0.528s
更多twisted参考：https://twistedmatrix.com/documents/current/web/howto/client.html#auto4
golang使用循环下载方式，和python使用循环下载方式耗时差不多，4分钟时间，瓶颈应该在网络
package main import ( "fmt" "net/http" "io/ioutil" ) var totaltimes = 1000 func worker(url string) { response, err := http.Get(url) if err != nil { return } defer response.Body.Close() body, _ := ioutil.ReadAll(response.Body) fmt.Println(len(body)) } func main() { for i := 0; i < totaltimes;i ++ { worker("http://web.kuaipan.cn/static/images/pc.png") } } //root:~/test # time ./got > goresult // //real 4m45.257s //user 0m0.628s //sys 0m0.632s
golang使用协程池方式模拟下载1000次，性能也要差很多（而且容易出现网络错误,最近出的go version go1.2rc4 linux/amd64要好一点 ,go1.1问题很多）
package main import ( "fmt" "net/http" "io/ioutil" "sync" ) var totaltimes = 1000 var poolsize = 250 func worker(linkChan chan string, wg *sync.WaitGroup) { // Decreasing internal counter for wait-group as soon as goroutine finishes defer wg.Done() for url := range linkChan { // Analyze value and do the job here response, err := http.Get(url) if err != nil { return } defer response.Body.Close() body, _ := ioutil.ReadAll(response.Body) fmt.Println(len(body)) //fmt.Println("Resp code", response.StatusCode) } } func main() { var i int lCh := make(chan string) wg := new(sync.WaitGroup) // Adding routines to workgroup and running then for i := 0; i < poolsize; i++ { wg.Add(1) go worker(lCh, wg) } for i = 0; i < totaltimes;i ++ { lCh <- "http://web.kuaipan.cn/static/images/pc.png" } close(lCh) // Waiting for all goroutines to finish (otherwise they die as main routine dies) wg.Wait() } //root:~/test # time ./gotest > goresult // //real 0m25.250s //user 0m0.772s //sys 0m0.380s
twisted支持定时器，我们可以用来动态添加任务
from twisted.web.client import getPage from twisted.internet import reactor class Getter(object): def __init__(self): self._sequence = 0 self._results = [] self._errors = [] def add(self, url): d = getPage(url) d.addCallbacks(self._on_success, self._on_error) d.addCallback(self._on_finish) self._sequence += 1 def _on_finish(self, *narg): self._sequence -= 1 print len(self._results), len(self._errors) # if not self._sequence: # reactor.stop() _on_success = lambda self, *res: self._results.append(res) _on_error = lambda self, *err: self._errors.append(err) def run(self): reactor.run() return self._results, self._errors def jobtimer(): for url in ('http://www.google.com', 'http://www.yahoo.com', 'http://www.baidu.com'): g.add(url) reactor.callLater(1,jobtimer) reactor.callLater(2,jobtimer) #定时添加任务 g = Getter() results, errors = g.run() #print len(results) #print len(errors)
