爬虫实例(二)：多线程，多进程对网页的爬取

采用多线程对韩寒的微博进行爬取，这个只是不需要进行模拟登陆的：

#--coding:utf-8---
#!/usr/bin/env python
import urllib
import os
import re
import time
from threading import Thread
from multiprocessing import Process

def downloadURL(urls,dirpath):
    ##在之前中分装一个list
    for url in urls:
        if(len(url)>0):
            content=urllib.urlopen(url).read()
            ##采用os模块中IO接口写写html文档
            if not os.path.exists(dirpath):
                os.makedirs(dirpath)
            open(dirpath+r'/'+url[-26:],'w').write(content)
            

def parseTarget(url):
    root_url=url
    urls=[]
    ##这里得到的东西是每一篇文章的链接
    content=urllib.urlopen(root_url).read()
    
    
    pattern=r'<a title="(.*?)" href="(.*?)">'
    
    hrefs=re.findall(pattern,content)
    
    for href in hrefs:
        #print href
        urls.append(href[1])

    return urls

def thread_or_process_job(n,thread_or_process,url_lists,job):
    local_time=time.time()
    ##args为前面函数的参数
    Thread_or_Process=[thread_or_process(target=job,args=(url_lists[i],str(n)+thread_or_process.__name__)) for i in xrange(n)]
        

    for t in Thread_or_Process:
        t.start()
        
    for t in Thread_or_Process:
        t.join()
        
    print n,thread_or_process.__name__," run job need ",time.time()-local_time

if __name__=='__main__':
    t=time.time()
    urls=[]
    for i in xrange(5):
        urls.extend(parseTarget('http://blog.sina.com.cn/s/articlelist_1191258123_0_'+str(i+1)+'.html'))
    url_len=len(urls)
    print 'urls_len:',len(urls)
    
    for n in[2,4,6,8]:
        url_list=[]
        url_split_len=url_len//n
        ##将总的url进行分为多少段同时进行处理
        for i in xrange(n):
            if i==n-1:
                url_list.append(urls[i*url_split_len:url_len])
            else:
                url_list.append(urls[i*url_split_len:(i+1)*url_split_len])
            
            thread_or_process_job(n,Thread,url_list,downloadURL)
            thread_or_process_job(n,Process,url_list,downloadURL)
    
    print "All done in ",time.time()-t