Python爬虫--进程和线程2

多线程

1、使用threading模块创建线程

方式一：把函数传入并创建Thread实例，然后调用start方法开始执行

# coding=utf8
"""
创建多线程的方式一：
    把函数传入并创建Thread实例，然后调用start方法开始执行
"""
import random, time
import threading

def thread_run(urls):
    print 'Current thread %s is running...' % threading.current_thread().name
    for url in urls:
        print '%s -->>> %s' % (threading.current_thread().name, url)
        time.sleep(random.random())
    print '%s ended.' % threading.current_thread().name

if __name__ == '__main__':
    print 'Current thread %s is running...' % threading.current_thread().name
    t1 = threading.Thread(target=thread_run, name='Thread_1', args=(['url_1','url_2','url_3'],))
    t2 = threading.Thread(target=thread_run, name='Thread_2', args=(['url_4','url_5','url_6'],))
    t1.start()
    t2.start()
    t1.join()
    t2.join()
    print '%s ended.' % threading.current_thread().name

方式二：继承Thread类，重写__init__和run方法

# coding=utf8
'''
创建多线程方式二：
    继承Thread类，重写__init__和run方法
'''
import threading
import time, random


class myThread(threading.Thread):
    def __init__(self, name, urls):
        threading.Thread.__init__(self, name=name)
        self.urls = urls

    def run(self):
        print 'Current thread %s is running...' % threading.current_thread().name
        for url in self.urls:
            print '%s -->>> %s' % (threading.current_thread().name, url)
            time.sleep(random.random())
        print '%s ended.' % threading.current_thread().name

if __name__ == '__main__':
    print 'Current thread %s is running...' % threading.current_thread().name
    t1 = myThread(name='Thread_1', urls=['url_1','url_2','url_3'])
    t2 = myThread(name='Thread_2', urls=['url_4','url_5','url_6'])
    t1.start()
    t2.start()
    t1.join()
    t2.join()
    print '%s ended.' % threading.current_thread().name

2、线程同步

使用Lock和RLock可以实现简单的线程同步，将每次只允许一个线程操作的数据放到acqure和release方法之间

# coding=utf8
import threading
mylock = threading.RLock()
num = 0
class myThread(threading.Thread):
    def __init__(self,name):
        threading.Thread.__init__(self, name=name)

    def run(self):
        global num
        while True:
            mylock.acquire()
            print '%s locked, Number: %d' % (threading.current_thread().name, num)
            if num >= 4:
                mylock.release()
                print '%s released, Number: %d' % (threading.current_thread().name, num)
                break
            num += 1
            print '%s released, Number: %d' % (threading.current_thread().name, num)
            mylock.release()

if __name__ == '__main__':
    thread1 = myThread('Thread_1')
    thread2 = myThread('Thread_2')
    thread1.start()
    thread2.start()

输出1：

输出2：