python多线程编程
Python中的多线程threading
常用方法：
1、join()方法：如果一个线程或者在函数执行的过程中调用另一个线程，并且希望待其完成操作后才能执行，那么在调用线程的时就可以使用被调线程的join方法join([timeout]) timeout：可选参数，线程运行的最长时间
2、isAlive()方法：查看线程是否还在运行
3、getName()方法：获得线程名
4、setDaemon()方法：主线程退出时，需要子线程随主线程退出，则设置子线程的setDaemon()
Python线程同步：
（1）Thread的Lock和RLock实现简单的线程同步：
import threading import time class mythread(threading.Thread): def __init__(self,threadname): threading.Thread.__init__(self,name=threadname) def run(self): global x lock.acquire() for i in range(3): x = x+1 time.sleep(1) print x lock.release() if __name__ == '__main__': lock = threading.RLock() t1 = [] for i in range(10): t = mythread(str(i)) t1.append(t) x = 0 for i in t1: i.start()
（2）使用条件变量保持线程同步：
# coding=utf-8 import threading class Producer(threading.Thread): def __init__(self,threadname): threading.Thread.__init__(self,name=threadname) def run(self): global x con.acquire() if x == 10000: con.wait() pass else: for i in range(10000): x = x+1 con.notify() print x con.release() class Consumer(threading.Thread): def __init__(self,threadname): threading.Thread.__init__(self,name=threadname) def run(self): global x con.acquire() if x == 0: con.wait() pass else: for i in range(10000): x = x-1 con.notify() print x con.release() if __name__ == '__main__': con = threading.Condition() x = 0 p = Producer('Producer') c = Consumer('Consumer') p.start() c.start() p.join() c.join() print x
（3）使用队列保持线程同步：
# coding=utf-8 import threading import Queue import time import random class Producer(threading.Thread): def __init__(self,threadname): threading.Thread.__init__(self,name=threadname) def run(self): global queue i = random.randint(1,5) queue.put(i) print self.getName(),' put %d to queue' %(i) time.sleep(1) class Consumer(threading.Thread): def __init__(self,threadname): threading.Thread.__init__(self,name=threadname) def run(self): global queue item = queue.get() print self.getName(),' get %d from queue' %(item) time.sleep(1) if __name__ == '__main__': queue = Queue.Queue() plist = [] clist = [] for i in range(3): p = Producer('Producer'+str(i)) plist.append(p) for j in range(3): c = Consumer('Consumer'+str(j)) clist.append(c) for pt in plist: pt.start() pt.join() for ct in clist: ct.start() ct.join()
生产者消费者模式的另一种实现：
# coding=utf-8 import time import threading import Queue class Consumer(threading.Thread): def __init__(self, queue): threading.Thread.__init__(self) self._queue = queue def run(self): while True: # queue.get() blocks the current thread until an item is retrieved. msg = self._queue.get() # Checks if the current message is the "quit" if isinstance(msg, str) and msg == 'quit': # if so, exists the loop break # "Processes" (or in our case, prints) the queue item print "I'm a thread, and I received %s!!" % msg # Always be friendly! print 'Bye byes!' class Producer(threading.Thread): def __init__(self, queue): threading.Thread.__init__(self) self._queue = queue def run(self): # variable to keep track of when we started start_time = time.time() # While under 5 seconds.. while time.time() - start_time < 5: # "Produce" a piece of work and stick it in the queue for the Consumer to process self._queue.put('something at %s' % time.time()) # Sleep a bit just to avoid an absurd number of messages time.sleep(1) # This the "quit" message of killing a thread. self._queue.put('quit') if __name__ == '__main__': queue = Queue.Queue() consumer = Consumer(queue) consumer.start() producer1 = Producer(queue) producer1.start()
使用线程池（Thread pool）+同步队列（Queue）的实现方式：
# A more realistic thread pool example # coding=utf-8 import time import threading import Queue import urllib2 class Consumer(threading.Thread): def __init__(self, queue): threading.Thread.__init__(self) self._queue = queue def run(self): while True: content = self._queue.get() if isinstance(content, str) and content == 'quit': break response = urllib2.urlopen(content) print 'Bye byes!' def Producer(): urls = [ 'http://www.python.org', 'http://www.yahoo.com' 'http://www.scala.org', 'http://cn.bing.com' # etc.. ] queue = Queue.Queue() worker_threads = build_worker_pool(queue, 4) start_time = time.time() # Add the urls to process for url in urls: queue.put(url) # Add the 'quit' message for worker in worker_threads: queue.put('quit') for worker in worker_threads: worker.join() print 'Done! Time taken: {}'.format(time.time() - start_time) def build_worker_pool(queue, size): workers = [] for _ in range(size): worker = Consumer(queue) worker.start() workers.append(worker) return workers if __name__ == '__main__': Producer()
Python中的多线程框架
concurrent.futures 模块： 这个模块包含 ThreadPoolExecutor 和 ProcessPoolExecutor，它们提供了一个高级接口，用于异步执行可调用对象。
from concurrent.futures import ThreadPoolExecutor, as_completed def task(n): # 任务函数 return n * n with ThreadPoolExecutor(max_workers=4) as executor: futures = [executor.submit(task, i) for i in range(10)] for future in as_completed(futures): print(future.result())
concurrent.futures 的 ThreadPoolExecutor： 一个更加高级的接口，允许提交任务并获取结果。
from concurrent.futures import ThreadPoolExecutor def task(n): return n * n with ThreadPoolExecutor(max_workers=4) as executor: future = executor.submit(task, 5) result = future.result() print(result)
multiprocessing 模块： 提供了基于进程的并发执行。虽然主要用于 CPU 密集型任务，但也可以创建线程池。
from multiprocessing.dummy import Pool as ThreadPool def task(n): # 任务函数 return n * n pool = ThreadPool(4) results = pool.map(task, range(10)) pool.close() pool.join() print(results)
gevent 库： 提供了基于协程的并发执行，适用于 I/O 密集型任务。
from gevent.pool import Pool import gevent def task(n): gevent.sleep(1) return n * n pool = Pool(4) results = pool.map(task, range(10)) print(results)
celery 框架： 一个分布式任务队列系统，适用于大规模任务分发和处理。
from celery import Celery app = Celery('tasks', broker='pyamqp://guest@localhost//') @app.task def task(n): return n * n
schedule库：是一个非常轻量级的调度库，易于使用，适合简单的调度任务。
import schedule import time def job(): print("Task is running...") # 每隔10秒运行一次 schedule.every(10).seconds.do(job) while True: schedule.run_pending() time.sleep(1)
APScheduler：是一个功能强大的调度框架，支持多种调度方式，包括固定间隔、特定时间点、cron 风格调度等。
from apscheduler.schedulers.background import BackgroundScheduler import time def my_daily_job(): print("Task executed at 22:00") scheduler = BackgroundScheduler() # 使用 cron 触发器来每天的22点执行任务 scheduler.add_job(my_daily_job, 'cron', hour=22, minute=0) scheduler.start() try: # 保持主线程运行 while True: time.sleep(1) except (KeyboardInterrupt, SystemExit): scheduler.shutdown()
