Python多线程和多进程

文章目录

• • 1.基本概念
  • 2.多线程
  • • 0.常用的线程方法
    • 1.用函数创建线程
    • 2.用类创建多线程
    • 3.锁
    • • 1.互斥锁
      • 2.可重入锁
      • 3.防止死锁的加锁机制
    • 4.共享数据
    • 5.线程通信机制:
    • 6.线程池
  • 3.多进程
  • • 0.常用的线程方法
    • 1.用函数创建多进程
    • 2.用类创建多进程
    • 3.锁
    • • 1.互斥锁
      • 2.可重入锁
      • 3.防止死锁的加锁机制
    • 4.共享数据:队列
    • 4.信号量和队列
    • 5.管道
    • 6.进程池
  • 3.多线程和多进程之间的比较

1.基本概念

定义	含义
并发:	concurrency,同一时刻只能有一条指令执行,但是多个线程的对应的指令被快速轮换地执行
并行	parallel,同一时刻,有多条指令在多个处理器上同时执行,并行必须要依赖于多个处理器
阻塞	程序未得到所需计算资源时被挂起的状态
非阻塞	程序在等待某操作过程中,自身不被阻塞,可以继续处理其他的事情
同步	不同程序单元为了完成某个任务,在执行过程中需靠某种通信方式以协调一致,我们称这些程序单元是同步执行的
异步	为完成某个任务,不同程序单元之间过程中无需通信协调,也能完成任务的方式,不相关的程序单元之间可以是异步的

2.多线程

0.常用的线程方法

# 如上所述，创建一个线程
t=Thread(target=func)

# 启动子线程
t.start()

# 阻塞子线程，主线程等待子线程运行完毕之后才退出
t.join()

# 判断线程是否在执行状态，在执行返回True，否则返回False
t.is_alive()
t.isAlive()

# 设置守护线程,随主线程退出而退出，默认为False
thread.setDaemon(True)

# 设置线程名
t.name = "My-Thread"

1.用函数创建线程

import threading
import time


def target(second):
    print(f'{threading.current_thread().name} is running')
    print(f'{threading.current_thread().name} sleep {second}s')
    time.sleep(second)
    print(f'{threading.current_thread().name} is ended')


print(f'{threading.current_thread().name} is running')
for i in [1, 5]:
    thread = threading.Thread(target=target, args=[i])
    thread.start()
print(f'{threading.current_thread().name} is ended')

MainThread is running
Thread-1 is running
Thread-1 sleep 1s
Thread-2 is running
Thread-2 sleep 5s
MainThread is ended
Thread-1 is ended
Thread-2 is ended

2.用类创建多线程

import threading
import time


class MyThread(threading.Thread):
    def __init__(self, second):
        threading.Thread.__init__(self)
        self.second = second

    def run(self):
        print(f'{threading.current_thread().name} is running')
        print(f'{threading.current_thread().name} sleep {self.second}s')
        time.sleep(self.second)
        print(f'{threading.current_thread().name} is ended')


print(f'{threading.current_thread().name} is running')
threads = []
for i in [1, 5]:
    thread = MyThread(i)
    threads.append(thread)
    thread.start()
for thread in threads:
    thread.join()
print(f'{threading.current_thread().name} is ended')

MainThread is running
Thread-1 is running
Thread-1 sleep 1s
Thread-2 is running
Thread-2 sleep 5s
Thread-1 is ended
Thread-2 is ended
MainThread is ended

3.锁

锁的分类:互斥锁,可重入锁

锁的应用场景:确保同一时间只有一个线程操作数据

1.互斥锁

import threading
import time

count = 0


class MyThread(threading.Thread):
    def __init__(self):
        threading.Thread.__init__(self)

    def run(self):
        global count

        lock.acquire()
        temp = count + 1
        time.sleep(0.001)
        count = temp
        lock.release()


lock = threading.Lock()
threads = []
for _ in range(100):
    thread = MyThread()
    thread.start()
    threads.append(thread)
for thread in threads:
    thread.join()
print(f'Final count: {count}')

2.可重入锁

import threading
import time

count = 0


class MyThread(threading.Thread):
    def __init__(self):
        super().__init__()

    def run(self):
        n = 0
        with lock:  
            for i in range(10):
                n += 1
                with lock:
                    print(n)


lock = threading.RLock()
thread1 = MyThread()
thread1.start()

3.防止死锁的加锁机制

import threading
from contextlib import contextmanager

# Thread-local state to stored information on locks already acquired
_local = threading.local()


@contextmanager
def acquire(*locks):
    # Sort locks by object identifier
    locks = sorted(locks, key=lambda x: id(x))

    # Make sure lock order of previously acquired locks is not violated
    acquired = getattr(_local, 'acquired', [])
    if acquired and max(id(lock) for lock in acquired) >= id(locks[0]):
        raise RuntimeError('Lock Order Violation')

    # Acquire all of the locks
    acquired.extend(locks)
    _local.acquired = acquired

    try:
        for lock in locks:
            lock.acquire()
        yield
    finally:
        # Release locks in reverse order of acquisition
        for lock in reversed(locks):
            lock.release()
        del acquired[-len(locks):]


x_lock = threading.Lock()
y_lock = threading.Lock()


def thread_1():
    while True:
        with acquire(x_lock):
            with acquire(y_lock):
                print('Thread-1')


def thread_2():
    while True:
        with acquire(y_lock):
            with acquire(x_lock):
                print('Thread-2')


if __name__ == '__main__':
    t1 = threading.Thread(target=thread_1)
    t1.daemon = True
    t1.start()

    t2 = threading.Thread(target=thread_2)
    t2.daemon = True
    t2.start()

4.共享数据

线程之间通过全局变量共享数据

5.线程通信机制:

​```python
# coding=utf-8
# /usr/bin/env python


from queue import Queue
from threading import Thread
import time


class Student:
    def __init__(self, name):
        self.name = name

    def speak(self):
        print("{}：到！".format(self.name))


class CallManager(Thread):
    def __init__(self, queue):
        super().__init__()
        self.students = {}
        self.queue = queue

    def put(self, student):
        self.students.setdefault(student.name, student)

    def run(self):
        while True:
            # 阻塞程序，时刻监听老师，接收消息
            student_name = queue.get()
            if student_name == "exit":
                break
            elif student_name in self.students:
                self.students[student_name].speak()
            else:
                print("老师，咱班，没有 {} 这个人".format(student_name))


class Teacher:
    def __init__(self, queue):
        super().__init__()
        self.queue = queue

    def call(self, student_name):
        if student_name == "exit":
            print("点名结束，开始上课..")
        else:
            print("老师：{}来了没？".format(student_name))
            # 发送消息，要点谁的名
        self.queue.put(student_name)


s1 = Student(name="小明")
s2 = Student(name="小亮")

queue = Queue()
cm = CallManager(queue)
cm.put(s1)
cm.put(s2)
cm.start()

teacher = Teacher(queue=queue)
print('开始点名~')
teacher.call('小明')
time.sleep(1)
teacher.call('小亮')
time.sleep(1)
teacher.call("exit")

​```

6.线程池

import time
import threading
from concurrent.futures import ThreadPoolExecutor


def target(n):
    for i in range(n):
        print('running thread-{}:{}:{}'.format(threading.get_ident(), i))
        time.sleep(1)
    return n


# 创建一个最大容纳数量为5的线程池
pool = ThreadPoolExecutor(5)
for i in range(10):
    # 往线程池上塞任务
    result=pool.submit(target(i))

3.多进程

0.常用的线程方法

# 启动子进程
t.start()

# 进程等待，主线程等待子线程运行完毕之后才退出
t.join()

# 判断进程是否在执行状态，在执行返回True，否则返回False
t.is_alive()

# 守护进程,随进程退出而退出，默认为False
t.daemon = True

# 设置主进程名
t.name = "My_Process"

#终止子进程
p.terminate()
p.join()

1.用函数创建多进程

import time
import multiprocessing


def process(index):
    time.sleep(index)

    print(f'Process: {index}')


if __name__ == '__main__':
    print(f'CPU number: {multiprocessing.cpu_count()}')
    for i in range(5):
        p = multiprocessing.Process(target=process, args=[i])
        p.start()

    for p in multiprocessing.active_children():
        print(f'Child process name: {p.name} id: {p.pid}')

2.用类创建多进程

import time
from multiprocessing import Process


class MyProcess(Process):

    def __init__(self, loop):
        super().__init__()
        self.loop = loop

    def run(self):
        for count in range(self.loop):
            time.sleep(1)
            print(f'Pid: {self.pid} LoopCount: {count}')


if __name__ == '__main__':

    for i in range(2, 5):
        p = MyProcess(i)
        p.start()

3.锁

锁的分类:互斥锁,可重入锁

锁的应用场景:确保同一时间只有一个线程操作数据

1.互斥锁

2.可重入锁

3.防止死锁的加锁机制

4.共享数据:队列

进程之间通过队列共享数据

4.信号量和队列

背景:允许多个进程来访问共享资源,同时还需要限制能访问共享资源的进程的数量

semaphore是一个内置的计数器,每当调用acquire()时，内置计数器-1 每当调用release()时，内置计数器+1

from multiprocessing import Process, Semaphore, Lock, Queue
import time
import secrets


class Base(Process):
    def __init__(self, buffer1, empty1, full1, lock1):
        super(Base, self).__init__()
        self.buffer = buffer1
        self.empty = empty1
        self.full = full1
        self.lock = lock1


class Consumer(Base):
    def run(self):
        while True:
            self.full.acquire()
            self.lock.acquire()
            num = self.buffer.get()
            print(f'Consumer pop an {num}')
            time.sleep(1)
            self.lock.release()
            self.empty.release()


class Producer(Base):
    def run(self):
        while True:
            # 缓冲区余数减1
            self.empty.acquire()
            self.lock.acquire()
            my_random = secrets.SystemRandom()
            num = my_random.randint(1, 10)
            self.buffer.put(num)
            print(f'Producer append an {num}')
            time.sleep(1)
            self.lock.release()
            # 缓存区占用数加1
            self.full.release()


if __name__ == '__main__':
    buffer = Queue(10)
    # 缓冲区余数
    empty = Semaphore(2)
    # 缓冲区占用数
    full = Semaphore(0)
    lock = Lock()
    p = Producer(buffer, empty, full, lock)
    c = Consumer(buffer, empty, full, lock)
    p.daemon = c.daemon = True
    p.start()
    c.start()
    p.join()
    c.join()
    print('Main Process Ended')

5.管道

进程之间的通信用管道Pipe

管道的分类:单向管道(half-duplex),双向管道(duplex),默认是双向管道,如果要创建单向管道，可以在初始化的时候传入 deplex 参数为 False

from multiprocessing import Process, Pipe


class Consumer(Process):
    def __init__(self, pipe):
        Process.__init__(self)
        self.pipe = pipe

    def run(self):
        self.pipe.send('Consumer Words')
        print(f'Consumer Received: {self.pipe.recv()}')


class Producer(Process):
    def __init__(self, pipe):
        Process.__init__(self)
        self.pipe = pipe

    def run(self):
        print(f'Producer Received: {self.pipe.recv()}')
        self.pipe.send('Producer Words')


if __name__ == '__main__':
    pipe = Pipe()
    p = Producer(pipe[0])
    c = Consumer(pipe[1])
    p.daemon = c.daemon = True
    p.start()
    c.start()
    p.join()
    c.join()
    print('Main Process Ended')

6.进程池

from multiprocessing import Pool
import time


def function(index):
    print(f'Start process: {index}')
    time.sleep(3)
    print(f'End process {index}', )


if __name__ == '__main__':
    pool = Pool(processes=3)
    # 方式一:apply_async
    # [pool.apply_async(function, args=(i,)) for i in range(4)]
    # 方式二:map
    pool.map(function, range(4))
    print('Main Process started')
    pool.close()
    pool.join()
    print('Main Process ended')

3.多线程和多进程之间的比较

名称	含义
多线程	操作系统进行运算调度的最小单位	IO密集型(磁盘IO，网络IO，数据库IO等,譬如爬虫，网站开发等)
多进程	系统进行资源分配和调度的一个独立单位	CPU密集型(大数据分析,机器学习等)