python3-使用进程方式进行并发请求

1：使用HTTP长连接的方式进行发送请求:

代码如下:

import os
import requests
from requests.adapters import HTTPAdapter
from requests.exceptions import ConnectionError

class AliveRequests(object):

    # 会话池
    session_pool = {}

    def __enter__(self):
        self.session()
        return self

    def __exit__(self, *excinfo):
        pass

    @staticmethod
    def session():
        cur_pid = os.getpid()
        if cur_pid not in list(AliveRequests.session_pool.keys()):
            session = requests.session()
            session.mount('http://', HTTPAdapter(pool_maxsize=1000, pool_block=True))
            AliveRequests.session_pool[cur_pid] = session
        return AliveRequests.session_pool[cur_pid]

    @staticmethod
    def request(method=None, url=None, headers=None, files=None, data=None, params=None, auth=None, cookies=None, hooks=None, json=None, timeout=None):
        session = AliveRequests.session()
        request = requests.Request(method, url, headers, files, data, params, auth, cookies, hooks, json)
        while True:
            try:
                with session.send(session.prepare_request(request), timeout=(10, timeout)) as response:
                    return response
            except ConnectionError as err:
                err_obj = err.message if isinstance(err.message, Exception) else err
                if "BadStatusLine" in err_obj.message or "reset by peer" in err_obj.message:
                    continue
                else:
                    raise err
            except Exception as ex:
                raise ex

 

并发请求:

from multiprocessing import Pool

def work(self):
        datas = []
        po = Pool(n)  # 设置进程数
        res_list = po.map(func(函数名), 参数列表)   # 传入参数列表，传递给函数时参数列表遍历之后将每个元素依此传递，
        for res in iter(res_list):
            accessToken = res[0].json()["data"]["access_token"] if res[0].status_code == 200 and "data" in res[0].json() else ""
            dict_s = {"name": res[2][0],
                      "accessToken":accessToken,
                      "user_id":res[2][1],
                      "time": res[1]}
            datas.append(dict_s)
        return datas

 

统计发起请求的事件，单位(s)

resp = AliveRequests.request('POST', self.login_url, params=params, headers=self.headers)

resp.elapsed.total_seconds()