python tornado TCPserver异步协程实例

项目所用知识点

  • tornado
  • socket
  • tcpserver
  • 协程
  • 异步

tornado tcpserver源码抛析

在tornado的tcpserver文件中，实现了TCPServer这个类，他是一个单线程的，非阻塞的tcp 服务。

为了与上层协议(在tornado中就是HTTPServer)交互，TCPServer提供了一个接口：handle_stream, 要求其子类必需实现该方法，该方法就是主要用来处理应用层逻辑的。

我们可以通过下面代码倒入模块查看源码

from tornado.tcpserver import TCPServer

源码中给了好多解释，先把源码注释贴进来

自己仔细看该类的其他方法

通过方法名就能看出来，而且开头已经给出实例怎么去用，所以这个就不一一解释了，我自己的用法如下

复制代码
from tornado.tcpserver import TCPServer
from tornado.iostream import IOStream, StreamClosedError
from tornado import gen
from tornado.ioloop import IOLoop
import struct

class ProxyServer(TCPServer):
    def __init__(self, *args, **kwargs):
        super(ProxyServer, self).__init__(*args, **kwargs)
        self.devices = dict()

    @gen.coroutine
    def handle_stream(self, stream, address):
          pass


if __name__ == "__main__":
    server = ProxyServer()
    server.listen(1234)
    IOLoop.current().start()
复制代码

具体步骤来分析 一下

TCPServer执行过程

1.server = ProxyServer()创建tcpserver对象，该步骤仅仅做了一个初始化操作

复制代码
def __init__(self, io_loop=None, ssl_options=None, max_buffer_size=None, read_chunk_size=None):
        self.io_loop = io_loop
        self.ssl_options = ssl_options
        self._sockets = {}  # fd -> socket object    用来存储文件描述符与socket对象的映射关系
        self._pending_sockets = []
        self._started = False
        self.max_buffer_size = max_buffer_size    # 最大缓冲长度
        self.read_chunk_size = read_chunk_size    # 每次读的chunk大小

        # 校验ssl选项. 
        if self.ssl_options is not None and isinstance(self.ssl_options, dict):
            if 'certfile' not in self.ssl_options:
                raise KeyError('missing key "certfile" in ssl_options')

            if not os.path.exists(self.ssl_options['certfile']):
                raise ValueError('certfile "%s" does not exist' % self.ssl_options['certfile'])
            if ('keyfile' in self.ssl_options and not os.path.exists(self.ssl_options['keyfile'])):
                raise ValueError('keyfile "%s" does not exist' % self.ssl_options['keyfile'])
复制代码

2.想都不要想肯定是开启socket

步骤是执行server.listen(1234)的时候，

复制代码
    def listen(self, port, address=""):
        """Starts accepting connections on the given port.

        This method may be called more than once to listen on multiple ports.
        `listen` takes effect immediately; it is not necessary to call
        `TCPServer.start` afterwards.  It is, however, necessary to start
        the `.IOLoop`.
        """
        sockets = bind_sockets(port, address=address)
        self.add_sockets(sockets)
复制代码

3.看下listen里面有调用bind_sockets()方法，来看下该方法

复制代码
def bind_sockets(port, address=None, family=socket.AF_UNSPEC, backlog=_DEFAULT_BACKLOG, flags=None, reuse_port=False):
    if reuse_port and not hasattr(socket, "SO_REUSEPORT"):
        raise ValueError("the platform doesn't support SO_REUSEPORT")

    sockets = []
    if address == "":
        address = None
    # address family参数指定调用者期待返回的套接口地址结构的类型。它的值包括四种：AF_UNIX，AF_INET，AF_INET6和AF_UNSPEC。
    # AF_UNIX用于同一台机器上的进程间通信
    # 如果指定AF_INET，那么函数就不能返回任何IPV6相关的地址信息；如果仅指定了AF_INET6，则就不能返回任何IPV4地址信息。
    # AF_UNSPEC则意味着函数返回的是适用于指定主机名和服务名且适合任何协议族的地址。
    # 如果某个主机既有AAAA记录(IPV6)地址，同时又有A记录(IPV4)地址，那么AAAA记录将作为sockaddr_in6结构返回，而A记录则作为sockaddr_in结构返回
    if not socket.has_ipv6 and family == socket.AF_UNSPEC: # 如果系统不支持ipv6
        family = socket.AF_INET
    if flags is None:
        flags = socket.AI_PASSIVE
    bound_port = None
    for res in set(socket.getaddrinfo(address, port, family, socket.SOCK_STREAM, 0, flags)):
        af, socktype, proto, canonname, sockaddr = res
        if (sys.platform == 'darwin' and address == 'localhost' and af == socket.AF_INET6 and sockaddr[3] != 0):
            # Mac OS X在“localhost”的getaddrinfo结果中包含一个链接本地地址fe80 :: 1％lo0。 
            # 但是，防火墙不了解这是一个本地地址，并且会提示访问。 所以跳过这些地址。
            continue
        try:
            sock = socket.socket(af, socktype, proto)
        except socket.error as e:
            # 如果协议不支持该地址
            if errno_from_exception(e) == errno.EAFNOSUPPORT:
                continue
            raise
        # 为 fd 设置 FD_CLOEXEC 标识
        set_close_exec(sock.fileno())
        if os.name != 'nt': # 非windows
            sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        if reuse_port:
            sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
        if af == socket.AF_INET6:
            # On linux, ipv6 sockets accept ipv4 too by default,
            # but this makes it impossible to bind to both
            # 0.0.0.0 in ipv4 and :: in ipv6.  On other systems,
            # separate sockets *must* be used to listen for both ipv4
            # and ipv6.  For consistency, always disable ipv4 on our
            # ipv6 sockets and use a separate ipv4 socket when needed.
            #
            # Python 2.x on windows doesn't have IPPROTO_IPV6.
            if hasattr(socket, "IPPROTO_IPV6"):
                sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1)

        # 自动端口分配，端口=None
        # 应该绑定在IPv4和IPv6地址上的同一个端口上
        host, requested_port = sockaddr[:2]
        if requested_port == 0 and bound_port is not None:
            sockaddr = tuple([host, bound_port] + list(sockaddr[2:]))
        # 设置socket为非阻塞
        sock.setblocking(0)    
        sock.bind(sockaddr)
        bound_port = sock.getsockname()[1]
        sock.listen(backlog)
        sockets.append(sock)
    return sockets
复制代码

4.接下来执行的是add_sockets()方法

def add_sockets(self, sockets):
        if self.io_loop is None:
            self.io_loop = IOLoop.current()    # 获取IOLoop实例对象

        for sock in sockets:
            self._sockets[sock.fileno()] = sock
            add_accept_handler(sock, self._handle_connection, io_loop=self.io_loop)

可以看到里面调用了add_accept_handler方法，来我们进去看看该方法干啥了

5.探析add_accept_handler方法

复制代码
def add_accept_handler(sock, callback, io_loop=None):
    if io_loop is None: # 获取IOLoop实例对象
        io_loop = IOLoop.current()

    def accept_handler(fd, events):
        # 我们处理回调时可能会有许多的连接等待建立; 为了防止其他任务的饥饿，我们必须限制我们一次接受的连接数。 
        # 理想情况下，我们接受在处理回调过程中等待的连接数，但此可能会对负载产生不利影响。 
        # 相反，我们使用listen backlog作为我们可以合理接受的连接数的。
        for i in xrange(_DEFAULT_BACKLOG): # _DEFAULT_BACKLOG默认为128
            try:
                connection, address = sock.accept()
            except socket.error as e:
                # _ERRNO_WOULDBLOCK 与EAGAIN相同，表示再尝试一下，很多情况下是因为资源不足，或者条件未达成
                # 当某个子进程与客户端建立了连接，其他子进程再次尝试与该客户端建立连接时就会产生该错误
                if errno_from_exception(e) in _ERRNO_WOULDBLOCK:
                    return
                # ECONNABORTED表示有一个连接，在他处于等待被服务端accept的时候主动关闭了。
                if errno_from_exception(e) == errno.ECONNABORTED:
                    continue
                raise
            callback(connection, address)
    io_loop.add_handler(sock, accept_handler, IOLoop.READ) # 为socket注册handler：当发生READ事件时运行accept_handler函数。
复制代码

 欣欣然我们来到了最后一步

6.IOLoop.current().start()，然我们看下源码

复制代码
def start(self):
        try:
            while True:    
                callbacks = self._callbacks
                self._callbacks = []
                due_timeouts = []
                # 将时间已到的定时任务放置到due_timeouts中，过程省略
                for callback in callbacks:          # 执行callback
                    self._run_callback(callback)
                for timeout in due_timeouts:        # 执行定时任务
                    if timeout.callback is not None:
                        self._run_callback(timeout.callback)       
                callbacks = callback = due_timeouts = timeout = None    # 释放内存
                # 根据情况设置poll_timeout的值，过程省略
                if not self._running:    # 终止ioloop运行时，在执行完了callback后结束循环
                    breaktry:
                    event_pairs = self._impl.poll(poll_timeout)
                except Exception as e:
                    if errno_from_exception(e) == errno.EINTR:  # 系统调用被信号处理函数中断，进行下一次循环
                        continue
                    else:
                        raise 
                self._events.update(event_pairs)
                while self._events: 
                    fd, events = self._events.popitem()             # 获取一个fd以及对应事件
                    try:
                        fd_obj, handler_func = self._handlers[fd]   # 获取该fd对应的事件处理函数
                        handler_func(fd_obj, events)                # 运行该事件处理函数
                    except (OSError, IOError) as e:         
                        if errno_from_exception(e) == errno.EPIPE:     # 当客户端关闭连接时会产生EPIPE错误                         
                            pass
                        # 其他异常处理已经省略
                fd_obj = handler_func = None       # 释放内存空间
复制代码

这一步想了解更多去参考这篇文章http://www.cnblogs.com/MnCu8261/p/6730691.html

代码实例

目前公司有这么一个需求，iphonex--server--ue4，面对两个客户端，达到iphonex把数据给ue4，server起一个代理作用，需求大概就是这样，具体实现代码如下

复制代码
from tornado.tcpserver import TCPServer
from tornado.iostream import IOStream, StreamClosedError
from tornado import gen
from tornado.ioloop import IOLoop
import struct

class ProxyServer(TCPServer):
    def __init__(self, *args, **kwargs):
        super(ProxyServer, self).__init__(*args, **kwargs)
        self.devices = dict()

    @gen.coroutine
    def handle_stream(self, stream, address):
        device = yield stream.read_bytes(1)
        if device == b"\x0a":
            self.handle_iphonex_stream(stream, address)
        elif device == b"\x0b":
            self.handle_ue4_stream(stream, address)
        else:
            print("protocol error.")

    @gen.coroutine
    def handle_iphonex_stream(self, stream, address):
        yield stream.write(b"\x00")
        print("iphonex")

        # uuid
        rlen = yield stream.read_bytes(4)
        rlen = struct.unpack(">I", rlen)[0]
        uuid = yield stream.read_bytes(rlen)
        uuid = uuid.decode()
        yield stream.write(b"\x00")
        print(uuid)

        # keys
        rlen = yield stream.read_bytes(4)
        rlen = struct.unpack(">I", rlen)[0]
        keys = yield stream.read_bytes(rlen)
        keys = keys.decode()
        yield stream.write(b"\x00")
        print(keys)

        # save
        self.devices[uuid] = {'keys': keys}

        # data
        keys = keys.split(',')
        fmt = "%df" % len(keys)

        while True:
            try:
                data = yield stream.read_bytes(struct.calcsize(fmt))
            except StreamClosedError:
                print 'iphonex is closed'
                break
            pdata = struct.unpack(fmt, data)
            print(pdata)

            ue4stream = self.devices[uuid].get('ue4')
            if ue4stream:
                try:
                    yield ue4stream.write(data)
                except Exception as e:
                    self.devices[uuid]['ue4'] = None
                    print('request for %s closed' % uuid)

    @gen.coroutine
    def handle_ue4_stream(self, stream, address):
        yield stream.write(b"\x00")
        print("ue4")

        # uuid
        rlen = yield stream.read_bytes(4)
        rlen = struct.unpack(">I", rlen)[0]
        uuid = yield stream.read_bytes(rlen)
        uuid = uuid.decode()
        print(uuid)

        if self.devices.get(uuid):
            yield stream.write(b"\x00")
        else:
            yield stream.write(b"\x01")
            raise Exception

        # send keys
        keys = self.devices[uuid].get('keys')
        stream.write(struct.pack(">I", len(keys)))
        stream.write(keys.encode())

        valid = yield stream.read_bytes(1)
        if valid == b'x\01':
            print('keys not support.')
            raise Exception

        self.devices[uuid]['ue4'] = stream


if __name__ == "__main__":
    server = ProxyServer()
    server.listen(1234)
    IOLoop.current().start()
复制代码

请点赞转发帮助身边更多的人

posted @   张岩林  阅读(1934)  评论(0)  编辑  收藏  举报
编辑推荐：
· Linux系列：如何用 C#调用 C方法造成内存泄露
· AI与.NET技术实操系列（二）：开始使用ML.NET
· 记一次.NET内存居高不下排查解决与启示
· 探究高空视频全景AR技术的实现原理
· 理解Rust引用及其生命周期标识（上）
阅读排行：
· 阿里最新开源QwQ-32B，效果媲美deepseek-r1满血版，部署成本又又又降低了！
· 单线程的Redis速度为什么快？
· SQL Server 2025 AI相关能力初探
· 展开说说关于C#中ORM框架的用法！
· AI编程工具终极对决：字节Trae VS Cursor，谁才是开发者新宠？
点击右上角即可分享
微信分享提示