muduo网络库核心代码阅读（Acceptor，TcpConnection，TcpServer）（5）

Acceptor

Acceptor类是连接的处理器，负责监听并处理到来的新连接

class Acceptor : noncopyable
{
public:
    typedef std::function<void (int sockfd, const InetAddress&)> NewConnectionCallback;

    Acceptor(EventLoop* loop, const InetAddress& listenAddr, bool reuseport);
    ~Acceptor();

    void setNewConnectionCallback(const NewConnectionCallback& cb)
    { newConnectionCallback_ = cb; }

    void listen();

    bool listening() const { return listening_; }

private:
    void handleRead();

    EventLoop* loop_;       //Acceptor所在的事件循环
    Socket acceptSocket_;       //监听套接字，Socket类封装了套接字的部分操作
    Channel acceptChannel_;     //Acceptor对应的Channel，负责向底层loop循环注册事件
    NewConnectionCallback newConnectionCallback_;   //处理新连接的回调函数
    bool listening_;
    int idleFd_;
};


//实现文件
Acceptor::Acceptor(EventLoop* loop, const InetAddress& listenAddr, bool reuseport)
:   loop_(loop),
    acceptSocket_(sockets::createNonblockingOrDie(listenAddr.family())),       //创建非阻塞的套接字
    acceptChannel_(loop, acceptSocket_.fd()),
    listening_(false),
    idleFd_(::open("/dev/null", O_RDONLY | O_CLOEXEC))  //打开一个特殊的文件并获取其文件描述符，用于处理文件描述符达到上限的情况。"/dev/null"是一个轻量的文件设备，打开不需要耗费太多资源
{
    assert(idleFd_ >= 0);

    //启用地址重用（SO_REUSEADDR）。在默认情况下，如果套接字绑定到某一地址，即使在该套接字关闭后，操作系统也会阻止其他套接字绑定到相同地址，如果设置了地址重用，则允许立即重新使用该地址
    acceptSocket_.setReuseAddr(true);   

    //根据标志变量是否启用端口重用（同上）
    acceptSocket_.setReusePort(reuseport);

    //将套接字绑定到对应监听地址
    acceptSocket_.bindAddress(listenAddr);

    //设置回调，当有新连接时，loop会调用该函数进行处理
    acceptChannel_.setReadCallback(
        std::bind(&Acceptor::handleRead, this));
}

Acceptor::~Acceptor()
{
    acceptChannel_.disableAll();
    acceptChannel_.remove();
    ::close(idleFd_);
}

//在地址上进行监听
void Acceptor::listen()
{
    loop_->assertInLoopThread();
    listening_ = true;
    acceptSocket_.listen();
    acceptChannel_.enableReading();     //设置读事件为感兴趣事件
}

//向EventLoop注册的读事件回调，当有有新连接到来时，会调用该回调函数
void Acceptor::handleRead()
{
    loop_->assertInLoopThread();
    InetAddress peerAddr;

    //调用accept，返回客户端套接字并写入对端地址信息到peerAddr中
    int connfd = acceptSocket_.accept(&peerAddr);   
    if (connfd >= 0)
    {
        //调用回调处理新连接，后续进行分配连接到其他事件循环操作
        if (newConnectionCallback_)
        {
            newConnectionCallback_(connfd, peerAddr);
        }
        else        //没有处理新连接的回调，直接关闭客户端套接字
        {
            sockets::close(connfd);
        }
    }
    else
    {
        LOG_SYSERR << "in Acceptor::handleRead";
        // 文件描述符的分配是存在上限的，并不是无限分配的。此处是对文件描述
        // 符耗尽时的一种处理方法：由于提前分配了一个文件描述符，因此文件描
        // 述符达到上限时，可以将新客户端套接字分配到提前分配的该文件描述符上，
        // 接着关闭文件描述符，提醒客户端“连接失败，对端套接字被关闭”
        // Read the section named "The special problem of
        // accept()ing when you can't" in libev's doc.
        // By Marc Lehmann, author of libev.
        if (errno == EMFILE)
        {
            ::close(idleFd_);
            idleFd_ = ::accept(acceptSocket_.fd(), NULL, NULL);
            ::close(idleFd_);
            idleFd_ = ::open("/dev/null", O_RDONLY | O_CLOEXEC);
        }
    }
}

TcpConnection

如果说Channel类是文件描述符的保姆，管理着文件描述符的事件及事件回调的调用，那么TcpConnection可以说是一个连接的保姆，它封装了一个 TCP 连接的生命周期、数据读写、事件回调等功能。

构造函数

TcpConnection::TcpConnection(EventLoop* loop,
                            const string& nameArg,
                            int sockfd,
                            const InetAddress& localAddr,
                            const InetAddress& peerAddr)
    :   loop_(CHECK_NOTNULL(loop)),     //连接所在的事件循环
        name_(nameArg),     //连接名
        state_(kConnecting),        //连接状态：正在连接
        reading_(true),         //默认开启读事件
        socket_(new Socket(sockfd)),        //该连接对应的套接字
        channel_(new Channel(loop, sockfd)),        //该连接对应的Channel
        localAddr_(localAddr),      //服务端地址
        peerAddr_(peerAddr),        //对端地址
        highWaterMark_(64*1024*1024)        //水位标志(此处暂不讨论)
{
    //设置事件的回调函数
    channel_->setReadCallback(
        std::bind(&TcpConnection::handleRead, this, _1));
    channel_->setWriteCallback(
        std::bind(&TcpConnection::handleWrite, this));
    channel_->setCloseCallback(
        std::bind(&TcpConnection::handleClose, this));
    channel_->setErrorCallback(
        std::bind(&TcpConnection::handleError, this));
    LOG_DEBUG << "TcpConnection::ctor[" <<  name_ << "] at " << this
                << " fd=" << sockfd;

    //保持连接活跃
    //void Socket::setKeepAlive(bool on)
    //{
    //  int optval = on ? 1 : 0;
    //  ::setsockopt(sockfd_, SOL_SOCKET, SO_KEEPALIVE,&optval, static_cast<socklen_t>(sizeof optval));
    //}
    socket_->setKeepAlive(true);
}

事件回调

class TcpConnection : noncopyable,
                    public std::enable_shared_from_this<TcpConnection>
{
public:
    void setConnectionCallback(const ConnectionCallback& cb) { connectionCallback_ = cb; }

    void setMessageCallback(const MessageCallback& cb) { messageCallback_ = cb; }

    void setWriteCompleteCallback(const WriteCompleteCallback& cb) { writeCompleteCallback_ = cb; }

    void setHighWaterMarkCallback(const HighWaterMarkCallback& cb, size_t highWaterMark) { highWaterMarkCallback_ = cb; highWaterMark_ = highWaterMark; }

private:
    void handleRead(Timestamp receiveTime);     //读事件处理
    void handleWrite();     //写事件处理
    void handleClose();     //关闭事件处理
    void handleError();     //错误事件处理

    ConnectionCallback connectionCallback_;     //连接回调，在连接建立与关闭时调用
    MessageCallback messageCallback_;   //消息回调，套接字收到对端发来的数据后，经过封装后调用
    WriteCompleteCallback writeCompleteCallback_;   //写完成回调
    HighWaterMarkCallback highWaterMarkCallback_;   //水位回调，当向缓冲区写入过多数据后会调用该回调
    CloseCallback closeCallback_;       //关闭回调
    size_t highWaterMark_;
};


//实现
//将对应的套接字读取的对端发来的数据封装到缓冲区中，并调用用户注册的消息回调，交由用户处理
void TcpConnection::handleRead(Timestamp receiveTime)
{
    loop_->assertInLoopThread();
    int savedErrno = 0;
    ssize_t n = inputBuffer_.readFd(channel_->fd(), &savedErrno);
    if (n > 0)
    {
        messageCallback_(shared_from_this(), &inputBuffer_, receiveTime);
    }
    else if (n == 0)
    {
        handleClose();
    }
    else
    {
        errno = savedErrno;
        LOG_SYSERR << "TcpConnection::handleRead";
        handleError();
    }
}

//将缓冲区的数据写入到套接字发送给对端
void TcpConnection::handleWrite()
{
    loop_->assertInLoopThread();
    if (channel_->isWriting())
    {
        ssize_t n = sockets::write(channel_->fd(),
                                outputBuffer_.peek(),
                                outputBuffer_.readableBytes());
        if (n > 0)
        {
            outputBuffer_.retrieve(n);
            //如果没有剩余的待写数据
            if (outputBuffer_.readableBytes() == 0)
            {
                //关闭写事件
                channel_->disableWriting();

                //存在写完成回调则调用回调
                if (writeCompleteCallback_)
                {
                    loop_->queueInLoop(std::bind(writeCompleteCallback_, shared_from_this()));
                }

                //如果连接正在关闭则关闭写入端
                //socket_->shutdownWrite();
                if (state_ == kDisconnecting)
                {
                    shutdownInLoop();
                }
            }
        }
        else
        {
            LOG_SYSERR << "TcpConnection::handleWrite";
        }
    }
    else
    {
        LOG_TRACE << "Connection fd = " << channel_->fd()
                << " is down, no more writing";
    }
}

//文件描述符在析构函数关闭
void TcpConnection::handleClose()
{
    loop_->assertInLoopThread();
    LOG_TRACE << "fd = " << channel_->fd() << " state = " << stateToString();
    assert(state_ == kConnected || state_ == kDisconnecting);

    //设置当前连接状态。muduo使用状态标志来确保连接状态清晰可控
    setState(kDisconnected);
    channel_->disableAll();

    //延长生命周期，确保资源不会被错误释放
    //typedef std::shared_ptr<TcpConnection> TcpConnectionPtr;
    TcpConnectionPtr guardThis(shared_from_this());
    connectionCallback_(guardThis);

    closeCallback_(guardThis);
}

void TcpConnection::handleError()
{
    int err = sockets::getSocketError(channel_->fd());
    LOG_ERROR << "TcpConnection::handleError [" << name_
            << "] - SO_ERROR = " << err << " " << strerror_tl(err);
}

数据发送

每个TcpConnection都存在两个缓冲区inputBuffer_和outputBuffer_。一个为读取缓冲区，对端发到套接字的数据会保存到读缓冲区中，另一个是写入缓冲区，保存待写入套接字的数据。TcpConnection的数据发送保证只在loop线程中操作，避免了不必要的数据竞争，同时保证数据发送的顺序性等等，可以避免多线程操作存在的许多问题。

//send函数保证数据发送操作在loop线程中
void TcpConnection::send(Buffer* buf)
{
    if (state_ == kConnected)
    {
        if (loop_->isInLoopThread())
        {
            sendInLoop(buf->peek(), buf->readableBytes());
            buf->retrieveAll();
        }
        else
        {
            void (TcpConnection::*fp)(const StringPiece& message) = &TcpConnection::sendInLoop;
            loop_->runInLoop(
                std::bind(fp,
                            this,
                            buf->retrieveAllAsString()));
        }
    }
}

//实际的发送操作
void TcpConnection::sendInLoop(const void* data, size_t len)
{
    loop_->assertInLoopThread();
    ssize_t nwrote = 0;
    size_t remaining = len;
    bool faultError = false;
    if (state_ == kDisconnected)
    {
        LOG_WARN << "disconnected, give up writing";
        return;
    }
    
    //首先尝试直接将数据写入到套接字，即直接发送数据
    if (!channel_->isWriting() && outputBuffer_.readableBytes() == 0)
    {
        nwrote = sockets::write(channel_->fd(), data, len);
        if (nwrote >= 0)
        {
            remaining = len - nwrote;
            //如果数据发送完毕，调用写完成回调
            if (remaining == 0 && writeCompleteCallback_)
            {
                loop_->queueInLoop(std::bind(writeCompleteCallback_, shared_from_this()));
            }
        }
        else // nwrote < 0
        {
            nwrote = 0;
            if (errno != EWOULDBLOCK)
            {
                LOG_SYSERR << "TcpConnection::sendInLoop";
                if (errno == EPIPE || errno == ECONNRESET)
                {
                    faultError = true;
                }
            }
        }
    }

    assert(remaining <= len);
    //数据未发送完，此时将数据添加到写入缓冲区中，同时开启对应Channel的写事件，之后套接字本身的缓冲区如果可写，则会触发写事件，调用先前注册的写事件回调，即hanldeWrite函数，将TcpConnection缓冲区的数据写入到套接字中
    if (!faultError && remaining > 0)
    {
        size_t oldLen = outputBuffer_.readableBytes();
        
        //判断当前待写数据是否超过了水位线，即当前待写数据过多，提醒用户
        if (oldLen + remaining >= highWaterMark_
            && oldLen < highWaterMark_
            && highWaterMarkCallback_)
        {
            loop_->queueInLoop(std::bind(highWaterMarkCallback_, shared_from_this(), oldLen + remaining));
        }
        outputBuffer_.append(static_cast<const char*>(data)+nwrote, remaining);
        if (!channel_->isWriting())
        {
            channel_->enableWriting();
        }
    }
}

TcpServer

TcpServer是muduo构建Tcp服务器的核心组件，封装了服务器监听，连接管理等功能。

核心组件间的关系：

构造和析构

TcpServer::TcpServer(EventLoop* loop,
                    const InetAddress& listenAddr,
                    const string& nameArg,
                    Option option)
    :   loop_(CHECK_NOTNULL(loop)),
        ipPort_(listenAddr.toIpPort()),
        name_(nameArg),
        acceptor_(new Acceptor(loop, listenAddr, option == kReusePort)),        //设置监听器
        threadPool_(new EventLoopThreadPool(loop, name_)),      //构造loop线程池
        connectionCallback_(defaultConnectionCallback),
        messageCallback_(defaultMessageCallback),
        nextConnId_(1)
{
    acceptor_->setNewConnectionCallback(std::bind(&TcpServer::newConnection, this, _1, _2));    //设置处理连接的回调
}

TcpServer::~TcpServer()
{
    loop_->assertInLoopThread();
    LOG_TRACE << "TcpServer::~TcpServer [" << name_ << "] destructing";

    //销毁所有连接
    for (auto& item : connections_)
    {
        //延长连接生命周期，调用连接的销毁函数进行处理
        TcpConnectionPtr conn(item.second);
        item.second.reset();
        conn->getLoop()->runInLoop(
        std::bind(&TcpConnection::connectDestroyed, conn));
    }
}

核心函数

//启动服务器
void TcpServer::start()
{
    if (started_.getAndSet(1) == 0)
    {
        //启动线程池
        threadPool_->start(threadInitCallback_);

        assert(!acceptor_->listening());
        //开启监听（在loop线程中开启）
        loop_->runInLoop(
            std::bind(&Acceptor::listen, get_pointer(acceptor_)));
    }
}

//处理新连接的回调，封装新连接
void TcpServer::newConnection(int sockfd, const InetAddress& peerAddr)
{
    loop_->assertInLoopThread();
    EventLoop* ioLoop = threadPool_->getNextLoop();
    char buf[64];
    snprintf(buf, sizeof buf, "-%s#%d", ipPort_.c_str(), nextConnId_);
    ++nextConnId_;
    string connName = name_ + buf;

    LOG_INFO << "TcpServer::newConnection [" << name_
            << "] - new connection [" << connName
            << "] from " << peerAddr.toIpPort();
    InetAddress localAddr(sockets::getLocalAddr(sockfd));
    TcpConnectionPtr conn(new TcpConnection(ioLoop,
                                            connName,
                                            sockfd,
                                            localAddr,
                                            peerAddr));
    connections_[connName] = conn;
    conn->setConnectionCallback(connectionCallback_);
    conn->setMessageCallback(messageCallback_);
    conn->setWriteCompleteCallback(writeCompleteCallback_);
    conn->setCloseCallback(
        std::bind(&TcpServer::removeConnection, this, _1)); // FIXME: unsafe
    ioLoop->runInLoop(std::bind(&TcpConnection::connectEstablished, conn));
}