一种长连接服务模型实现
模型图如下（event loop + thread pool）(event用epoll，线程同步用队列)
说明：一个中转线程负责连接外部服务器维持tcp连接，接收外部来的请求，转发请求给服务线程处理，服务线程处理完后通知中转线程回复，主要业务代码如下：
1，ProxyThread.cpp
#include "ProxyThread.h" #include "Singletone.h" #include "SwitchSvr.h" TC_Epoller *ProxyThread::_epoller; map<int, ProxyConnect *> *ProxyThread::_mapSocket; struct WupProtocol { static int checkRequest(const char *buf, size_t len) { if(len < 4){ return 0; } Tint ti; strncpy(ti.byte, buf, 4); int bodyLen = ti.integer; if(bodyLen <= (int)len - 4){ return bodyLen; } return 0; } static int parseProxy(string &in, string &out) { try { int bodylen = checkRequest(in.c_str(), in.length()); if(bodylen > 0) { out = in.substr(0, bodylen+4); in = in.substr(bodylen+4); return TC_EpollServer::PACKET_FULL; } else { return TC_EpollServer::PACKET_LESS; } } catch(exception &ex) { return TC_EpollServer::PACKET_ERR; } return TC_EpollServer::PACKET_LESS; } }; void ProxyThread::Init(const string &sConf) { LOG_DEBUG << "ProxyThread::Init succ" << endl; } void ProxyThread::Reload() { LOG_DEBUG << "ProxyThread::Reload succ" << endl; } void ProxyThread::CreateThread() { pthread_t thread; if( !m_bStart ) { m_bStart = true; if(pthread_create(&thread, NULL, Run, (void*)this) != 0) { throw runtime_error("Create ProxyThread fail"); } } } int ProxyThread::registerProxy(ProxyConnect *connect) { LOG_DEBUG << "registerProxy start ip:" << connect->ip << " port:" << connect->port << endl; char msg[1024] = "connect"; Tint ti; ti.integer = strlen(msg); SendData sendData; sendData.fd = connect->socket.getfd(); sendData.buffer.assign(ti.byte, 4); sendData.buffer += msg; int iRet = SendBuffer(sendData.fd, sendData); if(iRet < 0){ CloseFD(sendData.fd); return -1; } return 0; } int ProxyThread::checkSocket(ProxyConnect *connect) { if(!connect->socket.isValid()){ try{ int oldfd = connect->socket.getfd(); connect->socket.createSocket(SOCK_STREAM, AF_INET); connect->socket.setblock(false); connect->socket.setKeepAlive(); connect->first = true; connect->_recvbuffer.clear(); connect->_sendbuffer.clear(); try{ connect->socket.connect(connect->ip, connect->port); } catch(TC_SocketConnect_Exception &ex) { //connect->socket.close(); if(errno != EINPROGRESS) { connect->socket.close(); LOG_ERROR << "Connect Proxy " << connect->ip << ":" << connect->port << " Exception:" << errno << endl; return -1; } } if(errno != EINPROGRESS) { connect->socket.close(); return -1; } if(oldfd != connect->socket.getfd()){ _mapSocket->erase(oldfd); } _mapSocket->insert(pair<int, ProxyConnect *>(connect->socket.getfd(), connect)); LOG_DEBUG << "proxy connected fd:" << connect->socket.getfd() << " errno:" << errno << endl; _epoller->add(connect->socket.getfd(), connect->socket.getfd(), EPOLLIN); registerProxy(connect); } catch(TC_Socket_Exception &ex) { LOG_ERROR << "Connect Proxy Socket Error:" << string(ex.what()) << endl; connect->socket.close(); return -1; } catch(...) { LOG_ERROR << "Connect Proxy Unknown Error:" << errno << endl; return -1; } } return 0; } void ProxyThread::ConnectProxy() { LOG_DEBUG << "ProxyThread::ConnectProxy Begin" << endl; _epoller = new TC_Epoller(false); _epoller->create(100); TC_Config conf; conf.parseFile(ServerConfig::BasePath + ServerConfig::ServerName + ".conf"); vector<string> proxys = conf.getDomainVector("/Config/Proxys"); for(unsigned int i = 0;i < proxys.size();i ++){ ProxyConnect *connect = new ProxyConnect(); connect->ip = conf.get("/Config/Proxys/" + proxys[i] + "<ip>", "127.0.0.1"); connect->port = atoi(conf.get("/Config/Proxys/" + proxys[i] + "<port>", "9999").c_str()); checkSocket(connect); } LOG_DEBUG << "ProxyThread::ConnectProxy End" << endl; } int ProxyThread::ParseProtocol(string &_recv, int fd, bool &first){ try { while(true){ string ro; LOG_DEBUG << "ProxyThread:ParseProtocol:" << recv << endl; int b = WupProtocol::parseProxy(_recv, ro); if(b == TC_EpollServer::PACKET_LESS) { break; } else if(b == TC_EpollServer::PACKET_FULL) { if(first){ LOG_DEBUG << "First Package: " << ro << endl; first = false; } else { if(Singletone::getInstance()->from_queue.size()<10000){ // overloaded RecvData *o = new RecvData(); o->fd = fd; o->buffer = ro; Singletone::getInstance()->from_queue.push_back(o); LOG_DEBUG << "Push Message Package: " << ro << endl; }else{ LOG_ERROR << "Recv Queue OverLoaded . Drop Message !" <<endl; } } } if(_recv.empty()){ break; } } } catch(exception &ex) { LOG_ERROR << "recv protocol error: " << string(ex.what()) << endl; return -1; } catch(...) { LOG_ERROR << "recv protocol error: " << endl; return -1; } return 1; } int ProxyThread::RecvBuffer(int fd){ map<int ,ProxyConnect *>::iterator it = _mapSocket->find(fd); if(it != _mapSocket->end()){ ProxyConnect *connect = it->second; char buffer[8192] = "\0"; while(true) { int iBytesReceived = connect->socket.recv((void*)buffer, sizeof(buffer)); if (iBytesReceived < 0) { if(errno == EAGAIN) { LOG_DEBUG << "Recv Buffer EAGAIN" << endl; break; } else{ LOG_DEBUG << "Recv Buffer ERROR iBytesReceived: " << iBytesReceived << " error:" << errno << endl; perror("error"); return -1;//closed } } if(iBytesReceived == 0){ LOG_DEBUG << "Proxy Closed ip:" << connect->ip << " port:" << connect->port << endl; return -1; } connect->_recvbuffer.append(buffer, iBytesReceived); } return ParseProtocol(connect->_recvbuffer, fd, connect->first); } return -1;//not exist } int ProxyThread::SendBuffer(int fd, SendData &o) { map<int ,ProxyConnect *>::iterator it = _mapSocket->find(fd); if(it != _mapSocket->end()){ ProxyConnect *connect = it->second; if(o.fd == fd){ connect->_sendbuffer += o.buffer; } size_t pos = 0; size_t sendLen = connect->_sendbuffer.length(); const char *sendBegin = connect->_sendbuffer.c_str(); while (pos < sendLen) { int iBytesSent = 0; LOG_DEBUG << "Send Buffer buf:" << sendBegin << " iBytesSent:" << iBytesSent << " pos:" << pos << " sendLen:" << sendLen << endl; //iBytesSent = write(fd, (const void*)(sendBegin + pos), sendLen - pos); iBytesSent = connect->socket.send((const void*)(sendBegin + pos), sendLen - pos, 0); if (iBytesSent < 0) { if(errno == EAGAIN) { LOG_DEBUG << "Send Buffer EAGAIN" << endl; break; } else { LOG_DEBUG << "Send Buffer ERROR iBytesSent: " << iBytesSent << " error:" << errno << endl; return -1; } } pos += iBytesSent; if(pos < sendLen) { break; } } if(pos < sendLen){ //need to send later _epoller->mod(connect->socket.getfd(), connect->socket.getfd(), EPOLLIN|EPOLLOUT); } if(pos > 0){ connect->_sendbuffer = connect->_sendbuffer.substr(pos); } LOG_DEBUG << "Send Buffer Finished pos:" << pos << " sendLen:" << sendLen << endl; } return 0; } void ProxyThread::CloseFD(int fd){ _epoller->del(fd, fd, 0); map<int ,ProxyConnect *>::iterator it = _mapSocket->find(fd); if(it != _mapSocket->end()){ it->second->socket.close(); LOG_ERROR << "Close Proxy fd:" << fd << " ip" << it->second->ip << " port:" << it->second->port << endl; } } void* ProxyThread::Run(void* arg) { pthread_detach(pthread_self()); ProxyThread* pthis = (ProxyThread*) arg; pthis->SetRuning(true); _mapSocket = new map<int, ProxyConnect *>(); ConnectProxy(); while(pthis->IsStart()) { //check alive for(map<int ,ProxyConnect *>::iterator it = _mapSocket->begin();it != _mapSocket->end();it ++) { checkSocket(it->second); } //check io int iEvNum = _epoller->wait(100);//100ms if(iEvNum < 0){ perror("Epoll Wait Error"); continue; }else if(iEvNum > 0){ for(int i = 0; i < iEvNum; ++i) { const epoll_event &ev = _epoller->get(i); if (ev.events & EPOLLERR || ev.events & EPOLLHUP) { LOG_DEBUG << "epoll EPOLLERR|EPOLLHUP fd:" << ev.data.fd << endl; CloseFD(ev.data.fd); continue; } if(ev.events & EPOLLIN) //read data { LOG_DEBUG << "epoll EPOLLIN fd:" << ev.data.fd << endl; int ret = RecvBuffer(ev.data.fd); if(ret < 0) { CloseFD(ev.data.fd); continue; } _epoller->mod(ev.data.fd, ev.data.fd, EPOLLIN); } if (ev.events & EPOLLOUT) // write data { LOG_DEBUG << "epoll EPOLLOUT fd:" << ev.data.fd << endl; SendData sendData; sendData.fd = 0; int iRet = SendBuffer(ev.data.fd, sendData); if(iRet < 0){ CloseFD(ev.data.fd); continue; } } } } while(Singletone::getInstance()->to_queue.size() > 0){ LOG_DEBUG << "ProxyThread Has Data to send. Data Number:" << Singletone::getInstance()->to_queue.size() << endl; SendData *sendData = NULL; bool ret = Singletone::getInstance()->to_queue.pop_front(sendData,0); if(ret){ map<int ,ProxyConnect *>::iterator it = _mapSocket->find(sendData->fd); if(it != _mapSocket->end()){ int iRet = SendBuffer(sendData->fd, *sendData); delete sendData; if(iRet < 0){ CloseFD(sendData->fd); continue; } } break; } } } LOG_DEBUG << "thread end" << endl; pthis->SetRuning(false); pthis->SetStart(false); return NULL; }
2，WorkerThread.cpp
#include "WorkerThread.h" #include "Singletone.h" #include "SwitchSvr.h" TC_ThreadPool WorkerThread::tpool; TC_ThreadLock WorkerThread::l; int WorkerThread::threadnum; void WorkerThread::Init(const string &sConf) { LOG_DEBUG << "WorkerThread::Init succ" << endl; TC_Config conf; conf.parseFile(ServerConfig::BasePath + ServerConfig::ServerName + ".conf"); threadnum = atoi(conf.get("/Config/WorkerThread/<num>", "4").c_str()); } void WorkerThread::Reload() { LOG_DEBUG << "WorkerThread::Reload succ" << endl; } void WorkerThread::CreateThread() { pthread_t thread; if( !m_bStart ) { m_bStart = true; if(pthread_create(&thread, NULL, Run, (void*)this) != 0) { throw runtime_error("Create WorkerThread fail"); } } } void handlework(RecvData *o) { LOG_DEBUG << "WorkerThread threadid:" << pthread_self() << " handle " << o->buffer << endl; string msg = "Hello"+o->buffer.substr(4); Tint ti; ti.integer = msg.length(); if(Singletone::getInstance()->to_queue.size() < 10000){ SendData *sendData = new SendData(); sendData->fd = o->fd; sendData->buffer.assign(ti.byte, 4); sendData->buffer += msg; Singletone::getInstance()->to_queue.push_back(sendData); } else { LOG_ERROR << "Send Queue OverLoaded . Drop Message !" <<endl; } delete o; } void* WorkerThread::Run(void* arg) { LOG_DEBUG << "WorkerThread Run Start " << pthread_self() << endl; pthread_detach(pthread_self()); WorkerThread* pthis = (WorkerThread*) arg; pthis->SetRuning(true); sleep(1); tpool.init(threadnum); tpool.start(); TC_Functor<void, TL::TLMaker<RecvData *>::Result> cmd(handlework); while(true){ while(Singletone::getInstance()->from_queue.size() > 0){ LOG_DEBUG << "WorkerThread Has Work to Do. Work Number:" << Singletone::getInstance()->from_queue.size() << endl; RecvData *o = NULL; bool ret = Singletone::getInstance()->from_queue.pop_front(o,0); if(ret){ TC_Functor<void, TL::TLMaker<RecvData *>::Result>::wrapper_type fw(cmd, o); tpool.exec(fw); } break; } usleep(100); } tpool.waitForAllDone(-1); tpool.stop(); pthis->SetRuning(false); pthis->SetStart(false); return NULL; }
定义的结构体及数据如下：
struct ProxyConnect { TC_Socket socket; string ip; int port; string _recvbuffer; string _sendbuffer; bool first; };
#ifndef __SINGLETONE_H__ #define __SINGLETONE_H__ #include <pthread.h> #include "servant/Application.h" #include "util/tc_config.h" #include "util/tc_singleton.h" #include <wbl/pthread_util.h> #include <map> using namespace std; typedef union{ int integer; char byte[4]; } Tint; struct RecvData { int fd; string buffer; }; struct SendData { int fd; string buffer; }; typedef TC_ThreadQueue<RecvData *> from_proxy_queue; typedef TC_ThreadQueue<SendData *> to_proxy_queue; class Singletone : public TC_Singleton<Singletone> { public: Singletone(){} ~Singletone(){} void init(); public: map<int, taf::JceCurrentPtr> m_Current; from_proxy_queue from_queue; to_proxy_queue to_queue; };
