使用splice实现高效的代理服务器
很多网络应用场景下, 当原设备与目标设备无法直接建立连接时,这时就需要一台代理服务器进行中转。代理服务器只需要将来自源设备的报文 原封不动的转发给目标设备,而并不需要知道报文的具体内容。在这种情况下,服务器就没必要将客户端的报文读出来。每读取一次都需要从内核态到用户态的一次拷贝,在流量密集型的应用中,这样做显然很消耗服务器性能。而splice就是为了解决这个问题,它实现了不同文件描述符之间的0拷贝操作。关于splice函数的描述如下:
ssize_t splice(int fd_in, loff_t* off_in,int fd_out, loff_t* off_out, size_t len, unsigned int flags);
fd_in参数是带输入的文件描述符
off_in表示从输入数据流的何处开始读取数据,如果fd_in是管道文件描述符,则必须设置为NULL,表示从0开始
fd_out/off_out 表示输出的文件描述符与偏移
len表示要读取的长度
flags 标志,常用的两个标志SPLICE_F_NONBLOCK,表示非阻塞的splice操作,但要依赖于文件描述符本身的阻塞状态;SPLICE_F_MORE表示给内核一个提示,后续splice调用将读取更多数据
使用splice时,fd_in于fd_out必须至少有一个是管道文件描述符
下面给出一个demo,实现两个客户端之间的报文透传
服务器代码:
#include <sys/types.h> #include <sys/socket.h> #include <netinet/in.h> #include <arpa/inet.h> #include <assert.h> #include <stdio.h> #include <unistd.h> #include <errno.h> #include <string.h> #include <fcntl.h> #include <stdlib.h> #include <sys/epoll.h> #include <pthread.h> #define MAX_EVENT_NUMBER 1024 #define BUFFER_SIZE 10 int connfd[2] = {0}; int setnonblocking( int fd ) { int old_option = fcntl( fd, F_GETFL ); int new_option = old_option | O_NONBLOCK; fcntl( fd, F_SETFL, new_option ); return old_option; } void addfd( int epollfd, int fd, bool enable_et ) { epoll_event event; event.data.fd = fd; event.events = EPOLLIN; if( enable_et ) { event.events |= EPOLLET; } epoll_ctl( epollfd, EPOLL_CTL_ADD, fd, &event ); setnonblocking( fd ); } void lt( epoll_event* events, int number, int epollfd, int listenfd ) { char buf[ BUFFER_SIZE ]; for ( int i = 0; i < number; i++ ) { int sockfd = events[i].data.fd; if ( sockfd == listenfd ) { static int conntimes = 0; struct sockaddr_in client_address; socklen_t client_addrlength = sizeof( client_address ); int fd = accept( listenfd, ( struct sockaddr* )&client_address, &client_addrlength ); addfd( epollfd, fd, false ); if(conntimes <= 1) { connfd[conntimes] = fd; } conntimes++; printf("new conn, %d\n",fd); } else if ( events[i].events & EPOLLIN ) { int pipefd[2]; pipe(pipefd); if(sockfd == connfd[0]) { splice(connfd[0], NULL, pipefd[1], NULL, 4096,SPLICE_F_MORE); splice(pipefd[0], NULL, connfd[1], NULL, 4096,SPLICE_F_MORE); } else if(sockfd == connfd[1]) { splice(connfd[1], NULL, pipefd[1], NULL, 4096,SPLICE_F_MORE); splice(pipefd[0], NULL, connfd[0], NULL, 4096,SPLICE_F_MORE); } else { int recvlen = read(sockfd,buf,BUFFER_SIZE-1); buf[recvlen] = 0; printf("recv buf : %s\n",buf); } close(pipefd[0]); close(pipefd[1]); } else { printf( "something else happened \n" ); } } } int main( int argc, char* argv[] ) { if( argc <= 2 ) { printf( "usage: %s ip_address port_number\n", basename( argv[0] ) ); return 1; } const char* ip = argv[1]; int port = atoi( argv[2] ); int ret = 0; struct sockaddr_in address; bzero( &address, sizeof( address ) ); address.sin_family = AF_INET; inet_pton( AF_INET, ip, &address.sin_addr ); address.sin_port = htons( port ); int listenfd = socket( PF_INET, SOCK_STREAM, 0 ); assert( listenfd >= 0 ); ret = bind( listenfd, ( struct sockaddr* )&address, sizeof( address ) ); assert( ret != -1 ); ret = listen( listenfd, 5 ); assert( ret != -1 ); epoll_event events[ MAX_EVENT_NUMBER ]; int epollfd = epoll_create( 5 ); assert( epollfd != -1 ); addfd( epollfd, listenfd, true ); while( 1 ) { int ret = epoll_wait( epollfd, events, MAX_EVENT_NUMBER, -1 ); if ( ret < 0 ) { printf( "epoll failure\n" ); break; } lt( events, ret, epollfd, listenfd ); } close( listenfd ); return 0; }
客户端代码:
#include <sys/types.h> #include <sys/socket.h> #include <netinet/in.h> #include <arpa/inet.h> #include <assert.h> #include <stdio.h> #include <unistd.h> #include <string.h> #include <stdlib.h> #include <poll.h> #include <fcntl.h> int connectserver(const char *ip, int port) { int rcv_size = 0; size_t optlen = sizeof(int); struct sockaddr_in server_address; bzero( &server_address, sizeof( server_address ) ); server_address.sin_family = AF_INET; inet_pton( AF_INET, ip, &server_address.sin_addr ); server_address.sin_port = htons( port ); int sockfd = socket( PF_INET, SOCK_STREAM, 0 ); assert( sockfd >= 0 ); if ( connect( sockfd, ( struct sockaddr* )&server_address, sizeof( server_address ) ) < 0 ) { printf( "connection failed\n" ); close( sockfd ); return -1; } return sockfd; } int main( int argc, char* argv[] ) { if( argc <= 2 ) { printf( "usage: %s ip_address port_number\n", argv[0] ); return 1; } const char* ip = argv[1]; int port = atoi( argv[2] ); int pipefd[2]; pipe(pipefd); int sockfd = connectserver(ip,port); assert(sockfd >= 0); fd_set readsets; FD_ZERO(&readsets); while(1) { FD_SET(STDIN_FILENO,&readsets); FD_SET(sockfd, &readsets); int ret = select(sockfd+1 , &readsets,NULL,NULL,NULL); assert(ret >= 0); if(FD_ISSET(STDIN_FILENO, &readsets)) { splice(STDIN_FILENO, NULL, pipefd[1], NULL, 4096,SPLICE_F_MORE); splice(pipefd[0], NULL, sockfd, NULL, 4096,SPLICE_F_MORE); } else if(FD_ISSET(sockfd,&readsets)) { splice(sockfd, NULL, pipefd[1], NULL, 4096,SPLICE_F_MORE); splice(pipefd[0], NULL, STDOUT_FILENO, NULL, 4096,SPLICE_F_MORE); } } close(pipefd[0]); close(pipefd[1]); close(sockfd); return 1; }