linux下打满网卡的带宽和影响打满网卡的原因

 

1.打满网卡带宽的定义

网络传输场景很多，咱们这里用一个最常见的场景：

客户端发送数据，服务端接收数据，所以这里给一个打满的定义：

使用监控软件发现客户端的上行和服务端的下载带宽用满

 

2.测试软件

客户端：

// Client side C program to demonstrate Socket
// programming
#include <arpa/inet.h>
#include <stdio.h>
#include <string.h>
#include <sys/socket.h>
#include <unistd.h>
#include <errno.h>
#include <stdlib.h>
#include <sched.h>
#define PORT 8080

#define BUFF_SIZE (4*1024)
#define BUFF_SEND_SIZE (2*1024)


void loop_barrier(int loop_count)
{
        static count = 1;
        double sum = 0.0f;
        for(int i = 0 ; i < loop_count; i++)
                sum = sum * 1223491.3f * (double)(9401.f + i) * count++;
        
}


int main(int argc, char const* argv[])
{
    short port = atoi(argv[1]);
    int buff_send_size = atoi(argv[2]);
    int usleep_time = atoi(argv[3]);
    int loop_count = atoi(argv[4]);
    printf("%d %d %d %d\n", port, buff_send_size, usleep_time, loop_count);
    int status, valread, client_fd;
    struct sockaddr_in serv_addr;
    //char* hello = "Hello from client adsadfasdfasdfasdf0001234";
    char* hello = malloc(buff_send_size);
    for(int i = 0; i < buff_send_size; i++)
        hello[i] = '0' + i % 10;

    char buffer[BUFF_SIZE] = { 0 };
    if ((client_fd = socket(AF_INET, SOCK_STREAM, 0)) < 0) {
        printf("\n Socket creation error \n");
        return -1;
    }

    serv_addr.sin_family = AF_INET;
    //serv_addr.sin_port = htons(PORT);
    serv_addr.sin_port = htons(port);

    // Convert IPv4 and IPv6 addresses from text to binary
    // form
    if (inet_pton(AF_INET, "10.10.3.93", &serv_addr.sin_addr)
        <= 0) {
        printf(
            "\nInvalid address/ Address not supported \n");
        return -1;
    }

    if ((status
        = connect(client_fd, (struct sockaddr*)&serv_addr,
                sizeof(serv_addr)))
        < 0) {
        printf("\nConnection Failed \n");
        return -1;
    }

    while(1)
    {
        //usleep(usleep_time);
        //sched_yield();
        loop_barrier(loop_count);
        send(client_fd, hello, strlen(hello), 0);
        continue;
        //printf("[send] %s\n", hello);
        valread = read(client_fd, buffer, sizeof(buffer)); // subtract 1 for the null
        if (valread <= 0)
        {
            printf("recv %d %s", valread, errno);
            exit(-2);    
        }
        buffer[valread] = '\0';
        //printf("[recv] %s\n", buffer);
        //sleep(1);
    }

    // closing the connected socket
    close(client_fd);
    return 0;
}

测试命令：

nohup ./client 8080 2048 0 1000 &（连接服务端端口8080，一次发送buffer是2048字节，usleep 0，算术计算loop循环1000）
nohup ./client 8081 2048 0 1000 &
nohup ./client 8082 2048 0 1000 &

 

服务端：

// Server side C program to demonstrate Socket
// programming
#include <netinet/in.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/socket.h>
#include <unistd.h>
#include <errno.h>
#include <sched.h> 

#define PORT 8080

#define BUFF_SIZE (4*1024)

void loop_barrier(int loop_count)
{
    static count = 1;
    double sum = 0.0f;
    for(int i = 0 ; i < loop_count; i++)
        sum = sum * 1223491.3f * (double)(9401.f + i) * count++;
        
}

int main(int argc, char const* argv[])
{
    short port = atoi(argv[1]);
    int usleep_time = atoi(argv[2]);
    int loop_count = atoi(argv[3]);
    printf("%d %d %d\n",port, usleep_time, loop_count);
    int server_fd, new_socket;
    ssize_t valread;
    struct sockaddr_in address;
    int opt = 1;
    socklen_t addrlen = sizeof(address);
    char buffer[BUFF_SIZE] = { 0 };
    char* hello = "Hello from server asdfasdfsadfasdfasdf009999999999";

    // Creating socket file descriptor
    if ((server_fd = socket(AF_INET, SOCK_STREAM, 0)) < 0) {
        perror("socket failed");
        exit(EXIT_FAILURE);
    }

    // Forcefully attaching socket to the port 8080
    if (setsockopt(server_fd, SOL_SOCKET,
                SO_REUSEADDR | SO_REUSEPORT, &opt,
                sizeof(opt))) {
        perror("setsockopt");
        exit(EXIT_FAILURE);
    }
    address.sin_family = AF_INET;
    address.sin_addr.s_addr = INADDR_ANY;
    //address.sin_port = htons(PORT);
    address.sin_port = htons(port);

    // Forcefully attaching socket to the port 8080
    if (bind(server_fd, (struct sockaddr*)&address,
            sizeof(address))
        < 0) {
        perror("bind failed");
        exit(EXIT_FAILURE);
    }
    if (listen(server_fd, 3) < 0) {
        perror("listen");
        exit(EXIT_FAILURE);
    }
    if ((new_socket
        = accept(server_fd, (struct sockaddr*)&address,
                &addrlen))
        < 0) {
        perror("accept");
        exit(EXIT_FAILURE);
    }

    while(1)
    {
        valread = read(new_socket, buffer,
                sizeof(buffer)); // subtract 1 for the null
        //usleep(usleep_time);
        //sched_yield();  
        loop_barrier(loop_count);
        if (valread <= 0)
                {
                        printf("recv %d %s", valread, errno);
                        exit(-2);
                }
        continue;
        buffer[valread] = '\0';
    //    printf("[recv] %s\n", buffer);
        send(new_socket, hello, strlen(hello), 0);
    //    printf("[send] %s\n", hello);
    //    sleep(1);
    }
    // terminator at the end
    //printf("%s\n", buffer);
    //send(new_socket, hello, strlen(hello), 0);
    //printf("Hello message sent\n");

    // closing the connected socket
    close(new_socket);
    // closing the listening socket
    close(server_fd);
    return 0;
}

测试命令：

nohup ./server 8080 0 1000 &（监听端口8080，usleep 0，算术计算loop循环1000）
nohup ./server 8081 0 1000 &
nohup ./server 8082 0 1000 &

 

3.测试结果：

说明：因为是在一台宿主机进行测试，所以其实不走宿主机的网卡，走的是虚拟机的网卡，但是对网络敏感度更高，对比实验更有说服力：

usleep 100 结果:300Mb
usleep 10 结果:600Mb
usleep 1 结果:690Mb
empty loop 结果:20Gb
usleep 0 结果:708Mb (deactivate_task调用)
sched_yield 结果:10Gb
loop 10000 calculate multipie_hard 结果: 2.7Gb (用了static 避免走缓存)
loop 1000 calculate multipie_hard 结果: 8.2Gb (用了static 避免走缓存)
loop 100 calculate multipie_hard 结果: 15Gb (用了static 避免走缓存)

 

4.初步分析结论：

empty loop ： 很容易就把网络带宽打满。

usleep 0  ：网络带宽完全利用不上，猜测是deactivate_task等调用，引发更大的耗时。

sched_yield  ：出让cpu，网络带宽也能打的很高。

loop 10000 calculate multipie_hard : 就算进行很耗时的cpu计算，网络带宽依然可以打的很高。

 

5.总结：

所以网卡带宽打满，不仅仅是和其他操作耗时有关，还跟操作系统的一些进行内核态切换的api有关（例如usleep）