C语言终篇--基于epoll ET模式的非阻塞IO服务器模型

使用技术:

1 epoll事件驱动机制：使用epoll作为IO多路复用的技术，以高效地管理多个socket上的事件。

2 边缘触发（Edge Triggered, ET）模式：epoll事件以边缘触发模式运行，这要求代码必须负责消费所有可用的数据，直到收到EAGAIN或EWOULDBLOCK错误。

3 非阻塞IO：通过设置socket为非阻塞模式，确保IO操作不会导致服务器线程挂起等待。

4 信号处理：捕捉SIGINT信号，以允许服务器通过中断信号优雅地关闭和清理资源。

5 地址重用：设置SO_REUSEADDR套接字选项，允许服务器快速重启而不必等待TIME_WAIT状态的连接自然超时。

6 调整接收缓冲区为1字节,将接收的数据存在内存的容器中,用于模拟大数据来袭,无法全部接收的情景

7 可动态扩容的兴趣事件列表

8 使用whlie()循环,分别包裹ET模式下的accpet(),recv(),send(),确保读尽,写尽,及合适的时机退出

注意事项:

运行环境:

unix-like gun_c 因为涉及到信号需独立终端不能再IDE环境下运行

地址端口按需调整

赠送:

epoll LT模式服务端对比用双循环阻塞服务端测试用客户端

运行效果:

epoll ET模式服务端:

#define _GNU_SOURCE
#include <stdio.h>
#include <sys/socket.h>
#include <sys/un.h>
#include <fcntl.h>
#include <arpa/inet.h>
#include <netinet/in.h>
#include <sys/epoll.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <signal.h>
#include <errno.h>

#define SERVER_IP "192.168.142.132"
#define SERVER_PORT 50001
// 单线程访问count,count表示感兴趣的fd的总数
int count = 0;
// 用于清理函数
void *global_r_events;
char *global_buf_p;
int global_server_sockfd;
// 注册清理函数
void clean_up()
{
    free(global_r_events);
    free(global_buf_p);
    close(global_server_sockfd);
    printf("clean_up\n");
}
// 如果按了CTRL+C则退出并执行清理函数
void sig_handler(int sig)
{
    exit(EXIT_SUCCESS);
}
// 此函数功能将socket设为非阻塞
void set_non_blocking(int fd)
{
    int flags = fcntl(fd, F_GETFL);
    if (flags == -1)
    {
        perror("fcntl F_GETFL");
    }
    flags |= O_NONBLOCK;
    if (fcntl(fd, F_SETFL, flags) == -1)
    {
        perror("fcntl F_SETFL");
    }
}
// 此函数功能 调用设置非阻塞函数+将sockfd加入兴趣列表+将sockfd设为ET
void add_ins_events(int epoll_fd, int fd)
{
    // 设成非阻塞
    set_non_blocking(fd);
    struct epoll_event ins_event = {0};
    // 关注读+边缘触发
    ins_event.events = EPOLLIN | EPOLLET;
    ins_event.data.fd = fd;
    if (epoll_ctl(epoll_fd, EPOLL_CTL_ADD, fd, &ins_event) == -1)
    {
        perror("epoll_ctl");
    }
    // count表示感兴趣的fd的总数
    count++;
}

int main()
{
    int server_sockfd, client_sockfd;
    struct sockaddr_in server_sockaddr, client_sockaddr;
    memset(&server_sockaddr, 0, sizeof(server_sockaddr));
    memset(&client_sockaddr, 0, sizeof(client_sockaddr));
    socklen_t client_sockaddr_len = sizeof(client_sockaddr);
    socklen_t server_sockaddr_len = sizeof(server_sockaddr);
    ssize_t send_bytes, recv_bytes;
    // 预留
    char send_buf[1024] = {0};
    // 调整接收缓冲区为1字节,用于模拟大数据来袭,无法全部接收的情景
    char recv_buf[1] = {0};
    // 一个用于添加兴趣事件的结构体
    struct epoll_event ins_event = {0};

    // i用于循环,optval套接字属性用,size是用户空间就绪事件列表的长度
    // 用户空间兴趣事件的计数是count,遍历的上限也是count,就绪事件的个数一定小于count,size只作为容器大小存在
    int i = 0, optval = 1, size = 1024;
    // 需要发送的总字节数,已发送字节数,已接收字节总数
    int send_total, sent, recv_total;
    // 注册信号
    if (signal(SIGINT, sig_handler) == SIG_ERR)
    {
        perror("signal");
    }

    // 注册退出清理函数
    atexit(clean_up);
    // 创建socket ipv4 tcp
    server_sockfd = socket(AF_INET, SOCK_STREAM, 0);
    if (server_sockfd == -1)
    {
        perror("socket");
    }
    // 向全局变量传递值,便于关闭
    global_server_sockfd = server_sockfd;
    // 地址端口复用
    if (setsockopt(server_sockfd, SOL_SOCKET, SO_REUSEADDR, &optval, sizeof(optval)) == -1)
    {
        perror("setsockopt");
    }
    // 绑定
    server_sockaddr.sin_family = AF_INET;
    server_sockaddr.sin_port = htons(SERVER_PORT);
    if (inet_pton(AF_INET, SERVER_IP, &server_sockaddr.sin_addr.s_addr) == -1)
    {
        perror("inet_pton");
    }
    if (bind(server_sockfd, (struct sockaddr *)&server_sockaddr, server_sockaddr_len) == -1)
    {
        perror("bind");
    }
    // 监听
    if (listen(server_sockfd, 1024) == -1)
    {
        perror("listen");
    }
    // 创建epoll实例
    int epoll_fd = epoll_create1(0);
    if (epoll_fd == -1)
    {
        perror("epoll_create1");
    }
    // 将server_sockfd设成非阻塞 对读感兴趣 ET模式 count++
    add_ins_events(epoll_fd, server_sockfd);
    // 这是一个在堆上分配的就绪事件容器,epoll_wait将填充这个容器的一部分
    struct epoll_event *r_events = (struct epoll_event *)calloc(size, sizeof(struct epoll_event));
    if (r_events == NULL)
    {
        perror("calloc");
    }
    else
    {
        // 向全局变量传递值,便于释放
        global_r_events = r_events;
    }
    // 这是一个足够大的内存缓冲区 用于储存接收的数据
    // 因为每次接收被限流为1字节 模拟大数据到来时缓冲区的渺小
    // 每次接收的数据会用指针算数,拼在已接收数据的后面
    char *buf_p = (char *)calloc(1024, sizeof(char));
    if (buf_p == NULL)
    {
        perror("calloc");
    }
    else
    {
        // 向全局变量传递值,便于释放
        global_buf_p = buf_p;
    }

    printf("server start ...\n");
    // 服务器主循环 里面有3个 whlie(1)循环,分别用于包裹ET模式下的accpet,recv,send
    while (1)
    {
        // 清零就绪事件的容器
        memset(r_events, 0, size * sizeof(struct epoll_event));
        // count表示已添加兴趣列表事件的最大值,size是容器大小,因为就绪事件的大小永远小于等于兴趣事件,所以用count
        // -1表示没有就绪事件则一直阻塞,为除了算力问题外,整个架构设计的唯一IO阻塞点
        int r = epoll_wait(epoll_fd, r_events, count, -1);
        if (r > 0)
        {
            // r>0,r为就绪事件数量,所以遍历r
            for (i = 0; i < r; i++)
            {
                // 如果读就绪且fd是server_sockfd
                if ((r_events[i].events & EPOLLIN) && (r_events[i].data.fd == server_sockfd))
                {
                    // ET模式下包裹accept的循环
                    while (1)
                    {
                        // 接收连接
                        client_sockfd = accept(server_sockfd, (struct sockaddr *)&client_sockaddr, &client_sockaddr_len);
                        if (client_sockfd < 0)
                        {
                            if (errno == EAGAIN || errno == EWOULDBLOCK)
                            {
                                // 读尽,跳出循环
                                break;
                            }
                            else
                            {
                                // 这就是出错了
                                perror("accept");
                            }
                        }
                        else
                        {
                            // 打印日志
                            printf("[+] %u: connected\n", ntohs(client_sockaddr.sin_port));
                            // 将client_sockfd设成非阻塞 对读感兴趣 ET模式 count++
                            add_ins_events(epoll_fd, client_sockfd);
                            // 如果count的值将要接近容器容量,则容器容量翻倍
                            if ((size - count) < 100)
                            {
                                size += size;
                                r_events = (struct epoll_event *)realloc(r_events, size * sizeof(struct epoll_event));
                                if (r_events == NULL)
                                {
                                    perror("realloc");
                                }
                                // 赋值全局变量,方便释放
                                global_r_events = r_events;
                            }
                        }
                    }
                }
                // 如果fd读就绪且不是server_sockfd,此时处理读写
                if ((r_events[i].events & EPOLLIN) && (r_events[i].data.fd != server_sockfd))
                {
                    // 获取peer端口号,打印日志用
                    getpeername(r_events[i].data.fd, (struct sockaddr *)&client_sockaddr, &client_sockaddr_len);
                    // 打印日志
                    printf("[+] %u: recv ready\n", ntohs(client_sockaddr.sin_port));
                    // 已接收的字节总数,用于在buf_p(一个足够大的内存缓冲区)中定位
                    recv_total = 0;
                    // ET模式下 读尽策略 包裹recv 的循环
                    while (1)
                    {
                        // 读缓冲区被限流到1字节
                        recv_bytes = recv(r_events[i].data.fd, recv_buf, sizeof(recv_buf), 0);
                        // 如果读到了数据
                        if (recv_bytes > 0)
                        {
                            // 将数据追加在buf_p的最后
                            memcpy(buf_p + recv_total, recv_buf, recv_bytes);
                            // 计数
                            recv_total += recv_bytes;
                        }
                        if (recv_bytes < 0)
                        {
                            // 读尽
                            if (errno == EAGAIN || errno == EWOULDBLOCK)
                            {
                                // 打印buf_p(一个足够大的内存缓冲区)
                                printf("[+] %u: %s\n", ntohs(client_sockaddr.sin_port), buf_p);
                                /* send */
                                // 每次发送的字节
                                send_bytes = 0;
                                // 需要发送的总字节
                                send_total = strlen(buf_p);
                                // 已发送总字节
                                sent = 0;

                                // ET模式下 包裹send的循环
                                while (1)
                                {
                                    // buf_p + sent是指针运算表示发送位置,send_total - sent是剩余发送量
                                    send_bytes = send(r_events[i].data.fd, buf_p + sent, send_total - sent, 0);
                                    if (send_bytes == -1)
                                    {
                                        // 表示发送缓冲区已满
                                        if (errno == EAGAIN || errno == EWOULDBLOCK)
                                        {
                                            // 添加对写感兴趣
                                            ins_event.events = EPOLLET | EPOLLIN | EPOLLOUT;
                                            ins_event.data.fd = r_events[i].data.fd;
                                            epoll_ctl(epoll_fd, EPOLL_CTL_MOD, r_events[i].data.fd, &ins_event);
                                            // 直接跳出循环,如果写就绪epoll_wait会通知
                                            break;
                                        }
                                        // 对方异常关闭
                                        if (errno == ECONNRESET)
                                        {
                                            // 将fd移除兴趣事件列表
                                            epoll_ctl(epoll_fd, EPOLL_CTL_DEL, r_events[i].data.fd, NULL);
                                            // 关闭fd
                                            close(r_events[i].data.fd);
                                            // 兴趣列表最大值-1
                                            count--;
                                            // 跳出send循环
                                            break;
                                        }
                                    }
                                    // 如果发送了一些数据
                                    if (send_bytes > 0)
                                    {
                                        // 累加到已发送字节
                                        sent += send_bytes;
                                        // 如果 已发送字节等于发送总字节 则取消对于写的兴趣
                                        // 如果 之前fd没有关注写 则不变
                                        if (sent == send_total)
                                        {
                                            ins_event.events = EPOLLET | EPOLLIN;
                                            ins_event.data.fd = r_events[i].data.fd;
                                            epoll_ctl(epoll_fd, EPOLL_CTL_MOD, r_events[i].data.fd, &ins_event);
                                            break;
                                        }
                                    }
                                }
                                /* send end */
                                // 如果 不发送的话打印完接收数据就可以直接 跳出循环
                                break;
                            }
                            // recv 对方异常终止
                            else if (errno == ECONNRESET)
                            {
                                // 打印日志
                                char a[32] = {0};
                                snprintf(a, sizeof(a), "[-] %u", ntohs(client_sockaddr.sin_port));
                                perror(a);
                                // 从感兴趣列表删除
                                epoll_ctl(epoll_fd, EPOLL_CTL_DEL, r_events[i].data.fd, NULL);
                                // 关闭fd
                                close(r_events[i].data.fd);
                                // 兴趣列表最大值-1
                                count--;
                                break;
                            }
                            else
                            {
                                perror("recv");
                            }
                        }
                        // 如果读到0,表示peer正常关闭了连接
                        if (recv_bytes == 0)
                        {
                            getpeername(r_events[i].data.fd, (struct sockaddr *)&client_sockaddr, &client_sockaddr_len);
                            printf("[-] %u: closed\n", ntohs(client_sockaddr.sin_port));
                            // 从兴趣列表删除这个fd
                            epoll_ctl(epoll_fd, EPOLL_CTL_DEL, r_events[i].data.fd, NULL);
                            // 关闭fd
                            close(r_events[i].data.fd);
                            // 兴趣列表最大值-1
                            count--;
                            // 跳出读循环
                            break;
                        }
                    }
                }
            }
        }
        if (r == -1)
        {
            perror("epoll_wait");
        }
    }
    return 0;
}

epoll LT模式服务端 :

#define _GNU_SOURCE
#include <stdio.h>
#include <sys/socket.h>
#include <sys/un.h>
#include <sys/ioctl.h>
#include <fcntl.h>
#include <netdb.h>
#include <arpa/inet.h>
#include <netinet/in.h>
#include <sys/select.h>
#include <sys/poll.h>
#include <sys/epoll.h>
#include <sys/types.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <aio.h>
#include <signal.h>

#define SERVER_IP "192.168.142.132"
#define SERVER_PORT 50001

int count = 0;
void *global_r_events;
int global_server_sockfd;
void clean_up()
{
    free(global_r_events);
    close(global_server_sockfd);
    printf("clean_up\n");
}
void sig_handler(int sig)
{
    exit(EXIT_SUCCESS);
}

void set_non_blocking(int fd)
{
    int flags = fcntl(fd, F_GETFL);
    if (flags == -1)
    {
        perror("fcntl F_GETFL");
    }
    flags |= O_NONBLOCK;
    if (fcntl(fd, F_SETFL, flags) == -1)
    {
        perror("fcntl F_SETFL");
    }
}

void add_ins_events(int epoll_fd, int fd)
{
    set_non_blocking(fd);
    struct epoll_event ins_event = {0};
    ins_event.events = EPOLLIN;
    ins_event.data.fd = fd;
    if (epoll_ctl(epoll_fd, EPOLL_CTL_ADD, fd, &ins_event) == -1)
    {
        perror("epoll_ctl");
    }
    count++;
}

int main()
{
    int server_sockfd, client_sockfd;
    struct sockaddr_in server_sockaddr, client_sockaddr;
    memset(&server_sockaddr, 0, sizeof(server_sockaddr));
    memset(&client_sockaddr, 0, sizeof(client_sockaddr));
    socklen_t client_sockaddr_len = sizeof(client_sockaddr);
    socklen_t server_sockaddr_len = sizeof(server_sockaddr);
    ssize_t send_bytes, recv_bytes;
    char send_buf[1024] = "How can I help you today ?";
    char recv_buf[1024] = {0};
    int i = 0, optval = 1, size = 1024;

    signal(SIGINT, sig_handler);

    atexit(clean_up);

    server_sockfd = socket(AF_INET, SOCK_STREAM, 0);
    if (server_sockfd == -1)
    {
        perror("socket");
    }
    global_server_sockfd = server_sockfd;

    if (setsockopt(server_sockfd, SOL_SOCKET, SO_REUSEADDR, &optval, sizeof(optval)) == -1)
    {
        perror("setsockopt");
    }

    server_sockaddr.sin_family = AF_INET;
    server_sockaddr.sin_port = htons(SERVER_PORT);
    if (inet_pton(AF_INET, SERVER_IP, &server_sockaddr.sin_addr.s_addr) == -1)
    {
        perror("inet_pton");
    }
    if (bind(server_sockfd, (struct sockaddr *)&server_sockaddr, server_sockaddr_len) == -1)
    {
        perror("bind");
    }

    if (listen(server_sockfd, 1024) == -1)
    {
        perror("listen");
    }

    int epoll_fd = epoll_create1(0);
    if (epoll_fd == -1)
    {
        perror("epoll_create1");
    }

    add_ins_events(epoll_fd, server_sockfd);

    struct epoll_event *r_events = (struct epoll_event *)calloc(size, sizeof(struct epoll_event));
    if (r_events == NULL)
    {
        perror("calloc");
    }
    global_r_events = r_events;

    printf("server start ...\n");

    while (1)
    {
        memset(r_events, 0, size * sizeof(struct epoll_event));
        // count用于有效遍历,而size则是数组的实际大小
        int r = epoll_wait(epoll_fd, r_events, count, -1);
        if (r > 0)
        {
            for (i = 0; i < r; i++)
            {
                if ((r_events[i].events & EPOLLIN) && (r_events[i].data.fd == server_sockfd))
                {
                    printf("accept ready\n");
                    client_sockfd = accept(server_sockfd, NULL, NULL);
                    add_ins_events(epoll_fd, client_sockfd);
                    if ((size - count) < 10)
                    {
                        size += size;
                        r_events = (struct epoll_event *)realloc(r_events, size * sizeof(struct epoll_event));
                        if (r_events == NULL)
                        {
                            perror("realloc");
                        }
                        global_r_events = r_events;
                    }
                }
                if ((r_events[i].events & EPOLLIN) && (r_events[i].data.fd != server_sockfd))
                {
                    printf("recv ready\n");
                    recv_bytes = recv(r_events[i].data.fd, recv_buf, sizeof(recv_buf), 0);
                    if (recv_bytes < 0)
                    {
                        perror("recv");
                    }
                    if (recv_bytes == 0)
                    {
                        printf("close by peer\n");
                        close(r_events[i].data.fd);
                    }
                    if (recv_bytes > 0)
                    {
                        printf("%s\n", recv_buf);
                        send_bytes = send(r_events[i].data.fd, send_buf, strlen(send_buf), 0);
                        if (send_bytes == -1)
                        {
                            perror("send");
                        }
                    }
                }
            }
        }
    }

    return 0;
}

对比用双循环阻塞服务端:

#define _GNU_SOURCE
#include <stdio.h>
#include <sys/socket.h>
#include <sys/un.h>
#include <arpa/inet.h>
#include <netinet/in.h>
#include <string.h>
#include <unistd.h>

#define SERVER_IP "192.168.142.132"
#define SERVER_PORT 50001

int main()
{
    int server_sockfd, client_sockfd;
    struct sockaddr_in server_sockaddr, client_sockaddr;
    memset(&server_sockaddr, 0, sizeof(server_sockaddr));
    memset(&client_sockaddr, 0, sizeof(client_sockaddr));
    socklen_t client_sockaddr_len = sizeof(client_sockaddr);
    ssize_t send_bytes, recv_bytes;
    char send_buf[1024] = "How can I help you today ?";
    char recv_buf[1024] = {0};

    server_sockfd = socket(AF_INET, SOCK_STREAM, 0);
    if (server_sockfd == -1)
    {
        perror("socket");
    }

    int optval = 1;
    setsockopt(server_sockfd, SOL_SOCKET, SO_REUSEADDR, &optval, sizeof(optval));

    server_sockaddr.sin_family = AF_INET;
    inet_pton(AF_INET, SERVER_IP, &server_sockaddr.sin_addr.s_addr);
    server_sockaddr.sin_port = htons(SERVER_PORT);
    if (bind(server_sockfd, (struct sockaddr *)&server_sockaddr, sizeof(server_sockaddr)) == -1)
    {
        perror("bind");
    }

    if (listen(server_sockfd, 16) == -1)
    {
        perror("listen");
    }

    printf("server start...\n");

    while (1)
    {
        client_sockfd = accept(server_sockfd, (struct sockaddr *)&client_sockaddr, &client_sockaddr_len);
        if (client_sockfd == -1)
        {
            perror("accept");
        }
        while (1)
        {

            recv_bytes = recv(client_sockfd, recv_buf, sizeof(recv_buf), 0);
            if (recv_bytes == -1)
            {
                perror("recv");
            }
            else if (recv_bytes == 0)
            {
                printf("closed by peer\n");
                break;
            }
            else
            {
                printf("%s\n", recv_buf);
            }
            send_bytes = send(client_sockfd, send_buf, strlen(send_buf), 0);
            if (send_bytes == -1)
            {
                perror("send");
            }
        }
    }
    close(server_sockfd);
    return 0;
}

测试用客户端:

#define _GNU_SOURCE
#include <stdio.h>
#include <sys/socket.h>
#include <sys/un.h>
#include <arpa/inet.h>
#include <netinet/in.h>
#include <string.h>
#include <unistd.h>
#include <stdlib.h>
#include <time.h>
#include <fcntl.h>
#define SERVER_IP "192.168.142.132"
#define SERVER_PORT 50001
int set_non_blocking(int sockfd)
{
    int flags = fcntl(sockfd, F_GETFL, 0);
    if (flags == -1)
    {
        perror("fcntl(F_GETFL)");
    }

    flags |= O_NONBLOCK;

    if (fcntl(sockfd, F_SETFL, flags) == -1)
    {
        perror("fcntl(F_SETFL)");
    }

    return 0;
}
int main()
{
    int client_sockfd;
    struct sockaddr_in server_sockaddr, client_sockaddr;
    memset(&server_sockaddr, 0, sizeof(server_sockaddr));
    memset(&client_sockaddr, 0, sizeof(client_sockaddr));
    socklen_t client_sockaddr_len = sizeof(client_sockaddr);
    ssize_t send_bytes, recv_bytes;
    char send_buf[1024] = "he$$$llo ser$$$ver !!!";
    char recv_buf[1024] = {0};
    client_sockfd = socket(AF_INET, SOCK_STREAM, 0);
    if (client_sockfd == -1)
    {
        perror("socket");
    }
    inet_pton(AF_INET, SERVER_IP, &server_sockaddr.sin_addr.s_addr);
    server_sockaddr.sin_port = htons(SERVER_PORT);
    server_sockaddr.sin_family = AF_INET;
    if (connect(client_sockfd, (struct sockaddr *)&server_sockaddr, sizeof(server_sockaddr)) == -1)
    {
        perror("connect");
    }
    while (1)
    {
        send_bytes = send(client_sockfd, send_buf, strlen(send_buf), 0);
        if (send_bytes == -1)
        {
            perror("send");
        }
        recv_bytes = recv(client_sockfd, recv_buf, sizeof(recv_buf), 0);
        if (recv_bytes == -1)
        {
            perror("recv");
        }
        printf("%s\n", recv_buf);
        sleep(3);
    }
    close(client_sockfd);
    return 0;
}