簡約而不簡單epoll之EPOLLOUT

本篇主要分析epoll邊緣觸發，通過模擬各種場景，來介紹EPOLLOUT，不涉及epoll底層源碼實現。

一、前提

epoll_wait返回的條件

1、等待時間到期

2、發生信號事件，例如ctrl+c

3、The associated file is available for read(2) operations，如果註冊了EPOLLIN， socket接收緩衝區，有新的數據到來

4、The associated file is available for write(2) operations，如果註冊了EPOLLOUT， socket發送緩衝區可寫時

前三種場景比較容易理解，但是第4種場景卻需要深入研究一下，通過man epoll_ctl手冊，可以得到官方對於EPOLLOUT解釋，就是上面英文，那麼在寫代碼的時候我們應該如何考慮呢，EPOLLOUT呢？。

二、EPOLLIN|EPOLLOUT同時註冊到epoll事件模型

結論1：將EPOLLET|EPOLLIN|EPOLLOUT 註冊到epoll中， 然後調用epoll_wait會立刻返回，並且只有EPOLLOUT事件，這個屬於第一次EPOLLOUT事件，原因是緩衝區可寫。

模擬場景1： client 調用connect後，直接睡眠10s，不發送也不接受任何數據，10s後直接退出。觀察server端會有EPOLLOUT打印

代碼1：

服務端代碼：

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <strings.h>
#include <unistd.h>
#include <fcntl.h>
#include <errno.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <sys/socket.h>
#include <sys/epoll.h>


#define SERVPORT 9527
#define MAXBUF 8192
#define MAXFDS 5000
#define EVENTSIZE 100

char sndMsg[MAXBUF] = {0};

int setnonblocking(int fd)
{
    int opts;
    if( (opts = fcntl(fd, F_GETFL, 0)) == -1) {
        perror("fcntl");
        return -1;
    }

    opts = opts | O_NONBLOCK;
    if( (opts = fcntl(fd, F_SETFL, opts)) == -1) {
        perror("fcntl");
        return -1;
    }

    return 0;
}

int main(void)
{
    char buf[MAXBUF];
    int len, n;

    struct sockaddr_in servaddr;
    int sockfd, listenfd, epollfd, nfds;

    struct epoll_event ev;
    struct epoll_event events[EVENTSIZE];

    bzero(&servaddr, sizeof(servaddr));
    servaddr.sin_family = AF_INET;
    servaddr.sin_addr.s_addr = htonl(INADDR_ANY);
    servaddr.sin_port = htons(SERVPORT);

    if( (epollfd = epoll_create(MAXFDS)) == -1) {
        perror("epoll");
        exit(1);
    }

    if( (listenfd = socket(AF_INET, SOCK_STREAM, 0)) == -1) {
        perror("socket");
        exit(1);
    }

    if(setnonblocking(listenfd) == -1){
        perror("setnonblocking");
        exit(1);
    }

    if(bind(listenfd, (struct sockaddr *)&servaddr, sizeof(servaddr)) == -1) {
        perror("bind");
        exit(1);
    }

    if(listen(listenfd, 10) == -1) {
        perror("listen");
        exit(1);
    }

    // listen fd只註冊EPOLLIN事件， EPOLLOUT不需要註冊
    ev.events = EPOLLIN | EPOLLET;
    ev.data.fd = listenfd;
    if(epoll_ctl(epollfd, EPOLL_CTL_ADD, listenfd, &ev) == -1) {
        perror("epoll_ctl");
        exit(1);
    }

    for( ; ; ) {
        if( (nfds = epoll_wait(epollfd, events, EVENTSIZE, -1)) == -1) {
            perror("epoll_wait");
            exit(1);
        }

        for(n = 0; n < nfds; n++) {
            if(events[n].data.fd == listenfd) {
                while( (sockfd = accept(listenfd, (struct sockaddr *)NULL, NULL)) > 0) {
                    if(setnonblocking(sockfd) == -1) {
                        perror("setnonblocking");
                        exit(1);
                    }
                    //新的fd： 採用邊緣觸發且註冊IN、OUT事件
                    ev.events = EPOLLIN | EPOLLOUT | EPOLLET;
                    ev.data.fd = sockfd;
                    if(epoll_ctl(epollfd, EPOLL_CTL_ADD, sockfd, &ev) == -1) {
                        perror("epoll_ctl");
                        exit(1);
                    } else {
                        printf("new socketfd = %d register epoll success\n", sockfd);
                    }
                }
                continue;
            }
            printf("Events = 0x%x\n", events[n].events);
            if (events[n].events & (EPOLLIN | EPOLLOUT) == (EPOLLIN | EPOLLOUT)) {
                printf(">> EPOLLIN And EPOLLOUT event， socketfd = %d\n", events[n].data.fd);
            }
            else if (events[n].events & EPOLLIN) {
                printf(">> Only EPOLLIN event， socketfd = %d\n", events[n].data.fd);
            }
            else if (events[n].events & EPOLLOUT) {
                printf(">> Only EPOLLOUT， socketfd = %d\n", events[n].data.fd);
            }
        }
    }
}

客戶端代碼：

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <strings.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <sys/socket.h>

#define SERVPORT 9527
#define MAXBUF 1024

int main(void)
{
    char buf[MAXBUF];
    struct sockaddr_in servaddr;
    int fd;
    int n, len;

    bzero(&servaddr, sizeof(servaddr));
    servaddr.sin_family = AF_INET;
    servaddr.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
    servaddr.sin_port = htons(SERVPORT);

    if ((fd = socket(AF_INET, SOCK_STREAM, 0)) == -1) {
        perror("socket");
        exit(1);
    }

    if (connect(fd, (struct sockaddr *)&servaddr, sizeof(servaddr)) == -1) {
        perror("connect");
        exit(1);
    }
    printf("Connection ok, Send data after sleep 10s.\n");
    sleep(10);
    printf(">> bye bye\n");
    close(fd);// 會出發server端 EPOLLIN和EPOLLOUT
    return 0;
}

輸出結果：

結論2：在註冊的時候，將EPOLLET|EPOLLIN|EPOLLOUT註冊到epoll中，然後客戶端client發送數據，server就會觸發EPOLLIN|EPOLLOUT， 原因還是緩衝區是可以寫的。

模擬場景2：建立連接後clien立即發送數據，每次間隔5秒，server端不調用recv函數也不調用send函數。

輸出結果：執行了兩次，兩次基本相同，只不過是第二次多出現一個Only EPOLLOUT。

代碼：

服務端-與結論1中代碼相同，此處不在羅列

客戶端：

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <strings.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <sys/socket.h>

#define SERVPORT 9527
#define MAXBUF 1024

int main(void)
{
    char buf[MAXBUF];
    struct sockaddr_in servaddr;
    int fd;
    int n, len;

    bzero(&servaddr, sizeof(servaddr));
    servaddr.sin_family = AF_INET;
    servaddr.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
    servaddr.sin_port = htons(SERVPORT);

    if ((fd = socket(AF_INET, SOCK_STREAM, 0)) == -1)
    {
        perror("socket");
        exit(1);
    }

    if (connect(fd, (struct sockaddr *)&servaddr, sizeof(servaddr)) == -1)
    {
        perror("connect");
        exit(1);
    }
    printf(">> Send \"helloworld-1\"\n");
    send(fd, "helloworld-1", sizeof("helloworld-1") - 1, 0);
    sleep(5);
    printf(">> Send \"helloworld-2\"\n");
    send(fd, "helloworld-2", sizeof("helloworld-2") - 1, 0);
    printf("Will colse after sleep 8s.\n");
    sleep(8);
    close(fd);
    return 0;
}

結論3：在註冊的時候，將EPOLLET|EPOLLIN|EPOLLOUT註冊到epoll中， client不發送也不接收數據，sever一直
發送數據，直到將socket發送緩衝區打滿， 此時client發送數據到server端， 這時server只會觸發EPOLLIN事件
沒有EPOLLOUT，原因是發送緩衝區是滿的，不可寫，所以不會觸發EPOLLOUT。

模擬場景：設置server端socket 發送緩衝區是4096（4k）， 設置client端接收rcvbuf緩衝區3072（3k），客戶端不接收數據即不調用recv函數，服務端一直髮送數據，最終會把服務端發送緩衝區打滿，我們這裏改變緩衝區大小，只是爲了更快打滿緩衝區。

輸出數據：

代碼：

服務端代碼：

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <strings.h>
#include <unistd.h>
#include <fcntl.h>
#include <errno.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <sys/socket.h>
#include <sys/epoll.h>


#define SERVPORT 9527
#define MAXBUF 1024*1024  //1MB
#define MAXFDS 5000
#define EVENTSIZE 100

int setnonblocking(int fd)
{
    int opts;
    if( (opts = fcntl(fd, F_GETFL, 0)) == -1)
    {
        perror("fcntl");
        return -1;
    }

    opts = opts | O_NONBLOCK;
    if( (opts = fcntl(fd, F_SETFL, opts)) == -1)
    {
        perror("fcntl");
        return -1;
    }

    return 0;
}

int main(void)
{
    int len = 0, n = 0, once = 0;
    int  hasSend;
    char buf[MAXBUF] = {0};
    char sndMsg[MAXBUF] = {1};

    struct sockaddr_in servaddr;
    int sockfd, listenfd, epollfd, nfds;

    struct epoll_event ev;
    struct epoll_event events[EVENTSIZE];

    bzero(&servaddr, sizeof(servaddr));
    servaddr.sin_family = AF_INET;
    servaddr.sin_addr.s_addr = htonl(INADDR_ANY);
    servaddr.sin_port = htons(SERVPORT);

    if( (epollfd = epoll_create(MAXFDS)) == -1) {
        perror("epoll");
        exit(1);
    }

    if( (listenfd = socket(AF_INET, SOCK_STREAM, 0)) == -1) {
        perror("socket");
        exit(1);
    }

    if(setnonblocking(listenfd) == -1) {
        perror("setnonblocking");
        exit(1);
    }

    if(bind(listenfd, (struct sockaddr *)&servaddr, sizeof(servaddr)) == -1) {
        perror("bind");
        exit(1);
    }

    if(listen(listenfd, 10) == -1) {
        perror("listen");
        exit(1);
    }


    ev.events = EPOLLIN | EPOLLET;
    ev.data.fd = listenfd;
    if(epoll_ctl(epollfd, EPOLL_CTL_ADD, listenfd, &ev) == -1) {
        perror("epoll_ctl");
        exit(1);
    }

    for( ; ; ) {
        if( (nfds = epoll_wait(epollfd, events, EVENTSIZE, -1)) == -1) {
            perror("epoll_wait");
            exit(1);
        }

        for(n = 0; n < nfds; n++) {
            if(events[n].data.fd == listenfd) {
                while( (sockfd = accept(listenfd, (struct sockaddr *)NULL, NULL)) > 0) {
                    if(setnonblocking(sockfd) == -1) {
                        perror("setnonblocking");
                        exit(1);
                    }
                    ev.events = EPOLLIN | EPOLLOUT | EPOLLET;
                    ev.data.fd = sockfd;
                    int nSendBuf = 4096; //上層應用設置成4096 但是底層內核實際是4096*2 = 8192
                    socklen_t opt_len = sizeof(int);
                    int ret = setsockopt(sockfd, SOL_SOCKET, SO_SNDBUF, &nSendBuf, sizeof(int));
                    if (ret == -1) {
                        perror("setsockopt");
                    }
                    getsockopt(sockfd, SOL_SOCKET, SO_SNDBUF, &nSendBuf, &opt_len);
                    printf("SndBuf-new  = %d\n", nSendBuf);
                    if(epoll_ctl(epollfd, EPOLL_CTL_ADD, sockfd, &ev) == -1)
                    {
                        perror("epoll_ctl");
                        exit(1);
                    } else {
                        printf("new socketfd = %d register epoll success\n", sockfd);
                    }
                }
                continue;
            }

            printf("Events = 0x%x\n", events[n].events);
            if (events[n].events & EPOLLIN && events[n].events & EPOLLOUT) {
                printf(">> EPOLLIN And EPOLLOUT event， socketfd = %d\n", events[n].data.fd);
                len = recv(events[n].data.fd, buf, MAXBUF, 0);
                len = send(events[n].data.fd, sndMsg + hasSend, MAXBUF-hasSend, 0);
                if (len > 0) {// 說明發送成功
                    hasSend += len;
                    once = 0;
                    printf("EPOLLIN | EPOLLOUT ==》 Send ok， length = %d.\n", len);
                } else if (len == -1) {
                    if (errno == EAGAIN) {//說明發送緩衝區已經滿
                        printf("EPOLLIN | EPOLLOUT ==》 Socket SndBuf is fulled. \n");
                    }
                } else {//出現錯誤
                    perror("EPOLLIN | EPOLLOUT ==》 Send failed.\n");
                    exit(-1);
                }
            }
            else if (events[n].events & EPOLLIN) {
                printf(">> Only EPOLLIN event， socketfd = %d\n", events[n].data.fd);
            }
            else if (events[n].events & EPOLLOUT) {
                printf(">> Only EPOLLOUT， socketfd = %d\n", events[n].data.fd);

                while (1) {
                    len = send(events[n].data.fd, sndMsg + hasSend, MAXBUF-hasSend, 0);
                    if (len > 0) {// 說明發送成功
                        hasSend += len;
                        once = 0;
                        printf("Send ok， length = %d.\n", len);
                    } else if (len == -1) {
                        if (errno == EAGAIN) {//說明發送緩衝區已經滿
                            printf(" EPOLLOUT ==》 Socket SndBuf is fulled. \n");
                            break;
                        }
                    } else {//出現錯誤
                        perror(" EPOLLOUT ==》 Send failed.\n");
                        exit(-1);
                    }
                }
            }
        }
    }
}

客戶端代碼：

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <strings.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <sys/socket.h>

#define SERVPORT 9527
#define MAXBUF 1024

int main(void)
{
    char buf[MAXBUF];
    struct sockaddr_in servaddr;
    int fd;
    int n, len;

    bzero(&servaddr, sizeof(servaddr));
    servaddr.sin_family = AF_INET;
    servaddr.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
    servaddr.sin_port = htons(SERVPORT);

    if ((fd = socket(AF_INET, SOCK_STREAM, 0)) == -1)
    {
        perror("socket");
        exit(1);
    }

    int nSendBuf = 0;
    socklen_t opt_len = sizeof(int);
    nSendBuf = 3072;
    int ret = setsockopt(fd, SOL_SOCKET, SO_RCVBUF, &nSendBuf, sizeof(int));
    if (ret == -1) {
        perror("setsockopt");
    }
    getsockopt(fd, SOL_SOCKET, SO_RCVBUF, &nSendBuf, &opt_len);
    printf("Client recvbuf len = %d\n", nSendBuf);
    if (connect(fd, (struct sockaddr *)&servaddr, sizeof(servaddr)) == -1)
    {
        perror("connect");
        exit(1);
    }
    printf("Connection ok, sleep 30s\n");
    sleep(30); //睡眠60秒保證  服務端緩衝區被打滿
    printf("Send msg==>""hello server!! do you sndbuf is fulled?\n");
    send(fd, "hello server!! do you sndbuf is fulled?", strlen("hello server!! do you sndbuf is fulled?"), 0);
    sleep(10);
    send(fd, "hello server!! do you sndbuf is fulled?", strlen("hello server!! do you sndbuf is fulled?"), 0);
    sleep(10);
    close(fd);

    return 0;
}

結論4：在註冊的時候，將EPOLLET|EPOLLIN|EPOLLOUT註冊到epoll中， client不發送也不接收數據，sever一直髮送數據，直到將socket 發送緩衝區打滿， 此時client 開始連續接收n次數據（不發送數據）， 此時server又會產生EPOLLOUT，因爲發送緩衝區變成可寫（由滿->不滿），由於服務端要發送1MB數據，客戶端每次只讀取256字節，所有服務端發送緩衝區狀態變化是：滿->不滿->滿（不可寫->可寫->不可寫）。

輸出數據：

代碼：

服務端，與結論3中服務端代碼一樣

客戶端代碼：

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <strings.h>
#include <unistd.h>
#include <fcntl.h>
#include <errno.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <sys/socket.h>
#include <sys/epoll.h>

#define SERVPORT 9527
#define MAXBUF 1024

int setnonblocking(int fd)
{
    int opts;
    if( (opts = fcntl(fd, F_GETFL, 0)) == -1)
    {
        perror("fcntl");
        return -1;
    }

    opts = opts | O_NONBLOCK;
    if( (opts = fcntl(fd, F_SETFL, opts)) == -1)
    {
        perror("fcntl");
        return -1;
    }

    return 0;
}

int main(void)
{
    char buf[MAXBUF];
    struct sockaddr_in servaddr;
    int fd;
    int n, len;

    bzero(&servaddr, sizeof(servaddr));
    servaddr.sin_family = AF_INET;
    servaddr.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
    servaddr.sin_port = htons(SERVPORT);

    if ((fd = socket(AF_INET, SOCK_STREAM, 0)) == -1)
    {
        perror("socket");
        exit(1);
    }

    int nSendBuf = 0;
    socklen_t opt_len = sizeof(int);
    nSendBuf = 3072;
    int ret = setsockopt(fd, SOL_SOCKET, SO_RCVBUF, &nSendBuf, sizeof(int));
    if (ret == -1) {
        perror("setsockopt");
    }

    getsockopt(fd, SOL_SOCKET, SO_RCVBUF, &nSendBuf, &opt_len);
    printf("Client recvbuf len = %d\n", nSendBuf);

    if (connect(fd, (struct sockaddr *)&servaddr, sizeof(servaddr)) == -1)
    {
        perror("connect");
        exit(1);
    }
    setnonblocking(fd); //設置成非阻塞
    printf("Connection ok, sleep 10s\n");
    sleep(10); //睡眠10秒保證  服務端緩衝區被打滿

    while(1) {
        len = recv(fd, buf, 256, 0);// 由於服務端發送的數據是1MB所以需要多次調用
        if (len > 0) {
            n += len;
        } else if (len == -1 && errno == EAGAIN) {
            printf("EAGAIN, Has recv n = %d\n", n);
            sleep(3);
        } else {
            printf("Recv failed.\n");
            exit(-1);
        }
    }
    close(fd);
    return 0;
}

結論5：在註冊的時候，將EPOLLET|EPOLLIN|EPOLLOUT註冊到epoll中， client不發送也不接收數據，sever一直髮送數據，直到將socket 發送緩衝區打滿，打滿之後server再也不會發送數據。 此時client 開始連續接收n次數據（不發送數據）， 此時server又會產生EPOLLOUT，因爲發送緩衝區變成可寫（由滿->不滿）。

輸出數據：

代碼：

服務端代碼：

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <strings.h>
#include <unistd.h>
#include <fcntl.h>
#include <errno.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <sys/socket.h>
#include <sys/epoll.h>


#define SERVPORT 9527
#define MAXBUF 1024*1024  //1MB
#define MAXFDS 5000
#define EVENTSIZE 100

int setnonblocking(int fd)
{
    int opts;
    if( (opts = fcntl(fd, F_GETFL, 0)) == -1)
    {
        perror("fcntl");
        return -1;
    }

    opts = opts | O_NONBLOCK;
    if( (opts = fcntl(fd, F_SETFL, opts)) == -1)
    {
        perror("fcntl");
        return -1;
    }

    return 0;
}

int main(void)
{
    int len = 0, n = 0, once = 0;
    int  hasSend;
    char buf[MAXBUF] = {0};
    char sndMsg[MAXBUF] = {1};

    struct sockaddr_in servaddr;
    int sockfd, listenfd, epollfd, nfds;

    struct epoll_event ev;
    struct epoll_event events[EVENTSIZE];

    bzero(&servaddr, sizeof(servaddr));
    servaddr.sin_family = AF_INET;
    servaddr.sin_addr.s_addr = htonl(INADDR_ANY);
    servaddr.sin_port = htons(SERVPORT);

    if( (epollfd = epoll_create(MAXFDS)) == -1) {
        perror("epoll");
        exit(1);
    }

    if( (listenfd = socket(AF_INET, SOCK_STREAM, 0)) == -1) {
        perror("socket");
        exit(1);
    }

    if(setnonblocking(listenfd) == -1) {
        perror("setnonblocking");
        exit(1);
    }

    if(bind(listenfd, (struct sockaddr *)&servaddr, sizeof(servaddr)) == -1) {
        perror("bind");
        exit(1);
    }

    if(listen(listenfd, 10) == -1) {
        perror("listen");
        exit(1);
    }


    ev.events = EPOLLIN | EPOLLET;
    ev.data.fd = listenfd;
    if(epoll_ctl(epollfd, EPOLL_CTL_ADD, listenfd, &ev) == -1) {
        perror("epoll_ctl");
        exit(1);
    }

    for( ; ; ) {
        if( (nfds = epoll_wait(epollfd, events, EVENTSIZE, -1)) == -1) {
            perror("epoll_wait");
            exit(1);
        }

        for(n = 0; n < nfds; n++) {
            if(events[n].data.fd == listenfd) {
                while( (sockfd = accept(listenfd, (struct sockaddr *)NULL, NULL)) > 0) {
                    if(setnonblocking(sockfd) == -1) {
                        perror("setnonblocking");
                        exit(1);
                    }
                    ev.events = EPOLLIN | EPOLLOUT | EPOLLET;
                    ev.data.fd = sockfd;
                    int nSendBuf = 4096; //上層應用設置成4096 但是底層內核實際是4096*2 = 8192
                    socklen_t opt_len = sizeof(int);
                    int ret = setsockopt(sockfd, SOL_SOCKET, SO_SNDBUF, &nSendBuf, sizeof(int));
                    if (ret == -1) {
                        perror("setsockopt");
                    }
                    getsockopt(sockfd, SOL_SOCKET, SO_SNDBUF, &nSendBuf, &opt_len);
                    printf("SndBuf-new  = %d\n", nSendBuf);
                    if(epoll_ctl(epollfd, EPOLL_CTL_ADD, sockfd, &ev) == -1)
                    {
                        perror("epoll_ctl");
                        exit(1);
                    } else {
                        printf("new socketfd = %d register epoll success\n", sockfd);
                    }
                }
                continue;
            }

            printf("Events = 0x%x\n", events[n].events);
            if (events[n].events & EPOLLIN && events[n].events & EPOLLOUT) {
                printf(">> EPOLLIN And EPOLLOUT event， socketfd = %d\n", events[n].data.fd);
                len = recv(events[n].data.fd, buf, MAXBUF, 0);
                len = send(events[n].data.fd, sndMsg + hasSend, MAXBUF-hasSend, 0);
                if (len > 0) {// 說明發送成功
                    hasSend += len;
                    once = 0;
                    printf("EPOLLIN | EPOLLOUT ==》 Send ok， length = %d.\n", len);
                } else if (len == -1) {
                    if (errno == EAGAIN) {//說明發送緩衝區已經滿
                        printf("EPOLLIN | EPOLLOUT ==》 Socket SndBuf is fulled. \n");
                    }
                } else {//出現錯誤
                    perror("EPOLLIN | EPOLLOUT ==》 Send failed.\n");
                    exit(-1);
                }
            }
            else if (events[n].events & EPOLLIN) {
                printf(">> Only EPOLLIN event， socketfd = %d\n", events[n].data.fd);
            }
            else if (events[n].events & EPOLLOUT) {
                printf(">> Only EPOLLOUT， socketfd = %d\n", events[n].data.fd);

                while (!once) {
                    len = send(events[n].data.fd, sndMsg + hasSend, MAXBUF-hasSend, 0);
                    if (len > 0) {// 說明發送成功
                        hasSend += len;
                        printf("Send ok， length = %d.\n", len);
                    } else if (len == -1) {
                        if (errno == EAGAIN) {//說明發送緩衝區已經滿
                            printf(" EPOLLOUT ==》 Socket SndBuf is fulled. \n");
                            once = 1;// 以後再也不會發送數據了,保證緩衝區不會再滿
                            break;
                        }
                    } else {//出現錯誤
                        perror(" EPOLLOUT ==》 Send failed.\n");
                        exit(-1);
                    }
                }
            }
        }
    }
}

客戶端代碼：

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <strings.h>
#include <unistd.h>
#include <fcntl.h>
#include <errno.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <sys/socket.h>
#include <sys/epoll.h>

#define SERVPORT 9527
#define MAXBUF 4096

int setnonblocking(int fd)
{
    int opts;
    if( (opts = fcntl(fd, F_GETFL, 0)) == -1)
    {
        perror("fcntl");
        return -1;
    }

    opts = opts | O_NONBLOCK;
    if( (opts = fcntl(fd, F_SETFL, opts)) == -1)
    {
        perror("fcntl");
        return -1;
    }

    return 0;
}

int main(void)
{
    char buf[MAXBUF];
    struct sockaddr_in servaddr;
    int fd;
    int n, len;

    bzero(&servaddr, sizeof(servaddr));
    servaddr.sin_family = AF_INET;
    servaddr.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
    servaddr.sin_port = htons(SERVPORT);

    if ((fd = socket(AF_INET, SOCK_STREAM, 0)) == -1)
    {
        perror("socket");
        exit(1);
    }

    int nSendBuf = 0;
    socklen_t opt_len = sizeof(int);
    nSendBuf = 3072;
    int ret = setsockopt(fd, SOL_SOCKET, SO_RCVBUF, &nSendBuf, sizeof(int));
    if (ret == -1) {
        perror("setsockopt");
    }

    getsockopt(fd, SOL_SOCKET, SO_RCVBUF, &nSendBuf, &opt_len);
    printf("Client recvbuf len = %d\n", nSendBuf);

    if (connect(fd, (struct sockaddr *)&servaddr, sizeof(servaddr)) == -1)
    {
        perror("connect");
        exit(1);
    }
    setnonblocking(fd); //設置成非阻塞
    printf("Connection ok, sleep 10s\n");
    sleep(10); //睡眠10秒保證  服務端緩衝區被打滿
    int i = 0;
    while(1) {
        len = recv(fd, buf, MAXBUF, 0);
        if (len > 0) {
            n += len;
            printf("第%d次，recv len = %d\n", ++i, len);
            sleep(10);
        } else if (len == -1 && errno == EAGAIN) {
            printf("EAGAIN\n", n);
            sleep(20);
            break;//默認讀取完畢
        } else {
            printf("Recv failed.\n");
            exit(-1);
        }
    }
    printf("close\n");
    close(fd);
    return 0;
}

三、建議

1、對於服務端listen socket不需要將EPOLLOUT註冊到epoll事件模型中。因爲listen socket只是負責接收數據（接收客戶端建立連接請求），不會發送數據，所以不需要註冊時EPOLLOUT。

2、按需註冊EPOLLOUT。當我們調用send接口時，如果返回的-1且errno=EAGAIN時，再註冊EPOLLOUT，後續send發送成功後，再將EPOLLOUT從epoll事件模型中移除，這就是按需註冊EPOLLOUT。當然我們也可以不用移除，只不過需要判斷是否真的有數據需要發送。大名鼎鼎的nginx的做法是：發送完成後會將發送的回調函數設置成一個空函數（這個函數只是定義裏面什麼都沒有做）。nginx爲什麼不移除呢？因爲反覆添加、移除EPOLLOUT性能不友好，總是在用戶層和內核層來回切換。

下一篇介紹，EPOLLRDHUP。