關於AcceptEx

最基礎的IOCP例子, 沒有使用擴展函數AcceptEx:   IOCP模型

* 關於iocp的核心就一點: 

GetQueuedCompletionStatus 將攜帶返回2個重要的參數, 一個lpCompletionKey, 一個lpOverlapped.

lpCompletionKey : 是 CreateIoCompletionPort((HANDLE)clientSocket , hIOCP,(ULONG_PTR)自定義的結構,0); 跟

iocp綁定的一個自定義參數;

lpOverlapped : 是傳遞給 WSASend /  WSARecv 的參數;

這2個參數最終會被GetQueuedCompletionStatus 攜帶回來.

 

同樣的 , AcceptEx 也要傳遞一個Overlapped結構,現在問題來了,如果只調用了AcceptEx ,

GetQueuedCompletionStatus  是不會返回的, 因爲只有跟 iocp 關聯（CreateIoCompletionPort）的HANDLE / SOCKET 纔會

被觸發, 因此只需要把 監聽套接字 跟iocp 關聯即可;

   

 

下面代碼使用了AccpetEx 和一個用於獲取地址的擴展函數[此函數可以先忽略].

總體來說就是預先分配一些socket , 以及相關的內存塊[到時有客戶進來後,直接使用此內存塊接受數據]; 

不再讓accept系統調用來創建socket了. 

所有需要注意的點都寫在註釋裏了.

下面代碼裏沒有使用  CancelIo 之類的函數,如果實際需要直接用 CancelIoEx 來取消無關線程的Overlapped操作,

 

另:在發送數據[WSASend] 完成後 , 需要檢查是否發送完成, 如果沒有發完需要繼續發送.

#include <mswsock.h>
#include <deque>
#include <vector>
#define IO_ACCEPT 1
#define IO_READ  2
#define IO_WRITE 3
#define BUFFSIZE 4096
#define SPINCOUNT 4000
struct Per_IO_Data;
std::deque<Per_IO_Data*> io_pool;
CRITICAL_SECTION io_pool_cs;

LPFN_ACCEPTEX FuncAcceptEx = NULL;
LPFN_GETACCEPTEXSOCKADDRS FuncGetAddr  = NULL;

struct Per_Sock_Data{
    SOCKET sock;
    SOCKADDR_IN addr;
    HANDLE iocp;
    Per_Sock_Data():sock(INVALID_SOCKET), iocp(INVALID_HANDLE_VALUE){}
};
struct Per_IO_Data{
    OVERLAPPED ol;
    WSABUF wsabuf;
    char *buf;
    int ioMode; //讀 寫 接受
    SOCKET sAcceptSock; //acceptex 預先創建的socket
    int nTotalBytes;
    int nSendBytes;
    Per_IO_Data(): buf(NULL) ,ioMode(-1) , sAcceptSock(INVALID_SOCKET)
      ,nSendBytes(0), nTotalBytes(0)
    {
        memset(&ol, 0 , sizeof(ol));
    }
    ~Per_IO_Data(){
        if(buf) delete buf;
    }
};

void resetPerIOData(Per_IO_Data * pdata , int mode = -1){
    if(!pdata) return;
    memset(&pdata->ol, 0 ,sizeof(OVERLAPPED));
    if(pdata->buf)
        memset(&pdata->buf,0,sizeof (char) * BUFFSIZE);
    pdata->wsabuf.buf=pdata->buf;
    pdata->ioMode = mode;
    pdata->wsabuf.len = BUFFSIZE;
    pdata->nSendBytes = 0 ;
    pdata->nTotalBytes = 0 ;
    pdata->sAcceptSock = INVALID_SOCKET;
}

Per_IO_Data * getNewPerIOData(int mode = -1){
    Per_IO_Data * pdata = new Per_IO_Data;
    if(!pdata) return NULL;
    pdata->buf = new char[BUFFSIZE];
    pdata->ioMode = mode;
    pdata->wsabuf.buf  = pdata->buf;
    pdata->wsabuf.len = BUFFSIZE;
    return pdata;
}
bool doAccept(SOCKET listenfd){
    if(listenfd == INVALID_SOCKET) return  false;
    SOCKET sock = WSASocket(AF_INET,SOCK_STREAM,0,0,0,WSA_FLAG_OVERLAPPED);
    if(sock == INVALID_SOCKET)
        return false;
    Per_IO_Data *pData = getNewPerIOData(IO_ACCEPT);
    if(!pData) { closesocket(sock); return false;}
    pData->sAcceptSock = sock;
    DWORD nBytes = 0;
    BOOL ret = FuncAcceptEx(listenfd,
                 pData->sAcceptSock,
                 pData->buf,
                 0,                     //唯一這個參數注意一下, 自己找msdn
                 sizeof(SOCKADDR_IN) + 16,
                 sizeof(SOCKADDR_IN) + 16,
                 &nBytes,
                 &pData->ol);
    if(!ret && WSAGetLastError() != ERROR_IO_PENDING){
        closesocket(sock);
        EnterCriticalSection(&io_pool_cs);
        io_pool.push_back(pData);
        LeaveCriticalSection(&io_pool_cs);
        return false;
    }
    return true;
}
void handleIO( Per_IO_Data * pData, Per_Sock_Data * pSock , DWORD nBytesTransfered){
    if(!pData || !pSock) return;
    DWORD flag = 0;
    if(IO_READ == pData->ioMode ){
        memset(&pData->ol, 0 , sizeof(OVERLAPPED)); //清一下OVERLAPPED,雖然對socket沒啥用,還是習慣問題
        pData->ioMode = IO_WRITE; //把這塊數據直接發出去
        pData->wsabuf.len = nBytesTransfered;
        pData->nSendBytes = 0;
        pData->nTotalBytes = nBytesTransfered;
        WSASend(pSock->sock,&pData->wsabuf,1,NULL,flag,&pData->ol,NULL); //複製到發送緩衝區
        //每次創建Per_IO_Data 太浪費了, 因此把之前用過的內存塊存在全局deque中
        //如果太多了deque中太多了 自己想辦法解決去刪除一些,方法很多.
        //如果覺得麻煩,直接 new Per_IO_Data即可
        Per_IO_Data * pNewIOData = NULL;
        EnterCriticalSection(&io_pool_cs);
        if(!io_pool.empty()){
            pNewIOData = io_pool.front();
            io_pool.pop_front();
        }
        LeaveCriticalSection(&io_pool_cs);
        if(NULL == pNewIOData){
            pNewIOData = getNewPerIOData(IO_READ);
        }
        else{
            pNewIOData->ioMode = IO_READ;
        }
        //繼續接受
        flag = 0;
        WSARecv(pSock->sock , &pData->wsabuf,1,NULL,&flag,&pData->ol,NULL);
    }
    else if(IO_WRITE == pData->ioMode){
        
        // 檢查是否發送完畢, 如果沒有發送完畢 , 繼續發送剩餘的數據,
        // 如果發完了就進入 io_pool
        pData->ioMode = IO_WRITE;
        pData->nSendBytes += nBytesTransfered;
        if(pData->nSendBytes < pData->nTotalBytes){
           flag = 0;
          pData->wsabuf.len = pData->nTotalBytes - pData->nSendBytes;
          pData->wsabuf.buf = pData->buf  + pData->nSendBytes;
          WSASend(pSock->sock,&pData->wsabuf,1,NULL,flag,&pData->ol,NULL); //複製到發送緩衝區
        }
        else{
        //複製到發送緩存區後, 將回到這裏,此時爲了節省開銷, 把pData重置一下仍進io_pool中
            resetPerIOData(pData,IO_READ);
            EnterCriticalSection(&io_pool_cs);
            io_pool.push_back(pData);
            LeaveCriticalSection(&io_pool_cs);
        }
    }
    else if(IO_ACCEPT == pData->ioMode){ //核心點

        //會到這裏說明有連接進來了 , 由於listenfd 關聯到了IOCP,
        //因此對應的 key 也將一起傳遞進來 ,也就是這個pSock, pSock->sock就是監聽套接字
        //而這個pData就是在doAccept 中創建的那個內存塊,內部攜帶了預先創建的sAcceptSocket
        //但是這個sAcceptSocket 並沒有繼承監聽socket的屬性
        //因此繼承屬性,比如緩衝區大小
        setsockopt(pData->sAcceptSock, SOL_SOCKET, SO_UPDATE_ACCEPT_CONTEXT,
            (char *)&pSock->sock, sizeof(pSock->sock) );
        //爲此新的socket創建一個key , 用於之後的操作
        Per_Sock_Data * pNewSock = new Per_Sock_Data;
        pNewSock->sock = pData->sAcceptSock;

        //用擴展函數來獲取客戶端的地址 GetAcceptExSockaddrs
        //裏面很多參數都是和 AcceptEx中相同的
        SOCKADDR_IN * localAddr = NULL,  *remoteAddr = NULL;
        int localLen = 0 , remoteLen = 0;
        FuncGetAddr(
                    pData->buf,
                    0,  //此參數注意一下,翻一下msdn , 與AcceptEx那個要注意的參數相同
                    sizeof(SOCKADDR_IN)+16,
                    sizeof(SOCKADDR_IN)+16,
                    (SOCKADDR **)&localAddr,
                    &localLen,
                    (SOCKADDR **)&remoteAddr,
                    &remoteLen
                    );
        //把客戶端的地址複製進去
        memcpy(&pNewSock->addr,remoteAddr,remoteLen);

        //把新的key與iocp關聯;
        //爲了方便,我把iocp放到了與監聽套接字關聯的結構體中, 可以直接做爲全局變量更方便
        CreateIoCompletionPort((HANDLE)pNewSock->sock,pSock->iocp,(ULONG_PTR)pNewSock,0);
        //重置一下,把ioMode修改一下
        resetPerIOData(pData,IO_READ);
        pData->sAcceptSock = INVALID_SOCKET; //防止歧義
        //開始接受數據
        flag = 0;
        WSARecv(pNewSock->sock,&pData->wsabuf,1,NULL,&flag,&pData->ol,NULL);
        //繼續接受一個連接,我在一開始的時候只調用了一次,因此只能接受一個連接
        doAccept(pSock->sock);
    }
    else{
        cout << "???????????? 走錯地方了把" << endl;
    }
}
unsigned iocp_thread(void *arg){
    HANDLE iocp = *(HANDLE*)arg;
    Per_IO_Data * pData = NULL;
    Per_Sock_Data *pSock = NULL;
    DWORD nBytesTrans = 0;
    BOOL ret = FALSE;
    while(1){
        pSock = NULL;
        pData = NULL;
        ret = GetQueuedCompletionStatus(iocp,&nBytesTrans,(PULONG_PTR)&pSock,
                                        (LPOVERLAPPED *)&pData,INFINITE);

        //自己做 ret ==FALSE 的錯誤處理

        if(0 == nBytesTrans && ( IO_READ == pData->ioMode  || IO_WRITE == pData->ioMode)){
            cout << "peer closed :" << inet_ntoa(pSock->addr.sin_addr)  << endl;
            closesocket(pSock->sock);
            delete pSock;
            delete pData;
        }
        else
            handleIO(pData,pSock, nBytesTrans);
    }
    return  0;
}

int main(int argc, char *argv[])
{

    //起手式都一樣
    WSADATA wsadata;
    WSAStartup(MAKEWORD(2,2) , &wsadata);
    SOCKET listenfd = socket(AF_INET, SOCK_STREAM, 0); //自帶overlapped屬性 == wsasock(... , overlapped)
    int openReuse = 1;
    int openReuseLen = sizeof(openReuse);
    setsockopt(listenfd,SOL_SOCKET,SO_REUSEADDR,(char*)&openReuse,openReuseLen);
    struct sockaddr_in serv_addr;
    serv_addr.sin_family = AF_INET;
    serv_addr.sin_port = htons(PORT);
    serv_addr.sin_addr.S_un.S_addr = INADDR_ANY;
    bind(listenfd,(SOCKADDR*)&serv_addr,sizeof(struct sockaddr_in));
    listen(listenfd, SOMAXCONN);
    GUID GuidAcceptEx = WSAID_ACCEPTEX;
    GUID GuidGetAddr = WSAID_GETACCEPTEXSOCKADDRS;
    DWORD dwbytes = 0;


    //2個擴展函數, 沒什麼好說的, 固定模式, 看一邊msdn,照抄即可
    WSAIoctl(listenfd,SIO_GET_EXTENSION_FUNCTION_POINTER,
             &GuidAcceptEx,sizeof(GuidAcceptEx),
             &FuncAcceptEx,sizeof(FuncAcceptEx),
             &dwbytes,0,0);

    WSAIoctl(listenfd,SIO_GET_EXTENSION_FUNCTION_POINTER,
             &GuidGetAddr,sizeof(GuidGetAddr),
             &FuncGetAddr,sizeof(FuncGetAddr),
             &dwbytes,0,0);

    InitializeCriticalSectionAndSpinCount(&io_pool_cs, SPINCOUNT);
    SYSTEM_INFO sysinfo = {0};
    GetSystemInfo(&sysinfo);
    int thread_num = sysinfo.dwNumberOfProcessors * 2 + 2;//線程數
    HANDLE iocp = CreateIoCompletionPort(INVALID_HANDLE_VALUE,NULL,0,thread_num);
    //把監聽sock 關聯到iocp , 至此任何客戶端的connect, 都由iocp來完成,就像其他socket一樣
    Per_Sock_Data * pSock = new Per_Sock_Data; //給監聽套接字綁定一個key
    pSock->iocp = iocp;
    pSock->sock = listenfd;
    memcpy(&pSock->addr,&serv_addr,sizeof(serv_addr));
    CreateIoCompletionPort((HANDLE)listenfd, iocp,(ULONG_PTR)pSock,0);

    //接下來起線程
    vector<HANDLE> thread_handles;
    vector<unsigned int> tids;
    for(int i = 0; i < thread_num ; ++i){
        unsigned tid = 0;
        HANDLE t = (HANDLE)_beginthreadex(0,0,iocp_thread,(void*)&iocp,0 , &tid);
        thread_handles.push_back(t);
        tids.push_back( tid);
    }

    doAccept(listenfd); //只接受一個客戶端 , for(int i = 0 ; i < 10000; ++i) doAccept ...


    //接下來主線程沒事了 , 該創建的都創建了,可以幹其他事了
    while(1){
        Sleep(10000);
        cout << "main 睡覺 : " << io_pool.size() << endl;
    }



    return 0;
}