基於多線程的EventSelect

單線程的: 異步io通知 WSAEventSelect

單線程上最多隻能接受64個event;

改版多線程的 , 大概思路:

1. 主線程就接受一個監聽套接字, accept返回的socket 仍到全局隊列中

2. 一個分配socket的線程 用於控制 socket 分配到哪個 監聽事件的線程中去 [ dispath thread] ;

3. 創建一個線程池 默認4個線程, 每個線程分別監聽  WSA_MAXIMUM_WAIT_EVENTS - 1 個事件, 第一個事件用於分派線程通知

有新的socket 來了 ; 這樣 4個線程總共可監聽 (WSA_MAXIMUM_WAIT_EVENTS - 1 ) * 4 個socket;

代碼有點亂, 也沒測試過; 純粹一種思路

用了大量的線程同步,其實不太好 , 可優化很多地方;

#include <iostream>
#include <WinSock2.h>
#include <Windows.h>
#include <stdio.h>
#include <process.h>
#include <deque>
#include <vector>
#include <unordered_map>
#pragma comment(lib, "ws2_32.lib")
#define PORT 9988
using namespace std;


//socket對象
typedef struct _SOCK_OBJ
{
    enum {BUFFSIZE = 4096};
    SOCKET sock;
    char *buf;
    _SOCK_OBJ(SOCKET s):sock(s), buf(new char[BUFFSIZE]) {}
    ~_SOCK_OBJ(){
        if(sock != INVALID_SOCKET)
            closesocket(sock);
        if(buf){
            delete [] buf;
        }
    }
}SOCK_OBJ,*PSOCK_OBJ;


//每個線程一個
typedef struct _THREAD_INFO{
    WSAEVENT * event_arr; //所有的事件都在這
    HANDLE thread_handle; //線程handler
    vector<SOCK_OBJ*> sock_obj_arr; //socket對象
    char nCount; // 當前數量
    SRWLOCK lock; //讀寫鎖

    _THREAD_INFO():nCount(0),event_arr(0){
       sock_obj_arr.reserve(WSA_MAXIMUM_WAIT_EVENTS/2);
       event_arr = new WSAEVENT[WSA_MAXIMUM_WAIT_EVENTS];
       event_arr[0] = WSACreateEvent();
       InitializeSRWLock(&lock);
    }

    //沒用到 
    int get_sock_count() {
        int i = 0;
        AcquireSRWLockShared(&lock);
        i  = nCount;
        ReleaseSRWLockShared(&lock);
        return i;
    }
// 觸發一次添加操作, 有socket 來了
    void awake(){
        WSASetEvent(event_arr[0]);
    }

   //是否已經滿了
    bool aviliable(){
        bool ret = false;
       AcquireSRWLockShared(&lock);
        if( nCount < WSA_MAXIMUM_WAIT_EVENTS - 1)
            ret = true;
       ReleaseSRWLockShared(&lock);
        return ret;
    }
    ~_THREAD_INFO(){
        if(event_arr){
            WSACloseEvent(event_arr[0]);
        }
        for(int i =0 ; i < nCount; ++i){
            delete sock_obj_arr[i];
            WSACloseEvent(event_arr[i+1]);
        }
    }

}THREAD_INFO,*PTHREAD_INFO;

CRITICAL_SECTION cs_queue;  // 全局鎖, accept的socket放入gqueue
CONDITION_VARIABLE cond_queue; //同上, 用於線程同步
volatile bool bEnd = false;
deque<SOCKET> gqueue; //socket隊列


deque<SOCK_OBJ*> sock_queue; //socket對象隊列 
SRWLOCK glock; //鎖, 用於工作線程與 dispath thread
unordered_map<DWORD , THREAD_INFO*> gmap; //線程id , 線程信息對象


//重新排一下event_arr 
void rebulid(THREAD_INFO * pinfo , int idx , int total){
    for(int i  = idx; i < total ; ++i)
        pinfo->event_arr[i] = pinfo->event_arr[i+1];
}

//工作線程
unsigned int __stdcall workthread(void *args){
    printf("workthread %ld\n" , GetCurrentThreadId());
    THREAD_INFO * pinfo = (THREAD_INFO*)args;
    HANDLE t ;
    DuplicateHandle(GetCurrentProcess(),GetCurrentThread(),GetCurrentProcess(),&t,0,FALSE,DUPLICATE_SAME_ACCESS);
    pinfo->thread_handle = t;
    int pos = 0 , startindex= 0 ,nindex = 0,len = 0,nevents = 0;
    WSANETWORKEVENTS net_ev;
    while(1){
        nevents = pinfo->nCount+1;
        pos = WSAWaitForMultipleEvents(nevents,pinfo->event_arr,FALSE,WSA_INFINITE,FALSE);
        startindex = pos - WSA_WAIT_EVENT_0;
        for(int i = startindex ; i < nevents; ++i){
            nindex = WSAWaitForMultipleEvents(1,pinfo->event_arr+i,TRUE,0,FALSE);
            if(WSA_WAIT_FAILED == nindex || WSA_WAIT_TIMEOUT == nindex){
                continue;
            }
            else if(0 == i){

                // 有新的socket 添加, 加入event_arr 與 socket_arr
                SOCK_OBJ * psock = NULL;
                AcquireSRWLockExclusive(&glock);
                psock = sock_queue.front();
                sock_queue.pop_front();
                ReleaseSRWLockExclusive(&glock);

                if(pinfo->nCount < WSA_MAXIMUM_WAIT_EVENTS -1){
                    pinfo->sock_obj_arr.push_back(psock);
                    WSAEVENT ev = WSACreateEvent();
                    pinfo->event_arr[pinfo->nCount] =ev ;
                    WSAEventSelect(psock->sock,ev, FD_READ|FD_CLOSE);
                    ++pinfo->nCount;
                } else{
                    AcquireSRWLockExclusive(&glock);
                    sock_queue.push_back(psock);
                    ReleaseSRWLockExclusive(&glock);
                }

                //重置
                WSAResetEvent(pinfo->event_arr[0]);
            }
            else{
                WSAEnumNetworkEvents(pinfo->sock_obj_arr[i+1]->sock,pinfo->event_arr[i],&net_ev);

                // 讀事件 ,隨意處理, 這裏還可以加入線程池來處理 recv , send
                if(net_ev.lNetworkEvents & FD_READ){
                    if( net_ev.iErrorCode[FD_READ_BIT] != 0){
                        //出錯了
                    }
                    len = recv(pinfo->sock_obj_arr[i+1]->sock,pinfo->sock_obj_arr[i+1]->buf,SOCK_OBJ::BUFFSIZE-1,0);
                    pinfo->sock_obj_arr[i+1]->buf[len] = 0;
                    printf("sock:%ld , buf:%s\n",pinfo->sock_obj_arr[i+1]->sock,pinfo->sock_obj_arr[i+1]->buf);
                }

                    //關閉事件
                if (net_ev.lNetworkEvents & FD_CLOSE){
                    if(net_ev.iErrorCode[FD_CLOSE_BIT] != 0){
                        //出錯了
                    }
                    delete pinfo->sock_obj_arr[i+1];
                    WSACloseEvent(pinfo->event_arr[i]);
                    AcquireSRWLockExclusive(&pinfo->lock);
                    --pinfo->nCount;
                    rebulid(pinfo,i,pinfo->nCount);
                    ReleaseSRWLockExclusive(&pinfo->lock);
                }
            }

        }
    }


    return 0;
}


//創建一些線程，用於等待除了監聽socket以外的 事件
void create_thread_pool(unsigned poolsize = 4){
    for (int i = 0; i< poolsize; ++i) {
        unsigned int tid = 0;
        THREAD_INFO * p = new THREAD_INFO;
        _beginthreadex(0,0,workthread,p,0,&tid);
        gmap.insert(make_pair(tid,p));
    }
}


//用於分配的線程, 用條件變量來通知 一個新的client來了
unsigned int __stdcall dispath_thread(void * arg){
    puts("process sock thread start");
    SOCKET sock = INVALID_SOCKET;
    BOOL awaked = FALSE;
    InitializeSRWLock(&glock);
    while(1){
        EnterCriticalSection(&cs_queue);
        while(!bEnd && gqueue.empty()){
            SleepConditionVariableCS(&cond_queue,&cs_queue,INFINITE);
        }
        if(bEnd){
            LeaveCriticalSection(&cs_queue);
            break;
        }

        //取出client socket 
        sock = gqueue.front();
        gqueue.pop_front();
        LeaveCriticalSection(&cs_queue);

        //創建一個socket對象
        PSOCK_OBJ pobj = new SOCK_OBJ(sock);
        AcquireSRWLockExclusive(&glock);
        sock_queue.push_back(pobj);
        ReleaseSRWLockExclusive(&glock);


        // 從線程信息對象中去找 哪個線程還能存放,就放進去
begin_awake:
        awaked = FALSE;
        auto iter = gmap.begin();
        for(;iter != gmap.end() ; ++iter){
            if( iter->second->aviliable() ){
                iter->second->awake();
                awaked = TRUE;
                break;
            }
        }
        if(!awaked){
            create_thread_pool();
            goto begin_awake;
        }
    }

    puts("dispath done!");
    return 0;
}

int main(int argc, char *argv[])
{

    InitializeCriticalSectionAndSpinCount(&cs_queue,4000);
    InitializeConditionVariable(&cond_queue);
    WSADATA wsadata;
    WSAStartup(MAKEWORD(2,2) , &wsadata);
    SOCKET listenfd = socket(AF_INET, SOCK_STREAM,0);
    SOCKADDR_IN serv_addr = {0};
    serv_addr.sin_addr.S_un.S_addr = INADDR_ANY;
    serv_addr.sin_family = AF_INET;
    serv_addr.sin_port = htons(PORT);
    bind(listenfd,(SOCKADDR*)&serv_addr, sizeof(serv_addr));
    listen(listenfd,10);
    WSAEVENT listen_ev = WSACreateEvent();
    WSAEventSelect(listenfd,listen_ev,FD_ACCEPT);
    int pos = 0;
    WSANETWORKEVENTS net_ev;
    int clientfd = 0;
    while(1){
        pos = WSAWaitForMultipleEvents(1,&listen_ev,FALSE,WSA_INFINITE,FALSE);
        if( WSA_WAIT_FAILED == pos || WSA_WAIT_TIMEOUT == pos){
            printf("WSAWaitForMultipleEvents failed !\n");
            continue;
        }
        WSAEnumNetworkEvents(listenfd,listen_ev,&net_ev);
        if( net_ev.lNetworkEvents & FD_ACCEPT){
            if (net_ev.iErrorCode[FD_ACCEPT_BIT] != 0 )  {
                printf("accept events failed! error:%ld\n" , GetLastError());
                continue;
            }
            clientfd = accept(listenfd,NULL,NULL);
            EnterCriticalSection(&cs_queue);
            gqueue.push_back(clientfd);
            LeaveCriticalSection(&cs_queue);
            WakeConditionVariable(&cond_queue);
        }
    }

}