epoll + 多線程實現併發網絡連接處理


簡介

 

觸發方式

  條件觸發

  邊沿觸發

 

主要的應用接口

Epoll的創建

  根據man手冊介紹, epoll_create(int size) 用來創建一個epoll實例,向內核申請支持size個句柄的資源(存儲)。Size的大小不代表epoll支持的最大句柄個數,而隱射了內核擴展句柄存儲的尺寸,也就是說當後面需要再向epoll中添加句柄遇到存儲不夠的時候,內核會按照size追加分配。在2.6以後的內核中,該值失去了意義,但必須大於0。

epoll_create執行成功,返回一個非負的epoll描述句柄,用來指定該資源,否則返回-1。

例子:

         int epoll_fd = epoll_create(1);

Epoll的控制

         Epoll的控制主要通過epoll_ctl(int epfd, int op, int fd, struct epoll_event *event)完成。控制對象是用戶申請的句柄,即fd;Epfd指定所控制的epoll資源;op指對fd的動作,包括向epoll中添加一個句柄EPOLL_CTL_ADD,刪除一個句柄EPOLL_CTL_DEL,修改epoll對一個存在句柄的監控模式EPOLL_CTL_MOD;event指出需要讓epoll對fd的監控模式(收、發、觸發方式等)。

         epoll_ctl執行成功返回0, 否則返回-1。

在介紹該接口之前,我們先看看內核對epoll的事件類型的定義

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
typedef union epoll_data {
 
          void    *ptr;
          int      fd;
          uint32_t u32;
          uint64_t u64;
 
} epoll_data_t;
 
struct epoll_event {
 
         uint32_t   events;    /* Epoll events */
         epoll_data_t data;      /* User data variable */
 
};

該結構中我們主要看epoll_event。epoll_event->data涵蓋了調用epoll_ctl增加或者修改某指定句柄時寫入的信息,epoll_event->event,則包含了返回事件的位域。

例子:

向epoll中增加句柄

增加新的常規句柄:

?
1
2
3
4
5
6
7
8
9
10
11
12
struct epoll_event ev;
 
if(邊沿觸發)
         ev.events = EPOLIN | EPOLLOUT |EPOLLLET
else
 
條件觸發(默認)
 
         ev.events = EPOLIN | EPOLLOUT
         ev.data = newfd;
 
         epoll_ctl(epoll_fd, EPOLL_CTL_ADD, newfd, &ev);

 增加網絡監聽句柄

?
1
2
3
4
5
6
7
8
9
10
11
12
Struct epoll_event ev;
 
if(邊沿觸發)
         ev.events = EPOLIN | EPOLLLET     (監聽句柄只關心輸入)
else
 
條件觸發(默認)
 
         ev.events = EPOLIN;
         ev.data = newfd;
 
         epoll_ctl(epoll_fd, EPOLL_CTL_ADD, newfd, &ev);

 修改某個句柄的模式

?
1
2
3
4
5
6
struct epoll_event newev;
 
newev.events = NEWMOD;(新的觸發方式可通過該接口修改)
newev.data = oldfd;
 
epoll_ctl(epoll_fd, EPOLL_CTL_MOD, oldfd, &newev);

 刪除某個句柄

?
1
2
3
4
5
struct epoll_event newev;
 
epoll_ctl(epoll_fd,EPOLL_CTL_DEL,oldfd,& newev);
 
注:在2.6.9以前的內核中,當執行EPOLL_CTL_DEL時,event須非空,但在之後,event可空,一般爲了兼容以前的內核版本,我們最好將event非空。

Epoll的監控

         當向epoll中添加若干句柄後,就要進入監控狀態,此時通過系統調用epoll_wait(int epfd, struct epoll_event *events,  int maxevents,  int timeout)完成。epoll_wait在執行的時候,在timeout內,將有動作的句柄的信息填充到event,event和maxevents決定了epoll監控句柄的上限。timeout的單位是微妙級別,當爲-1時,除非內部句柄有動作,否則持續等待。

         epoll_wait執行成功返回有動作的句柄的總數,句柄信息在events中包含;如果在超時timeout內返回零,表示沒有io請求的句柄;否則返回-1。

下面是一個結合網上我做了修整的例子貼出來,簡單總結一下epoll的用處。該例子是一個網絡環回測試例程,服務器的地址默認,請求連接的端口號是11111

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
#include "xs_epoll.h"   /* epoll releated include file */
/* create an epoll 實例 */
int xs_epoll_create(int __c)
{
    XSOCKET __efd__ = -1;
 
    __efd__ = epoll_create(__c);
    if(__efd__ < 0){
        xs_dump_error_str(errno);
        return EXIT_ERROR;
    }
         
    return (__efd__);
}
int xs_epoll_ctl(XSOCKET __efd, int __method, XSOCKET __fd, struct epoll_event* __p)
{
    int __result__ = -1;
 
    __result__ = (epoll_ctl(__efd, __method, __fd, __p) < 0);
    if(__result__ < 0){
        xs_dump_error_str(errno);
        return EXIT_ERROR;
    }
     
    return EXIT_OK;
}
int xs_epoll_wait(XSOCKET __efd, struct epoll_event* __ev, int __c,int __tw)
{
    int __num__ = -1;
 
    __num__ = epoll_wait(__efd, __ev, __c, __tw);
    if(__num__ < 0)
        __num__ = EXIT_ERROR;
 
    return (__num__);
}
int xs_epoll_close(XSOCKET __f)
{
    return xs_release_socket(__f);
}
 
/*
    Add fd to epollfd
    Register the target file descriptor fd on the epoll instance referred to by the file descriptor epfd
    and associate the event with the internal file linked to fd.
*/
int xs_epoll_add(int __epollfd,int __fd,int __mod)
{
    xs_epoll_event __xs_ev__;
     
    __xs_ev__.events = __mod;
    __xs_ev__.data.fd = __fd;
 
    if(xs_epoll_ctl(__epollfd, EPOLL_CTL_ADD, __fd, &__xs_ev__) < 0)
            return EXIT_ERROR; 
 
    return EXIT_OK;
}
 
/* 
    Remove  (deregister) the target file descriptor fd from the epoll instance referred to by epollfd. 
    The event is ignored and can be NULL.
    In kernel versions before 2.6.9, the EPOLL_CTL_DEL operation required a non-NULL pointer in event
    Since Linux 2.6.9, event can be specified as NULL when using EPOLL_CTL_DEL.
*/
int xs_epoll_del(int __epollfd,int __fd)
{
    xs_epoll_event __xs_ev__;
 
    /* We should better set event not NULL */
    __xs_ev__.events = __xs_ev__.events;
    __xs_ev__.data.fd = __fd;
 
    if(xs_epoll_ctl(__epollfd, EPOLL_CTL_DEL, __fd, &__xs_ev__) < 0)
            return EXIT_ERROR; 
 
    return EXIT_OK;
}
 
/* Change the event associated with the target file descriptor __fd. */
int xs_epoll_mode(int __epollfd,int __fd,int __newmod)
{
    xs_epoll_event __xs_ev__;
 
    __xs_ev__.events = __newmod;
    __xs_ev__.data.fd = __fd;
 
    if(xs_epoll_ctl(__epollfd, EPOLL_CTL_MOD, __fd, &__xs_ev__) < 0)
            return EXIT_ERROR; 
 
    return EXIT_OK;
}
 
/*
    xs_epoll_init: creates  an epoll "instance", requesting the kernel to allocate an
    event backing store dimensioned for size descriptors.
    Since Linux 2.6.8, the size argument is unused, but must be greater than zero.
    (The kernel  dynamically  sizes  the  required  data
       structures without needing this initial hint.)
*/<br>int xs_epoll_init(int size)
{
    int epollfd = -1;
     
    if(size == 0)
        epollfd = xs_epoll_create(EPOLL_DEFAULT_SIZE);
    else epollfd = xs_epoll_create(size);
     
 
    if(epollfd < 0) return EXIT_ERROR;
 
    xs_logd("epoll create success -> fd : %d", epollfd);
 
    return epollfd;
}
 
 
//#define __VECTOR__
#define xs_defult_thread_size   10
Typedef pthread_cond_t  xs_pthread_cond_t;
Typedef pthread_mutex_t xs_pthread_mutex_t;
Typedefint         XSOCKET;
#define EPOLL_DEFAULT_SIZE  10
typedef struct block_queue 
#ifdef __VECTOR__
    vector queue;
#else
    int queue[xs_defult_thread_size];
    long size;   
#endif
    xs_pthread_cond_t cond; 
    xs_pthread_mutex_t mutex; 
}block_queue_t; 
block_queue_t *bq; 
 
typedef struct block_queue_param 
    void* func; 
    void* queue;    /* Point to block queue structure */
}block_queue_param_t; 
block_queue_param_t bqp;
 
int g_xs_thread_count = xs_defult_thread_size;
#define BUFFER_SIZE         1024
#include <sys/resource.h>
int g_epoll_fd = -1;
xs_epoll_event xs_ev, xs_events[EPOLL_DEFAULT_SIZE];
int g_epoll_size = EPOLL_DEFAULT_SIZE;
int g_serv_fd = -1;
 
static int xs_queue_init( block_queue_t *__q)
{
    if(__q == NULL) return (-1);
     
#ifdef __VECTOR__
    __q->queue = vector_init(xs_defult_thread_size);
#else
    __q->size = 0;
#endif
 
    xs_pthread_cond_init(&(__q->cond), NULL); 
    xs_pthread_mutex_init(&(__q->mutex), NULL); 
 
    return 0;
}
 
block_queue_t *xs_epoll_queue_create( void )
{
    block_queue_t *__q;
 
    __q = xs_malloc(sizeof(block_queue_t));
 
    assert(__q);
 
    return ((xs_queue_init(__q) == 0) ? __q : NULL);
}
static inline void xs_network_epoll_loop(void* data)
{
     int socket;
 
      socket = *(int *)data;
 
    xs_logd("%d !\n", socket);
    char buffer[BUFFER_SIZE];
    xs_pthread_t id = pthread_self();
    xs_logd("thread id is: %ld", id);
 
    /* We only send what recevied just now */
    int   length = xs_net_recv(socket, buffer, BUFFER_SIZE);
    if(length){
        xs_net_send(socket, buffer, strlen(buffer));
        memset(buffer, 0, BUFFER_SIZE);
    }
}
 
void *xs_handle_queue(void *param) 
    void(* func)(void* ); 
    int fd;
 
    block_queue_t* bque = ((block_queue_param_t*)param)->queue; 
    func = ((block_queue_param_t*)param)->func;
 
    xs_pthread_cond_init(&bque->cond,  NULL);
    xs_pthread_mutex_init(&bque->mutex,  NULL);
     
    for(;;) 
    
        if(xs_pthread_mutex_lock(&bque->mutex) == EXIT_OK) {
             
            xs_pthread_cond_wait(&bque->cond, &bque->mutex); 
#ifdef __VECTOR__
            if(bque->queue->active == 0){
                xs_pthread_mutex_unlock(&bque->mutex);
                continue;
            }else {
                fd = *(int *)(vector_lookup(bque->queue, 0));
            }
#else
            if(bque->size==0)  { 
                xs_pthread_mutex_unlock(&bque->mutex);
                continue;
            }else {    
                 
                int i;
                fd = bque->queue[0]; 
 
                for(i = 0; i < bque->size - 1; ++i) 
                    bque->queue[i] = bque->queue[i + 1]; 
                 
                bque->queue[bque->size-1] = 0; 
                bque->size--; 
            
#endif
                xs_pthread_mutex_unlock(&bque->mutex); 
        }
         
        func((void *)&fd); 
    
 
int xs_init_threads(void 
    int i = 0, ret; 
    xs_pthread_t child_thread[g_xs_thread_count]; 
    xs_pthread_attr_t child_thread_attr[g_xs_thread_count]; 
 
    bqp.func = (void*)xs_network_epoll_loop;
    bqp.queue = (void *)bq;
 
    for( i = 0; i < g_xs_thread_count; ++i)  { 
        ret = xs_pthread_attr_init(&child_thread_attr[i]); 
        if(ret != 0) xs_logd("error to init attr !\n");
        pthread_attr_setdetachstate(&child_thread_attr[i], PTHREAD_CREATE_DETACHED); 
        if( pthread_create(&child_thread[i],
            &child_thread_attr[i], xs_handle_queue, (void *)&bqp) < 0 ) { 
                xs_logd("pthread_create Failed : %s - %m\n",strerror(errno)); 
                return 1; 
        
        
    return 0;
int xs_init_server(const char *name,short int port)
{
    struct rlimit rt; 
    int server_socket = -1;
     
    server_socket = xs_create_server(name, port);
 
    rt.rlim_max = rt.rlim_cur = g_epoll_size; 
    if (setrlimit(RLIMIT_NOFILE, &rt) == -1) { 
        xs_logd("setrlimit - %m"); 
        exit(1); 
    
 
    return server_socket; 
}
 
static void xs_insert_queue(block_queue_t *bque, int *fd) 
    xs_pthread_mutex_lock(&bque->mutex); 
     
#ifdef __VECTOR__
    vector_set(bque->queue, fd);
#else
    if(bque->size == g_xs_thread_count) 
        return
 
    bque->queue[bque->size] = *fd; 
    bque->size++;
    if(bque->size > g_xs_thread_count)  { 
        fprintf(stderr,"Queue size over folow.%ld", bque->size); 
        exit (1); 
    
#endif
 
    xs_pthread_cond_signal(&bque->cond); 
    xs_pthread_mutex_unlock(&bque->mutex); 
 
}
 
static inline void xs_handler(void* fd) 
    printf("handler:fd => %d\n", *(int *)(fd)); 
    xs_insert_queue(bq, fd);
 
int xs_epoll_entry()
{
    int nfds, n;
     
    g_serv_fd = xs_init_server(NULL, 11111);
    xs_logd("server thread [FD:%d] is ready for ...", g_serv_fd);
 
     
    bq = xs_epoll_queue_create();
    assert(bq);
     if(xs_init_threads() == 0) 
            xs_logd("Threads ready to use !"); 
 
    g_epoll_fd = xs_epoll_init(g_epoll_size);
    xs_epoll_add(g_epoll_fd, g_serv_fd, EPOLLIN | EPOLLET);
 
     for(;;) { 
            struct sockaddr_in local; 
            socklen_t length = sizeof(local); 
            int client = -1; 
               
            nfds = xs_epoll_wait(g_epoll_fd, xs_events,
EPOLL_DEFAULT_SIZE, epoll_wait_indefinite); 
 
            for(n = 0; n < nfds; ++n) { 
 
            if(xs_events[n].data.fd == g_serv_fd) { 
                client = xs_net_accept(g_serv_fd, (struct sockaddr *)&local, &length); 
                if(client < 0) { 
                    xs_logd("%s",strerror(errno)); 
                    continue
                else {
                    xs_logd("add socket pool : %d", client);
                    set_nonblocking(client); 
                    xs_epoll_add(g_epoll_fd, client, EPOLLIN | EPOLLOUT | EPOLLET);
                    client = -1;
                }
            
            else  /* It's a client fd that needed to process */
                xs_handler((void *)&xs_events[n].data.fd);
            
        
 
    xs_pthread_mutex_destroy(&bq->mutex);
    xs_pthread_cond_destroy(&bq->cond);
    xs_close_socket(g_serv_fd);
    xs_epoll_close(g_epoll_fd);
#ifdef __VECTOR__
    vector_free(bq->queue);
#endif
 
    xs_free(bq);
    return 0;
}
 
int main(int argc,char *argv[])
{
    argc = argc;
    argv = argv;
 
    xs_epoll_entry();
 
    return 0;
}

 

 

 >>>>>>查看作者原文>>>>>>


發表評論
所有評論
還沒有人評論,想成為第一個評論的人麼? 請在上方評論欄輸入並且點擊發布.
相關文章