一、爲什麼需要多線程
單線程的劣勢：
- 不能利用多核
- 當value很大時，redis的QPS會下降的很厲害
主要消耗在同步IO上（假設帶寬和內存足夠）
從socket讀取時，會從內核態將數據拷貝到用戶態
將數據回寫到socket，會將數據從用戶態拷貝到內核態
這些讀寫會佔用大量的cpu時間，導致瓶頸，所以引入多線程，分攤這部分消耗，使redis的吞吐量更上一層樓。
多線程優勢：
- 充分利用多核優勢
- I/O線程同時只是在讀或則寫socket
- I/O線程只負責讀寫，不負責執行命令
- 命令執行只在主線程中，保證命令的串行執行
二、整體邏輯
三、代碼邏輯
3.1 配置
默認是單線程進行讀寫操作
standardConfig configs[] = {
...
createBoolConfig("io-threads-do-reads", NULL, IMMUTABLE_CONFIG, server.io_threads_do_reads, 0,NULL, NULL), /* Read + parse from threads? */
...
createIntConfig("io-threads", NULL, IMMUTABLE_CONFIG, 1, 128, server.io_threads_num, 1, INTEGER_CONFIG, NULL, NULL), /* Single threaded by default */
...
}redis.conf文件進行配置
#只用作併發寫
# io-threads 4
//開啓後，併發讀
# io-threads-do-reads no3.2 創建線程
main() ->
...
InitServerLast() ->
...
initThreadedIO() ->
...
io_threads_active = 0; /* We start with threads not active. */
...
/* Spawn and initialize the I/O threads. */
for (int i = 0; i < server.io_threads_num; i++) {
/* Things we do for all the threads including the main thread. */
io_threads_list[i] = listCreate();
if (i == 0) continue; /* Thread 0 is the main thread. */
/* Things we do only for the additional threads. */
pthread_t tid;
pthread_mutex_init(&io_threads_mutex[i],NULL);
io_threads_pending[i] = 0;
pthread_mutex_lock(&io_threads_mutex[i]); /* Thread will be stopped. */
if (pthread_create(&tid,NULL,IOThreadMain,(void*)(long)i) != 0) {
serverLog(LL_WARNING,"Fatal: Can't initialize IO thread.");
exit(1);
}
io_threads[i] = tid;
}
...
...
...3.3 io線程循環等待事件
IOThreadMain()
...
while (1) {
/* Wait for start */
for (int j = 0; j < 1000000; j++) {
if (io_threads_pending[id] != 0) break;
}
/* Give the main thread a chance to stop this thread. */
if (io_threads_pending[id] == 0) {
pthread_mutex_lock(&io_threads_mutex[id]);
pthread_mutex_unlock(&io_threads_mutex[id]);
continue;
}
serverAssert(io_threads_pending[id] != 0);
if (tio_debug) printf("[%ld] %d to handle\n", id, (int)listLength(io_threads_list[id]));
/* Process: note that the main thread will never touch our list
* before we drop the pending count to 0. */
listIter li;
listNode *ln;
listRewind(io_threads_list[id],&li);
while((ln = listNext(&li))) {
client *c = listNodeValue(ln);
if (io_threads_op == IO_THREADS_OP_WRITE) {
writeToClient(c,0);
} else if (io_threads_op == IO_THREADS_OP_READ) {
readQueryFromClient(c->conn);
} else {
serverPanic("io_threads_op value is unknown");
}
}
listEmpty(io_threads_list[id]);
io_threads_pending[id] = 0;
if (tio_debug) printf("[%ld] Done\n", id);
}3.4 主線程將寫事件發送給io線程
handleClientsWithPendingWritesUsingThreads() ->
...
/* Start threads if needed. */
if (!io_threads_active) startThreadedIO();
//分發寫事件
//1. 將寫事件隨機分發到各個線程對應的隊列中
listRewind(server.clients_pending_write,&li);
int item_id = 0;
while((ln = listNext(&li))) {
client *c = listNodeValue(ln);
c->flags &= ~CLIENT_PENDING_WRITE;
int target_id = item_id % server.io_threads_num;
listAddNodeTail(io_threads_list[target_id],c);
item_id++;
}
/* Give the start condition to the waiting threads, by setting the
* start condition atomic var. */
io_threads_op = IO_THREADS_OP_WRITE;
//2. 設置線程需要處理的事件個數， io線程讀取這個值進行判斷是否有事件需要處理
for (int j = 1; j < server.io_threads_num; j++) {
int count = listLength(io_threads_list[j]);
io_threads_pending[j] = count;
}
//可能分發到主線程上一些事件，主線程也進行處理
/* Also use the main thread to process a slice of clients. */
listRewind(io_threads_list[0],&li);
while((ln = listNext(&li))) {
client *c = listNodeValue(ln);
writeToClient(c,0);
}
listEmpty(io_threads_list[0]);
// 等待所有線程處理完成
/* Wait for all the other threads to end their work. */
while(1) {
unsigned long pending = 0;
for (int j = 1; j < server.io_threads_num; j++)
pending += io_threads_pending[j];
if (pending == 0) break;
}
...3.5 啓停線程
//停止線程
stopThreadedIO()
...
for (int j = 1; j < server.io_threads_num; j++)
pthread_mutex_lock(&io_threads_mutex[j]);
io_threads_active = 0;
...
//啓動線程
startThreadedIO()
...
for (int j = 1; j < server.io_threads_num; j++)
pthread_mutex_unlock(&io_threads_mutex[j]);
io_threads_active = 1;
...io_threads_active 控制主線程不將事件發給io線程
io_threads_mutex 互斥鎖將使io線程阻塞在獲取鎖那裏
3.6 使用io線程進行讀數據
首先啓用io線程進行讀
io-threads-do-reads yes
將讀事件寫入到io隊列中
//select回調函數進行處理
readQueryFromClient() ->
postponeClientRead()
...
//開啓了讀線程，將客戶端加入到隊列中
listAddNodeHead(server.clients_pending_read,c);
...將讀事件分發到各個線程中
handleClientsWithPendingReadsUsingThreads() ->
...
//隨機分配
listRewind(server.clients_pending_read,&li);
int item_id = 0;
while((ln = listNext(&li))) {
client *c = listNodeValue(ln);
int target_id = item_id % server.io_threads_num;
listAddNodeTail(io_threads_list[target_id],c);
item_id++;
}
//通知各個線程
/* Give the start condition to the waiting threads, by setting the
* start condition atomic var. */
io_threads_op = IO_THREADS_OP_READ;
for (int j = 1; j < server.io_threads_num; j++) {
int count = listLength(io_threads_list[j]);
io_threads_pending[j] = count;
}
//主線程也算一個線程，處理分配到的讀事件
/* Also use the main thread to process a slice of clients. */
listRewind(io_threads_list[0],&li);
while((ln = listNext(&li))) {
client *c = listNodeValue(ln);
readQueryFromClient(c->conn);
}
listEmpty(io_threads_list[0]);
//等待線程操作結束
/* Wait for all the other threads to end their work. */
while(1) {
unsigned long pending = 0;
for (int j = 1; j < server.io_threads_num; j++)
pending += io_threads_pending[j];
if (pending == 0) break;
}io線程進行讀取操作
IOThreadMain()
...
while((ln = listNext(&li))) {
client *c = listNodeValue(ln);
if (io_threads_op == IO_THREADS_OP_WRITE) {
writeToClient(c,0);
} else if (io_threads_op == IO_THREADS_OP_READ) {
readQueryFromClient(c->conn);
} else {
serverPanic("io_threads_op value is unknown");
}
}readQueryFromClient() ->
...
//讀取數據
connRead()
...
processInputBuffer() ->
...
//解析命令行
processInlineBuffer() / processMultibulkBuffer()
...
//重點
//這裏直接退出了，把命令的執行放到了主線程中，保證了命令的串行執行
if (c->flags & CLIENT_PENDING_READ) {
c->flags |= CLIENT_PENDING_COMMAND;
break;
}
/* We are finally ready to execute the command. */
if (processCommandAndResetClient(c) == C_ERR) {
/* If the client is no longer valid, we avoid exiting this
* loop and trimming the client buffer later. So we return
* ASAP in that case. */
return;
}3.7 主線程進行命令的執行
handleClientsWithPendingReadsUsingThreads() ->
...
//遍歷讀事件
while(listLength(server.clients_pending_read)) {
ln = listFirst(server.clients_pending_read);
client *c = listNodeValue(ln);
c->flags &= ~CLIENT_PENDING_READ; //設置狀態， 重要， 處理命令里根據此狀態 判斷是加入到讀線程，還是主線程進行執行命令
listDelNode(server.clients_pending_read,ln);
if (c->flags & CLIENT_PENDING_COMMAND) {
c->flags &= ~CLIENT_PENDING_COMMAND;
if (processCommandAndResetClient(c) == C_ERR) {
/* If the client is no longer valid, we avoid
* processing the client later. So we just go
* to the next. */
continue;
}
}
//處理請求以及執行命令
processInputBuffer(c);
}
...