Netty源碼解析 -- PoolSubpage實現原理

{"type":"doc","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"前面文章說了PoolChunk如何管理Normal內存塊，本文分享PoolSubpage如何管理Small內存塊。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"源碼分析基於Netty 4.1.52"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":4},"content":[{"type":"text","text":"內存管理算法"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"PoolSubpage負責管理Small內存塊。一個PoolSubpage中的內存塊size都相同，該size對應SizeClasses#sizeClasses表格的一個索引index。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"新創建的PoolSubpage都必須加入到PoolArena#smallSubpagePools[index]鏈表中。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"PoolArena#smallSubpagePools是一個PoolSubpage數組，數組中每個元素都是一個PoolSubpage鏈表，PoolSubpage之間可以通過next，prev組成鏈表。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"感興趣的同學可以參考《內存對齊類SizeClasses》。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"注意，Small內存size並不一定小於pageSize(默認爲8K)"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"默認Small內存size <= 28672(28KB)"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"關於Normal內存塊，Small內存塊，pageSize，可參考《PoolChunk實現原理》。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"PoolSubpage實際上就是PoolChunk中的一個Normal內存塊，大小爲其管理的內存塊size與pageSize最小公倍數。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"PoolSubpage使用位圖的方式管理內存塊。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"PoolSubpage#bitmap是一個long數組，其中每個long元素上每個bit位都可以代表一個內存塊是否使用。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":4},"content":[{"type":"text","text":"內存分配"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"分配Small內存塊有兩個步驟"}]},{"type":"numberedlist","attrs":{"start":"1","normalizeStart":1},"content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":1,"align":null,"origin":null},"content":[{"type":"text","text":"PoolChunk中分配PoolSubpage。"}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"如果PoolArena#smallSubpagePools中已經有對應的PoolSubpage緩衝，則不需要該步驟。"}]},{"type":"numberedlist","attrs":{"start":"2","normalizeStart":"2"},"content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":2,"align":null,"origin":null},"content":[{"type":"text","text":"PoolSubpage上分配內存塊"}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"PoolChunk#allocateSubpage"}]},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"private long allocateSubpage(int sizeIdx) {\n // #1\n PoolSubpage head = arena.findSubpagePoolHead(sizeIdx);\n synchronized (head) {\n //allocate a new run\n // #2\n int runSize = calculateRunSize(sizeIdx);\n //runSize must be multiples of pageSize\n // #3\n long runHandle = allocateRun(runSize);\n if (runHandle < 0) {\n return -1;\n }\n // #4\n int runOffset = runOffset(runHandle);\n int elemSize = arena.sizeIdx2size(sizeIdx);\n\n PoolSubpage subpage = new PoolSubpage(head, this, pageShifts, runOffset,\n runSize(pageShifts, runHandle), elemSize);\n\n subpages[runOffset] = subpage;\n // #5\n return subpage.allocate();\n }\n}"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#1"}]},{"type":"text","text":" 這裏涉及修改PoolArena#smallSubpagePools中的PoolSubpage鏈表，需要同步操作"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#2"}]},{"type":"text","text":" 計算內存塊size和pageSize最小公倍數"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#3"}]},{"type":"text","text":" 分配一個Normal內存塊，作爲PoolSubpage的底層內存塊，大小爲Small內存塊size和pageSize最小公倍數"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#4"}]},{"type":"text","text":" 構建PoolSubpage"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"runOffset，即Normal內存塊偏移量，也是該PoolSubpage在整個Chunk中的偏移量"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"elemSize，Small內存塊size"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#5"}]},{"type":"text","text":" 在subpage上分配內存塊"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"PoolSubpage(PoolSubpage head, PoolChunk chunk, int pageShifts, int runOffset, int runSize, int elemSize) {\n // #1\n this.chunk = chunk;\n this.pageShifts = pageShifts;\n this.runOffset = runOffset;\n this.runSize = runSize;\n this.elemSize = elemSize;\n bitmap = new long[runSize >>> 6 + LOG2_QUANTUM]; // runSize / 64 / QUANTUM\n init(head, elemSize);\n}\n\nvoid init(PoolSubpage head, int elemSize) {\n doNotDestroy = true;\n if (elemSize != 0) {\n // #2\n maxNumElems = numAvail = runSize / elemSize;\n nextAvail = 0;\n bitmapLength = maxNumElems >>> 6;\n if ((maxNumElems & 63) != 0) {\n bitmapLength ++;\n }\n\n for (int i = 0; i < bitmapLength; i ++) {\n bitmap[i] = 0;\n }\n }\n // #3\n addToPool(head);\n}"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#1"}]},{"type":"text","text":" bitmap長度爲runSize / 64 / QUANTUM，從《內存對齊類SizeClasses》可以看到，runSize都是2^LOG2_QUANTUM的倍數。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#2"}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"elemSize：每個內存塊的大小"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"maxNumElems：內存塊數量"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"bitmapLength：bitmap使用的long元素個數，使用bitmap中一部分元素足以管理全部內存塊。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"(maxNumElems & 63) != 0"}]},{"type":"text","text":"，代表maxNumElems不能整除64，所以bitmapLength要加1，用於管理餘下的內存塊。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#3"}]},{"type":"text","text":" 添加到PoolSubpage鏈表中"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"前面分析《Netty內存池與PoolArena》中說過，在PoolArena中分配Small內存塊時，首先會從PoolArena#smallSubpagePools中查找對應的PoolSubpage​。如果找到了，直接從該PoolSubpage​上分配內存。否則，分配一個Normal內存塊，創建PoolSubpage​，再在上面分配內存塊。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"PoolSubpage#allocate"}]},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"long allocate() {\n // #1\n if (numAvail == 0 || !doNotDestroy) {\n return -1;\n }\n // #2\n final int bitmapIdx = getNextAvail();\n // #3\n int q = bitmapIdx >>> 6;\n int r = bitmapIdx & 63;\n assert (bitmap[q] >>> r & 1) == 0;\n bitmap[q] |= 1L << r;\n // #4\n if (-- numAvail == 0) {\n removeFromPool();\n }\n // #5\n return toHandle(bitmapIdx);\n}"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#1"}]},{"type":"text","text":" 沒有可用內存塊，分配失敗。通常PoolSubpage分配完成後會從PoolArena#smallSubpagePools中移除，不再在該PoolSubpage上分配內存，所以一般不會出現這種場景。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#2"}]},{"type":"text","text":" 獲取下一個可用內存塊的bit下標"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#3"}]},{"type":"text","text":" 設置對應bit爲1，即已使用"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"bitmapIdx >>> 6"}]},{"type":"text","text":"，獲取該內存塊在bitmap數組中第q元素"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"bitmapIdx & 63"}]},{"type":"text","text":"，獲取該內存塊是bitmap數組中第q個元素的第r個bit位"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"bitmap[q] |= 1L << r"}]},{"type":"text","text":"，將bitmap數組中第q個元素的第r個bit位設置爲1，表示已經使用"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#4"}]},{"type":"text","text":" 所有內存塊已分配了，則將其從PoolArena中移除。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#5"}]},{"type":"text","text":" toHandle 轉換爲最終的handle"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"private int getNextAvail() {\n int nextAvail = this.nextAvail;\n if (nextAvail >= 0) {\n this.nextAvail = -1;\n return nextAvail;\n }\n return findNextAvail();\n}"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"nextAvail爲初始值或free時釋放的值。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"如果nextAvail存在，設置爲不可用後直接返回該值。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"如果不存在，調用findNextAvail查找下一個可用內存塊。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"private int findNextAvail() {\n final long[] bitmap = this.bitmap;\n final int bitmapLength = this.bitmapLength;\n // #1\n for (int i = 0; i < bitmapLength; i ++) {\n long bits = bitmap[i];\n if (~bits != 0) {\n return findNextAvail0(i, bits);\n }\n }\n return -1;\n}\n\nprivate int findNextAvail0(int i, long bits) {\n final int maxNumElems = this.maxNumElems;\n final int baseVal = i << 6;\n\n // #2\n for (int j = 0; j < 64; j ++) {\n if ((bits & 1) == 0) {\n int val = baseVal | j;\n if (val < maxNumElems) {\n return val;\n } else {\n break;\n }\n }\n bits >>>= 1;\n }\n return -1;\n}"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#1"}]},{"type":"text","text":" 遍歷bitmap，"},{"type":"codeinline","content":[{"type":"text","text":"~bits != 0"}]},{"type":"text","text":"，表示存在一個bit位不爲1，即存在可用內存塊。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#2"}]},{"type":"text","text":" 遍歷64個bit位，"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"(bits & 1) == 0"}]},{"type":"text","text":"，檢查最低bit位是否爲0（可用），爲0則返回val。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"val等於 "},{"type":"codeinline","content":[{"type":"text","text":"(i << 6) | j"}]},{"type":"text","text":"，即"},{"type":"codeinline","content":[{"type":"text","text":"i * 64 + j"}]},{"type":"text","text":"，該bit位在bitmap中是第幾個bit位。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"bits >>>= 1"}]},{"type":"text","text":"，右移一位，處理下一個bit位。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":4},"content":[{"type":"text","text":"內存釋放"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"釋放Small內存塊可能有兩個步驟"}]},{"type":"numberedlist","attrs":{"start":"1","normalizeStart":1},"content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":1,"align":null,"origin":null},"content":[{"type":"text","text":"釋放PoolSubpage的上內存塊"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":2,"align":null,"origin":null},"content":[{"type":"text","text":"如果PoolSubpage中的內存塊已全部釋放，則從Chunk中釋放該PoolSubpage，同時從PoolArena#smallSubpagePools移除它。"}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"PoolSubpage#free"}]},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"boolean free(PoolSubpage head, int bitmapIdx) {\n if (elemSize == 0) {\n return true;\n }\n // #1\n int q = bitmapIdx >>> 6;\n int r = bitmapIdx & 63;\n assert (bitmap[q] >>> r & 1) != 0;\n bitmap[q] ^= 1L << r;\n\n setNextAvail(bitmapIdx);\n // #2\n if (numAvail ++ == 0) {\n addToPool(head);\n return true;\n }\n\n // #3\n if (numAvail != maxNumElems) {\n return true;\n } else {\n // #4\n if (prev == next) {\n // Do not remove if this subpage is the only one left in the pool.\n return true;\n }\n\n // #5\n doNotDestroy = false;\n removeFromPool();\n return false;\n }\n}"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#1"}]},{"type":"text","text":" 將對應bit位設置爲可以使用"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#2"}]},{"type":"text","text":" 在PoolSubpage的內存塊全部被使用時，釋放了某個內存塊，這時重新加入到PoolArena中。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#3"}]},{"type":"text","text":" 未完全釋放，即還存在已分配內存塊，返回true"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#4"}]},{"type":"text","text":" 邏輯到這裏，是處理所有內存塊已經完全釋放的場景。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"PoolArena#smallSubpagePools鏈表組成雙向鏈表，鏈表中只有head和當前PoolSubpage時，當前PoolSubpage的prev，next都指向head。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"這時當前​PoolSubpage是PoolArena中該鏈表最後一個PoolSubpage，不釋放該PoolSubpage，以便下次申請內存時直接從該PoolSubpage上分配。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#5"}]},{"type":"text","text":" 從PoolArena中移除，並返回false，這時PoolChunk會將釋放對應Page節點。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"void free(long handle, int normCapacity, ByteBuffer nioBuffer) {\n if (isSubpage(handle)) {\n // #1\n int sizeIdx = arena.size2SizeIdx(normCapacity);\n PoolSubpage head = arena.findSubpagePoolHead(sizeIdx);\n\n PoolSubpage subpage = subpages[runOffset(handle)];\n assert subpage != null && subpage.doNotDestroy;\n\n synchronized (head) {\n // #2\n if (subpage.free(head, bitmapIdx(handle))) {\n //the subpage is still used, do not free it\n return;\n }\n }\n }\n\n // #3\n ...\n}"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#1"}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"查找head節點，同步"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#2"}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"調用subpage#free釋放Small內存塊"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"如果subpage#free返回false，將繼續向下執行，這時會釋放PoolSubpage整個內存塊，否則，不釋放PoolSubpage內存塊。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#3"}]},{"type":"text","text":" 釋放Normal內存塊，就是釋放PoolSubpage整個內存塊。該部分內容可參考《PoolChunk實現原理》。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"如果您覺得本文不錯，歡迎關注我的微信公衆號，系列文章持續更新中。您的關注是我堅持的動力！"}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/5c/5cb935abd5751b075beefdc1bf4914a5.png","alt":null,"title":"","style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":"","fromPaste":false,"pastePass":false}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}}]}