Netty源碼解析 -- PoolChunk實現原理(jemalloc 3的算法)

{"type":"doc","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"前面文章已經分享了Netty如何引用jemalloc 4算法管理內存。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"本文主要分享Netty 4.1.52之前版本中，PoolChunk如何使用jemalloc 3算法管理內存。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"感興趣的同學可以對比兩種算法。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"源碼分析基於Netty 4.1.29"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"首先說明PoolChunk內存組織方式。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"PoolChunk的內存大小默認是16M，它將內存組織成爲一顆完美二叉樹。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"二叉樹的每一層每個節點所代表的內存大小都是均等的，並且每一層節點所代表的內存大小總和加起來都是16M。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"每一層節點可分配內存是父節點的1/2。整顆二叉樹的總層數爲12，層數從0開始。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"示意圖如下"}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/2d/2d18a24a3a839fe92dc798cfb33ce922.png","alt":null,"title":"","style":[{"key":"width","value":"100%"},{"key":"bordertype","value":"none"}],"href":"","fromPaste":false,"pastePass":false}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"先看一下PoolChunk的構造函數"}]},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"PoolChunk(PoolArena arena, T memory, int pageSize, int maxOrder, int pageShifts, int chunkSize, int offset) {\n\tunpooled = false;\n\tthis.arena = arena;\n\tthis.memory = memory;\n\tthis.pageSize = pageSize;\n\tthis.pageShifts = pageShifts;\n\tthis.maxOrder = maxOrder;\n\tthis.chunkSize = chunkSize;\n\tthis.offset = offset;\n\tunusable = (byte) (maxOrder + 1);\n\tlog2ChunkSize = log2(chunkSize);\n\tsubpageOverflowMask = ~(pageSize - 1);\n\tfreeBytes = chunkSize;\n\n\tassert maxOrder < 30 : \"maxOrder should be < 30, but is: \" + maxOrder;\n\tmaxSubpageAllocs = 1 << maxOrder;\n\n\t// Generate the memory map.\n\tmemoryMap = new byte[maxSubpageAllocs << 1];\n\tdepthMap = new byte[memoryMap.length];\n\tint memoryMapIndex = 1;\n\tfor (int d = 0; d <= maxOrder; ++ d) { // move down the tree one level at a time\n\t\tint depth = 1 << d;\n\t\tfor (int p = 0; p < depth; ++ p) {\n\t\t\t// in each level traverse left to right and set value to the depth of subtree\n\t\t\tmemoryMap[memoryMapIndex] = (byte) d;\n\t\t\tdepthMap[memoryMapIndex] = (byte) d;\n\t\t\tmemoryMapIndex ++;\n\t\t}\n\t}\n\n\tsubpages = newSubpageArray(maxSubpageAllocs);\n}"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"unpooled： 是否使用內存池"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"arena：該PoolChunk所屬的PoolArena"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"memory：底層的內存塊，對於堆內存，它是一個byte數組，對於直接內存，它是(jvm)ByteBuffer，但無論是哪種形式，其內存大小默認都是16M。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"pageSize：葉子節點大小，默認爲8192，即8K。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"maxOrder：表示二叉樹最大的層數，從0開始。默認爲11。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"chunkSize：整個PoolChunk的內存大小，默認爲16777216，即16M。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"offset：底層內存對齊偏移量，默認爲0。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"unusable：表示節點已被分配，不用了，默認爲12。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"freeBytes：空閒內存字節數。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"每個PoolChunk都要按內存使用率關聯到一個PoolChunkList上，內存使用率正是通過freeBytes計算。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"maxSubpageAllocs：葉子節點數量，默認爲2048，即2^11。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"log2ChunkSize：用於計算偏移量，默認爲24。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"subpageOverflowMask：用於判斷申請內存是否爲PoolSubpage，默認爲-8192。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"pageShifts：用於計算分配內存所在二叉樹層數，默認爲13。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"memoryMap：初始化內存管理二叉樹，將每一層節點值設置爲層數d。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"使用數組維護二叉樹，第d層的開始下標爲 "},{"type":"codeinline","content":[{"type":"text","text":"1<= pageSize\n\t\treturn allocateRun(normCapacity);\n\t} else {\n\t\treturn allocateSubpage(normCapacity);\n\t}\n}"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"若申請內存大於pageSize，調用allocateRun方法分配Chunk級別的內存。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"否則調用allocateSubpage方法分配PoolSubpage，再在PoolSubpage上分配所需內存。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"PoolChunk#allocateRun"}]},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"private long allocateRun(int normCapacity) {\n\t// #1\n\tint d = maxOrder - (log2(normCapacity) - pageShifts);\n\t// #2\n\tint id = allocateNode(d);\n\tif (id < 0) {\n\t\treturn id;\n\t}\n\t// #2\n\tfreeBytes -= runLength(id);\n\treturn id;\n}"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#1"}]},{"type":"text","text":" 計算應該在哪層分配分配內存"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"maxOrder - (log2(normCapacity) - pageShifts)"}]},{"type":"text","text":"，如16K， 即2^14，計算結果爲10，即在10層分配。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#2"}]},{"type":"text","text":" 減少空閒內存字節數。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"PoolChunk#allocateNode，在d層分配一個節點"}]},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"private int allocateNode(int d) {\n\tint id = 1;\n\tint initial = - (1 << d); // has last d bits = 0 and rest all = 1\n\t// #1\n\tbyte val = value(id);\n\tif (val > d) { // unusable\n\t\treturn -1;\n\t}\n\t// #2\n\twhile (val < d || (id & initial) == 0) { // id & initial == 1 << d for all ids at depth d, for < d it is 0\n\t\t// #3\n\t\tid <<= 1;\n\t\tval = value(id);\n\t\t// #4\n\t\tif (val > d) {\n\t\t\t// #5\n\t\t\tid ^= 1;\n\t\t\tval = value(id);\n\t\t}\n\t}\n\tbyte value = value(id);\n\tassert value == d && (id & initial) == 1 << d : String.format(\"val = %d, id & initial = %d, d = %d\",\n\t\t\tvalue, id & initial, d);\n\t// #6\n\tsetValue(id, unusable); // mark as unusable\n\t// #7\n\tupdateParentsAlloc(id);\n\treturn id;\n}"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#1"}]},{"type":"text","text":" memoryMap[1] > d，第0層的可分配內存不足，表明該PoolChunk內存不能滿足分配，分配失敗。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#2"}]},{"type":"text","text":" 遍歷二叉樹，找到滿足內存分配的節點。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"val < d"}]},{"type":"text","text":"，即該節點內存滿足分配。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"id & initial = 0"}]},{"type":"text","text":"，即 "},{"type":"codeinline","content":[{"type":"text","text":"id < 1< d的場景，但會出現val == d的場景，如"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"PoolChunk當前可分配內存爲2M，即memoryMap[1] = 3，這時申請2M內存，在0-2層，都是val == d。可參考後面的實例。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#3"}]},{"type":"text","text":" 向下找到下一層下標，注意，子樹左節點的下標是父節點下標的2倍。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#4"}]},{"type":"text","text":" "},{"type":"codeinline","content":[{"type":"text","text":"val > d"}]},{"type":"text","text":"，表示當前節點不能滿足分配"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#5"}]},{"type":"text","text":" "},{"type":"codeinline","content":[{"type":"text","text":"id ^= 1"}]},{"type":"text","text":"，查找同一父節點下的兄弟節點，在兄弟節點上分配內存。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"id ^= 1"}]},{"type":"text","text":"，當id爲偶數，即爲"},{"type":"codeinline","content":[{"type":"text","text":"id+=1"}]},{"type":"text","text":"， 當id爲奇數，即爲"},{"type":"codeinline","content":[{"type":"text","text":"id-=1"}]},{"type":"text","text":"。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"由於前面通過"},{"type":"codeinline","content":[{"type":"text","text":"id <<= 1"}]},{"type":"text","text":"找到下一層下標都是偶數，這裏等於id+=1。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#6"}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"因爲一開始判斷了PoolChunk內存是否足以分配，所以這裏一定可以找到一個可分配節點。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"這裏標註找到的節點已分配。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#7"}]},{"type":"text","text":" 更新找到節點的父節點最大可分配內存塊大小"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"private void updateParentsAlloc(int id) {\n\t// #1\n\twhile (id > 1) {\n\t\t// #2\n\t\tint parentId = id >>> 1;\n\t\tbyte val1 = value(id);\n\t\tbyte val2 = value(id ^ 1);\n\t\tbyte val = val1 < val2 ? val1 : val2;\n\t\tsetValue(parentId, val);\n\t\tid = parentId;\n\t}\n}"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#1"}]},{"type":"text","text":" 向父節點遍歷，直到根節點"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#2"}]},{"type":"text","text":" id >>> 1，找到父節點"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"取當前節點和兄弟節點中較小值，作爲父節點的值，表示父節點最大可分配內存塊大小。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"如memoryMap[1] = 0，表示最大可分配內存塊爲16M。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"分配8M後，memoryMap[1] = 1，表示當前最大可分配內存塊爲8M。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"下面看一則實例，大家可以結合實例理解上面的代碼"}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/b4/b4263b3dae4b74d6000fbe37c9227cce.png","alt":null,"title":"","style":[{"key":"width","value":"100%"},{"key":"bordertype","value":"none"}],"href":"","fromPaste":false,"pastePass":false}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":4},"content":[{"type":"text","text":"內存釋放"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"PoolChunk#free"}]},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"void free(long handle) {\n\t// #1\n int memoryMapIdx = memoryMapIdx(handle);\n int bitmapIdx = bitmapIdx(handle);\n // #2\n if (bitmapIdx != 0) { // free a subpage\n ...\n }\n freeBytes += runLength(memoryMapIdx);\n setValue(memoryMapIdx, depth(memoryMapIdx));\n updateParentsFree(memoryMapIdx);\n}"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#1"}]},{"type":"text","text":" 獲取memoryMapIdx和bitmapIdx"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#2"}]},{"type":"text","text":" 內存塊在PoolSubpage中分配，通過PoolSubpage釋放內存。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#3"}]},{"type":"text","text":" 處理到這裏，就是釋放Chunk級別的內存塊了。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"增加空閒內存字節數。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"設置二叉樹中對應的節點爲未分配"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"對應修改該節點的父節點。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"另外，Netty 4.1.52對PoolArena內存級別劃分的算法也做了調整。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Netty 4.1.52的具體算法前面文章《Netty內存池與PoolArena》已經說過了，這裏簡單說一下Netty 4.1.52前的算法。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"PoolArena中將維護的內存塊按大小劃分爲以下級別："}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Tiny < 512"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Small < 8192(8K)"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Chunk < 16777216(16M)"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Huge >= 16777216"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"PoolArena#tinySubpagePools，smallSubpagePools兩個數組用於維護Tiny，Small級別的內存塊。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"tinySubpagePools，32個元素，每個數組之間差16個字節，大小分別爲0,16,32,48,64, ... ,496"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"smallSubpagePools，4個元素，每個數組之間大小翻倍，大小分別爲512,1025,2048,4096"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"這兩個數組都是PoolSubpage數組，PoolSubpage大小默認都是8192，Tiny，Small級別的內存都是在PoolSubpage上分配的。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Chunk內存塊則都是8192的倍數。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"在Netty 4.1.52，已經刪除了Small級別內存塊，並引入了SizeClasses計算對齊內存塊或計算對應的索引。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"SizeClasses默認將16M劃分爲75個內存塊size，內存劃分更細，也可以減少內存對齊的空間浪費，更充分利用內存。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"如果您覺得本文不錯，歡迎關注我的微信公衆號，系列文章持續更新中。您的關注是我堅持的動力！"}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/5c/5cb935abd5751b075beefdc1bf4914a5.png","alt":null,"title":"","style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":"","fromPaste":false,"pastePass":false}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}}]}