Netty源码解析 -- PoolChunk实现原理(jemalloc 3的算法)

{"type":"doc","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"前面文章已经分享了Netty如何引用jemalloc 4算法管理内存。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"本文主要分享Netty 4.1.52之前版本中，PoolChunk如何使用jemalloc 3算法管理内存。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"感兴趣的同学可以对比两种算法。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"源码分析基于Netty 4.1.29"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"首先说明PoolChunk内存组织方式。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"PoolChunk的内存大小默认是16M，它将内存组织成为一颗完美二叉树。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"二叉树的每一层每个节点所代表的内存大小都是均等的，并且每一层节点所代表的内存大小总和加起来都是16M。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"每一层节点可分配内存是父节点的1/2。整颗二叉树的总层数为12，层数从0开始。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"示意图如下"}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/2d/2d18a24a3a839fe92dc798cfb33ce922.png","alt":null,"title":"","style":[{"key":"width","value":"100%"},{"key":"bordertype","value":"none"}],"href":"","fromPaste":false,"pastePass":false}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"先看一下PoolChunk的构造函数"}]},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"PoolChunk(PoolArena arena, T memory, int pageSize, int maxOrder, int pageShifts, int chunkSize, int offset) {\n\tunpooled = false;\n\tthis.arena = arena;\n\tthis.memory = memory;\n\tthis.pageSize = pageSize;\n\tthis.pageShifts = pageShifts;\n\tthis.maxOrder = maxOrder;\n\tthis.chunkSize = chunkSize;\n\tthis.offset = offset;\n\tunusable = (byte) (maxOrder + 1);\n\tlog2ChunkSize = log2(chunkSize);\n\tsubpageOverflowMask = ~(pageSize - 1);\n\tfreeBytes = chunkSize;\n\n\tassert maxOrder < 30 : \"maxOrder should be < 30, but is: \" + maxOrder;\n\tmaxSubpageAllocs = 1 << maxOrder;\n\n\t// Generate the memory map.\n\tmemoryMap = new byte[maxSubpageAllocs << 1];\n\tdepthMap = new byte[memoryMap.length];\n\tint memoryMapIndex = 1;\n\tfor (int d = 0; d <= maxOrder; ++ d) { // move down the tree one level at a time\n\t\tint depth = 1 << d;\n\t\tfor (int p = 0; p < depth; ++ p) {\n\t\t\t// in each level traverse left to right and set value to the depth of subtree\n\t\t\tmemoryMap[memoryMapIndex] = (byte) d;\n\t\t\tdepthMap[memoryMapIndex] = (byte) d;\n\t\t\tmemoryMapIndex ++;\n\t\t}\n\t}\n\n\tsubpages = newSubpageArray(maxSubpageAllocs);\n}"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"unpooled： 是否使用内存池"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"arena：该PoolChunk所属的PoolArena"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"memory：底层的内存块，对于堆内存，它是一个byte数组，对于直接内存，它是(jvm)ByteBuffer，但无论是哪种形式，其内存大小默认都是16M。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"pageSize：叶子节点大小，默认为8192，即8K。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"maxOrder：表示二叉树最大的层数，从0开始。默认为11。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"chunkSize：整个PoolChunk的内存大小，默认为16777216，即16M。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"offset：底层内存对齐偏移量，默认为0。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"unusable：表示节点已被分配，不用了，默认为12。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"freeBytes：空闲内存字节数。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"每个PoolChunk都要按内存使用率关联到一个PoolChunkList上，内存使用率正是通过freeBytes计算。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"maxSubpageAllocs：叶子节点数量，默认为2048，即2^11。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"log2ChunkSize：用于计算偏移量，默认为24。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"subpageOverflowMask：用于判断申请内存是否为PoolSubpage，默认为-8192。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"pageShifts：用于计算分配内存所在二叉树层数，默认为13。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"memoryMap：初始化内存管理二叉树，将每一层节点值设置为层数d。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"使用数组维护二叉树，第d层的开始下标为 "},{"type":"codeinline","content":[{"type":"text","text":"1<= pageSize\n\t\treturn allocateRun(normCapacity);\n\t} else {\n\t\treturn allocateSubpage(normCapacity);\n\t}\n}"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"若申请内存大于pageSize，调用allocateRun方法分配Chunk级别的内存。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"否则调用allocateSubpage方法分配PoolSubpage，再在PoolSubpage上分配所需内存。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"PoolChunk#allocateRun"}]},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"private long allocateRun(int normCapacity) {\n\t// #1\n\tint d = maxOrder - (log2(normCapacity) - pageShifts);\n\t// #2\n\tint id = allocateNode(d);\n\tif (id < 0) {\n\t\treturn id;\n\t}\n\t// #2\n\tfreeBytes -= runLength(id);\n\treturn id;\n}"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#1"}]},{"type":"text","text":" 计算应该在哪层分配分配内存"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"maxOrder - (log2(normCapacity) - pageShifts)"}]},{"type":"text","text":"，如16K， 即2^14，计算结果为10，即在10层分配。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#2"}]},{"type":"text","text":" 减少空闲内存字节数。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"PoolChunk#allocateNode，在d层分配一个节点"}]},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"private int allocateNode(int d) {\n\tint id = 1;\n\tint initial = - (1 << d); // has last d bits = 0 and rest all = 1\n\t// #1\n\tbyte val = value(id);\n\tif (val > d) { // unusable\n\t\treturn -1;\n\t}\n\t// #2\n\twhile (val < d || (id & initial) == 0) { // id & initial == 1 << d for all ids at depth d, for < d it is 0\n\t\t// #3\n\t\tid <<= 1;\n\t\tval = value(id);\n\t\t// #4\n\t\tif (val > d) {\n\t\t\t// #5\n\t\t\tid ^= 1;\n\t\t\tval = value(id);\n\t\t}\n\t}\n\tbyte value = value(id);\n\tassert value == d && (id & initial) == 1 << d : String.format(\"val = %d, id & initial = %d, d = %d\",\n\t\t\tvalue, id & initial, d);\n\t// #6\n\tsetValue(id, unusable); // mark as unusable\n\t// #7\n\tupdateParentsAlloc(id);\n\treturn id;\n}"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#1"}]},{"type":"text","text":" memoryMap[1] > d，第0层的可分配内存不足，表明该PoolChunk内存不能满足分配，分配失败。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#2"}]},{"type":"text","text":" 遍历二叉树，找到满足内存分配的节点。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"val < d"}]},{"type":"text","text":"，即该节点内存满足分配。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"id & initial = 0"}]},{"type":"text","text":"，即 "},{"type":"codeinline","content":[{"type":"text","text":"id < 1< d的场景，但会出现val == d的场景，如"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"PoolChunk当前可分配内存为2M，即memoryMap[1] = 3，这时申请2M内存，在0-2层，都是val == d。可参考后面的实例。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#3"}]},{"type":"text","text":" 向下找到下一层下标，注意，子树左节点的下标是父节点下标的2倍。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#4"}]},{"type":"text","text":" "},{"type":"codeinline","content":[{"type":"text","text":"val > d"}]},{"type":"text","text":"，表示当前节点不能满足分配"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#5"}]},{"type":"text","text":" "},{"type":"codeinline","content":[{"type":"text","text":"id ^= 1"}]},{"type":"text","text":"，查找同一父节点下的兄弟节点，在兄弟节点上分配内存。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"id ^= 1"}]},{"type":"text","text":"，当id为偶数，即为"},{"type":"codeinline","content":[{"type":"text","text":"id+=1"}]},{"type":"text","text":"， 当id为奇数，即为"},{"type":"codeinline","content":[{"type":"text","text":"id-=1"}]},{"type":"text","text":"。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"由于前面通过"},{"type":"codeinline","content":[{"type":"text","text":"id <<= 1"}]},{"type":"text","text":"找到下一层下标都是偶数，这里等于id+=1。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#6"}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"因为一开始判断了PoolChunk内存是否足以分配，所以这里一定可以找到一个可分配节点。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"这里标注找到的节点已分配。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#7"}]},{"type":"text","text":" 更新找到节点的父节点最大可分配内存块大小"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"private void updateParentsAlloc(int id) {\n\t// #1\n\twhile (id > 1) {\n\t\t// #2\n\t\tint parentId = id >>> 1;\n\t\tbyte val1 = value(id);\n\t\tbyte val2 = value(id ^ 1);\n\t\tbyte val = val1 < val2 ? val1 : val2;\n\t\tsetValue(parentId, val);\n\t\tid = parentId;\n\t}\n}"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#1"}]},{"type":"text","text":" 向父节点遍历，直到根节点"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#2"}]},{"type":"text","text":" id >>> 1，找到父节点"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"取当前节点和兄弟节点中较小值，作为父节点的值，表示父节点最大可分配内存块大小。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"如memoryMap[1] = 0，表示最大可分配内存块为16M。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"分配8M后，memoryMap[1] = 1，表示当前最大可分配内存块为8M。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"下面看一则实例，大家可以结合实例理解上面的代码"}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/b4/b4263b3dae4b74d6000fbe37c9227cce.png","alt":null,"title":"","style":[{"key":"width","value":"100%"},{"key":"bordertype","value":"none"}],"href":"","fromPaste":false,"pastePass":false}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":4},"content":[{"type":"text","text":"内存释放"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"PoolChunk#free"}]},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"void free(long handle) {\n\t// #1\n int memoryMapIdx = memoryMapIdx(handle);\n int bitmapIdx = bitmapIdx(handle);\n // #2\n if (bitmapIdx != 0) { // free a subpage\n ...\n }\n freeBytes += runLength(memoryMapIdx);\n setValue(memoryMapIdx, depth(memoryMapIdx));\n updateParentsFree(memoryMapIdx);\n}"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#1"}]},{"type":"text","text":" 获取memoryMapIdx和bitmapIdx"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#2"}]},{"type":"text","text":" 内存块在PoolSubpage中分配，通过PoolSubpage释放内存。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#3"}]},{"type":"text","text":" 处理到这里，就是释放Chunk级别的内存块了。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"增加空闲内存字节数。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"设置二叉树中对应的节点为未分配"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"对应修改该节点的父节点。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"另外，Netty 4.1.52对PoolArena内存级别划分的算法也做了调整。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Netty 4.1.52的具体算法前面文章《Netty内存池与PoolArena》已经说过了，这里简单说一下Netty 4.1.52前的算法。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"PoolArena中将维护的内存块按大小划分为以下级别："}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Tiny < 512"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Small < 8192(8K)"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Chunk < 16777216(16M)"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Huge >= 16777216"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"PoolArena#tinySubpagePools，smallSubpagePools两个数组用于维护Tiny，Small级别的内存块。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"tinySubpagePools，32个元素，每个数组之间差16个字节，大小分别为0,16,32,48,64, ... ,496"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"smallSubpagePools，4个元素，每个数组之间大小翻倍，大小分别为512,1025,2048,4096"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"这两个数组都是PoolSubpage数组，PoolSubpage大小默认都是8192，Tiny，Small级别的内存都是在PoolSubpage上分配的。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Chunk内存块则都是8192的倍数。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"在Netty 4.1.52，已经删除了Small级别内存块，并引入了SizeClasses计算对齐内存块或计算对应的索引。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"SizeClasses默认将16M划分为75个内存块size，内存划分更细，也可以减少内存对齐的空间浪费，更充分利用内存。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"如果您觉得本文不错，欢迎关注我的微信公众号，系列文章持续更新中。您的关注是我坚持的动力！"}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/5c/5cb935abd5751b075beefdc1bf4914a5.png","alt":null,"title":"","style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":"","fromPaste":false,"pastePass":false}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}}]}