Netty源码解析 -- PoolSubpage实现原理

{"type":"doc","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"前面文章说了PoolChunk如何管理Normal内存块，本文分享PoolSubpage如何管理Small内存块。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"源码分析基于Netty 4.1.52"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":4},"content":[{"type":"text","text":"内存管理算法"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"PoolSubpage负责管理Small内存块。一个PoolSubpage中的内存块size都相同，该size对应SizeClasses#sizeClasses表格的一个索引index。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"新创建的PoolSubpage都必须加入到PoolArena#smallSubpagePools[index]链表中。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"PoolArena#smallSubpagePools是一个PoolSubpage数组，数组中每个元素都是一个PoolSubpage链表，PoolSubpage之间可以通过next，prev组成链表。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"感兴趣的同学可以参考《内存对齐类SizeClasses》。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"注意，Small内存size并不一定小于pageSize(默认为8K)"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"默认Small内存size <= 28672(28KB)"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"关于Normal内存块，Small内存块，pageSize，可参考《PoolChunk实现原理》。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"PoolSubpage实际上就是PoolChunk中的一个Normal内存块，大小为其管理的内存块size与pageSize最小公倍数。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"PoolSubpage使用位图的方式管理内存块。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"PoolSubpage#bitmap是一个long数组，其中每个long元素上每个bit位都可以代表一个内存块是否使用。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":4},"content":[{"type":"text","text":"内存分配"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"分配Small内存块有两个步骤"}]},{"type":"numberedlist","attrs":{"start":"1","normalizeStart":1},"content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":1,"align":null,"origin":null},"content":[{"type":"text","text":"PoolChunk中分配PoolSubpage。"}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"如果PoolArena#smallSubpagePools中已经有对应的PoolSubpage缓冲，则不需要该步骤。"}]},{"type":"numberedlist","attrs":{"start":"2","normalizeStart":"2"},"content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":2,"align":null,"origin":null},"content":[{"type":"text","text":"PoolSubpage上分配内存块"}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"PoolChunk#allocateSubpage"}]},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"private long allocateSubpage(int sizeIdx) {\n // #1\n PoolSubpage head = arena.findSubpagePoolHead(sizeIdx);\n synchronized (head) {\n //allocate a new run\n // #2\n int runSize = calculateRunSize(sizeIdx);\n //runSize must be multiples of pageSize\n // #3\n long runHandle = allocateRun(runSize);\n if (runHandle < 0) {\n return -1;\n }\n // #4\n int runOffset = runOffset(runHandle);\n int elemSize = arena.sizeIdx2size(sizeIdx);\n\n PoolSubpage subpage = new PoolSubpage(head, this, pageShifts, runOffset,\n runSize(pageShifts, runHandle), elemSize);\n\n subpages[runOffset] = subpage;\n // #5\n return subpage.allocate();\n }\n}"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#1"}]},{"type":"text","text":" 这里涉及修改PoolArena#smallSubpagePools中的PoolSubpage链表，需要同步操作"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#2"}]},{"type":"text","text":" 计算内存块size和pageSize最小公倍数"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#3"}]},{"type":"text","text":" 分配一个Normal内存块，作为PoolSubpage的底层内存块，大小为Small内存块size和pageSize最小公倍数"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#4"}]},{"type":"text","text":" 构建PoolSubpage"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"runOffset，即Normal内存块偏移量，也是该PoolSubpage在整个Chunk中的偏移量"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"elemSize，Small内存块size"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#5"}]},{"type":"text","text":" 在subpage上分配内存块"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"PoolSubpage(PoolSubpage head, PoolChunk chunk, int pageShifts, int runOffset, int runSize, int elemSize) {\n // #1\n this.chunk = chunk;\n this.pageShifts = pageShifts;\n this.runOffset = runOffset;\n this.runSize = runSize;\n this.elemSize = elemSize;\n bitmap = new long[runSize >>> 6 + LOG2_QUANTUM]; // runSize / 64 / QUANTUM\n init(head, elemSize);\n}\n\nvoid init(PoolSubpage head, int elemSize) {\n doNotDestroy = true;\n if (elemSize != 0) {\n // #2\n maxNumElems = numAvail = runSize / elemSize;\n nextAvail = 0;\n bitmapLength = maxNumElems >>> 6;\n if ((maxNumElems & 63) != 0) {\n bitmapLength ++;\n }\n\n for (int i = 0; i < bitmapLength; i ++) {\n bitmap[i] = 0;\n }\n }\n // #3\n addToPool(head);\n}"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#1"}]},{"type":"text","text":" bitmap长度为runSize / 64 / QUANTUM，从《内存对齐类SizeClasses》可以看到，runSize都是2^LOG2_QUANTUM的倍数。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#2"}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"elemSize：每个内存块的大小"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"maxNumElems：内存块数量"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"bitmapLength：bitmap使用的long元素个数，使用bitmap中一部分元素足以管理全部内存块。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"(maxNumElems & 63) != 0"}]},{"type":"text","text":"，代表maxNumElems不能整除64，所以bitmapLength要加1，用于管理余下的内存块。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#3"}]},{"type":"text","text":" 添加到PoolSubpage链表中"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"前面分析《Netty内存池与PoolArena》中说过，在PoolArena中分配Small内存块时，首先会从PoolArena#smallSubpagePools中查找对应的PoolSubpage​。如果找到了，直接从该PoolSubpage​上分配内存。否则，分配一个Normal内存块，创建PoolSubpage​，再在上面分配内存块。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"PoolSubpage#allocate"}]},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"long allocate() {\n // #1\n if (numAvail == 0 || !doNotDestroy) {\n return -1;\n }\n // #2\n final int bitmapIdx = getNextAvail();\n // #3\n int q = bitmapIdx >>> 6;\n int r = bitmapIdx & 63;\n assert (bitmap[q] >>> r & 1) == 0;\n bitmap[q] |= 1L << r;\n // #4\n if (-- numAvail == 0) {\n removeFromPool();\n }\n // #5\n return toHandle(bitmapIdx);\n}"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#1"}]},{"type":"text","text":" 没有可用内存块，分配失败。通常PoolSubpage分配完成后会从PoolArena#smallSubpagePools中移除，不再在该PoolSubpage上分配内存，所以一般不会出现这种场景。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#2"}]},{"type":"text","text":" 获取下一个可用内存块的bit下标"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#3"}]},{"type":"text","text":" 设置对应bit为1，即已使用"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"bitmapIdx >>> 6"}]},{"type":"text","text":"，获取该内存块在bitmap数组中第q元素"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"bitmapIdx & 63"}]},{"type":"text","text":"，获取该内存块是bitmap数组中第q个元素的第r个bit位"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"bitmap[q] |= 1L << r"}]},{"type":"text","text":"，将bitmap数组中第q个元素的第r个bit位设置为1，表示已经使用"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#4"}]},{"type":"text","text":" 所有内存块已分配了，则将其从PoolArena中移除。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#5"}]},{"type":"text","text":" toHandle 转换为最终的handle"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"private int getNextAvail() {\n int nextAvail = this.nextAvail;\n if (nextAvail >= 0) {\n this.nextAvail = -1;\n return nextAvail;\n }\n return findNextAvail();\n}"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"nextAvail为初始值或free时释放的值。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"如果nextAvail存在，设置为不可用后直接返回该值。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"如果不存在，调用findNextAvail查找下一个可用内存块。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"private int findNextAvail() {\n final long[] bitmap = this.bitmap;\n final int bitmapLength = this.bitmapLength;\n // #1\n for (int i = 0; i < bitmapLength; i ++) {\n long bits = bitmap[i];\n if (~bits != 0) {\n return findNextAvail0(i, bits);\n }\n }\n return -1;\n}\n\nprivate int findNextAvail0(int i, long bits) {\n final int maxNumElems = this.maxNumElems;\n final int baseVal = i << 6;\n\n // #2\n for (int j = 0; j < 64; j ++) {\n if ((bits & 1) == 0) {\n int val = baseVal | j;\n if (val < maxNumElems) {\n return val;\n } else {\n break;\n }\n }\n bits >>>= 1;\n }\n return -1;\n}"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#1"}]},{"type":"text","text":" 遍历bitmap，"},{"type":"codeinline","content":[{"type":"text","text":"~bits != 0"}]},{"type":"text","text":"，表示存在一个bit位不为1，即存在可用内存块。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#2"}]},{"type":"text","text":" 遍历64个bit位，"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"(bits & 1) == 0"}]},{"type":"text","text":"，检查最低bit位是否为0（可用），为0则返回val。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"val等于 "},{"type":"codeinline","content":[{"type":"text","text":"(i << 6) | j"}]},{"type":"text","text":"，即"},{"type":"codeinline","content":[{"type":"text","text":"i * 64 + j"}]},{"type":"text","text":"，该bit位在bitmap中是第几个bit位。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"bits >>>= 1"}]},{"type":"text","text":"，右移一位，处理下一个bit位。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":4},"content":[{"type":"text","text":"内存释放"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"释放Small内存块可能有两个步骤"}]},{"type":"numberedlist","attrs":{"start":"1","normalizeStart":1},"content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":1,"align":null,"origin":null},"content":[{"type":"text","text":"释放PoolSubpage的上内存块"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":2,"align":null,"origin":null},"content":[{"type":"text","text":"如果PoolSubpage中的内存块已全部释放，则从Chunk中释放该PoolSubpage，同时从PoolArena#smallSubpagePools移除它。"}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"PoolSubpage#free"}]},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"boolean free(PoolSubpage head, int bitmapIdx) {\n if (elemSize == 0) {\n return true;\n }\n // #1\n int q = bitmapIdx >>> 6;\n int r = bitmapIdx & 63;\n assert (bitmap[q] >>> r & 1) != 0;\n bitmap[q] ^= 1L << r;\n\n setNextAvail(bitmapIdx);\n // #2\n if (numAvail ++ == 0) {\n addToPool(head);\n return true;\n }\n\n // #3\n if (numAvail != maxNumElems) {\n return true;\n } else {\n // #4\n if (prev == next) {\n // Do not remove if this subpage is the only one left in the pool.\n return true;\n }\n\n // #5\n doNotDestroy = false;\n removeFromPool();\n return false;\n }\n}"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#1"}]},{"type":"text","text":" 将对应bit位设置为可以使用"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#2"}]},{"type":"text","text":" 在PoolSubpage的内存块全部被使用时，释放了某个内存块，这时重新加入到PoolArena中。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#3"}]},{"type":"text","text":" 未完全释放，即还存在已分配内存块，返回true"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#4"}]},{"type":"text","text":" 逻辑到这里，是处理所有内存块已经完全释放的场景。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"PoolArena#smallSubpagePools链表组成双向链表，链表中只有head和当前PoolSubpage时，当前PoolSubpage的prev，next都指向head。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"这时当前​PoolSubpage是PoolArena中该链表最后一个PoolSubpage，不释放该PoolSubpage，以便下次申请内存时直接从该PoolSubpage上分配。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#5"}]},{"type":"text","text":" 从PoolArena中移除，并返回false，这时PoolChunk会将释放对应Page节点。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"void free(long handle, int normCapacity, ByteBuffer nioBuffer) {\n if (isSubpage(handle)) {\n // #1\n int sizeIdx = arena.size2SizeIdx(normCapacity);\n PoolSubpage head = arena.findSubpagePoolHead(sizeIdx);\n\n PoolSubpage subpage = subpages[runOffset(handle)];\n assert subpage != null && subpage.doNotDestroy;\n\n synchronized (head) {\n // #2\n if (subpage.free(head, bitmapIdx(handle))) {\n //the subpage is still used, do not free it\n return;\n }\n }\n }\n\n // #3\n ...\n}"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#1"}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"查找head节点，同步"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#2"}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"调用subpage#free释放Small内存块"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"如果subpage#free返回false，将继续向下执行，这时会释放PoolSubpage整个内存块，否则，不释放PoolSubpage内存块。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#3"}]},{"type":"text","text":" 释放Normal内存块，就是释放PoolSubpage整个内存块。该部分内容可参考《PoolChunk实现原理》。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"如果您觉得本文不错，欢迎关注我的微信公众号，系列文章持续更新中。您的关注是我坚持的动力！"}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/5c/5cb935abd5751b075beefdc1bf4914a5.png","alt":null,"title":"","style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":"","fromPaste":false,"pastePass":false}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}}]}