Netty源码解析 -- FastThreadLocal与HashedWheelTimer

{"type":"doc","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Netty源码分析系列文章已接近尾声,本文再来分析Netty中两个常见组件:FastThreadLoca与HashedWheelTimer。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"源码分析基于Netty 4.1.52","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":4},"content":[{"type":"text","text":"FastThreadLocal","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"FastThreadLocal比较简单。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"FastThreadLocal和FastThreadLocalThread是配套使用的。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"FastThreadLocalThread继承了Thread,FastThreadLocalThread#threadLocalMap 是一个InternalThreadLocalMap,该InternalThreadLocalMap对象只能用于当前线程。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"InternalThreadLocalMap#indexedVariables是一个数组,存放了当前线程所有FastThreadLocal对应的值。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"而每个FastThreadLocal都有一个index,用于定位InternalThreadLocalMap#indexedVariables。","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/96/967a02e1c526cf5295e7af4297fd7536.png","alt":null,"title":"","style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":"","fromPaste":false,"pastePass":false}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"FastThreadLocal#get","attrs":{}}]},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"public final V get() {\n\t// #1\n\tInternalThreadLocalMap threadLocalMap = InternalThreadLocalMap.get();\n\t// #2\n\tObject v = threadLocalMap.indexedVariable(index);\n\tif (v != InternalThreadLocalMap.UNSET) {\n\t\treturn (V) v;\n\t}\n\t// #3\n\treturn initialize(threadLocalMap);\n}","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#1","attrs":{}}],"attrs":{}},{"type":"text","text":" 获取该线程的InternalThreadLocalMap","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"如果是FastThreadLocalThread,直接获取FastThreadLocalThread#threadLocalMap。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"否则,从UnpaddedInternalThreadLocalMap.slowThreadLocalMap获取该线程InternalThreadLocalMap。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"注意,UnpaddedInternalThreadLocalMap.slowThreadLocalMap是一个ThreadLocal,这里实际回退到使用ThreadLocal了。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#2","attrs":{}}],"attrs":{}},{"type":"text","text":" 每个FastThreadLocal都有一个index。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"通过该index,获取InternalThreadLocalMap#indexedVariables中存放的值","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#3","attrs":{}}],"attrs":{}},{"type":"text","text":" 找不到值,通过initialize方法构建新对象。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"可以看到,FastThreadLocal中连hash算法都不用,通过下标获取对应的值,复杂度为log(1),自然很快啦。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":4},"content":[{"type":"text","text":"HashedWheelTimer","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"HashedWheelTimer是Netty提供的时间轮调度器。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"时间轮是一种充分利用线程资源进行批量化任务调度的调度模型,能够高效的管理各种延时任务。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"简单说,就是将延时任务存放到一个环形队列中,并通过执行线程定时执行该队列的任务。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"例如,","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"环形队列上有60个格子,","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"执行线程每秒移动一个格子,则环形队列每轮可存放1分钟内的任务。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"现在有两个定时任务","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"task1,32秒后执行","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"task2,2分25秒后执行","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"而执行线程当前位于第6格子","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"则task1放到32+6=38格,轮数为0","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"task2放到25+6=31个,轮数为2","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"执行线程将执行当前格子轮数为0的任务,并将其他任务轮数减1。","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/85/852f9978026c760fd68224666b3ba2b4.png","alt":null,"title":"","style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":"","fromPaste":false,"pastePass":false}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"缺点,时间轮调度器的时间精度不高。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"因为时间轮算法的精度取决于执行线程移动速度。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"例如上面例子中执行线程每秒移动一个格子,则调度精度小于一秒的任务就无法准时调用。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"HashedWheelTimer关键字段","attrs":{}}]},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"// 任务执行器,负责执行任务\nWorker worker = new Worker();\n// 任务执行线程\nThread workerThread;\n// HashedWheelTimer状态, 0 - init, 1 - started, 2 - shut down\nint workerState;\n// 时间轮队列,使用数组实现\nHashedWheelBucket[] wheel;\n// 暂存新增的任务\nQueue timeouts = PlatformDependent.newMpscQueue();\n// 已取消任务\nQueue cancelledTimeouts = PlatformDependent.newMpscQueue();","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"添加延迟任务","attrs":{}},{"type":"text","text":" HashedWheelTimer#newTimeout","attrs":{}}]},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"public Timeout newTimeout(TimerTask task, long delay, TimeUnit unit) {\n ...\n\n // #1\n start();\n\n // #2\n long deadline = System.nanoTime() + unit.toNanos(delay) - startTime;\n\n ...\n HashedWheelTimeout timeout = new HashedWheelTimeout(this, task, deadline);\n timeouts.add(timeout);\n return timeout;\n}","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#1","attrs":{}}],"attrs":{}},{"type":"text","text":" 如果HashedWheelTimer未启动,则启动该HashedWheelTimer","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"HashedWheelTimer#start方法负责是启动workerThread线程","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#2","attrs":{}}],"attrs":{}},{"type":"text","text":" startTime是HashedWheelTimer启动时间","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"deadline是相对HashedWheelTimer启动的延迟时间","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"构建HashedWheelTimeout,添加到HashedWheelTimer#timeouts","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"时间轮运行","attrs":{}},{"type":"text","text":" Worker#run","attrs":{}}]},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"public void run() {\n\t...\n\n\t// #1\n\tstartTimeInitialized.countDown();\n\n\tdo {\n\t\t// #2\n\t\tfinal long deadline = waitForNextTick();\n\t\tif (deadline > 0) {\n\t\t\t// #3\n\t\t\tint idx = (int) (tick & mask);\n\t\t\tprocessCancelledTasks();\n\t\t\tHashedWheelBucket bucket = wheel[idx];\n\t\t\t// #4\n\t\t\ttransferTimeoutsToBuckets();\n\t\t\t// #5\n\t\t\tbucket.expireTimeouts(deadline);\n\t\t\t// #6\n\t\t\ttick++;\n\t\t}\n\t} while (WORKER_STATE_UPDATER.get(HashedWheelTimer.this) == WORKER_STATE_STARTED);\n\n\t// #7\n\t...\n}","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#1","attrs":{}}],"attrs":{}},{"type":"text","text":" HashedWheelTimer#start方法阻塞HashedWheelTimer线程直到Worker启动完成,这里解除HashedWheelTimer线程阻塞。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#2","attrs":{}}],"attrs":{}},{"type":"text","text":" 计算下一格子开始执行的时间,然后sleep到下次格子开始执行时间","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#2","attrs":{}}],"attrs":{}},{"type":"text","text":" tick是从HashedWheelTimer启动后移动的总格子数,这里获取tick对应的格子索引。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"由于Long类型足够大,这里并不考虑溢出问题。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#4","attrs":{}}],"attrs":{}},{"type":"text","text":" 将HashedWheelTimer#timeouts的任务迁移到对应的格子中","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#5","attrs":{}}],"attrs":{}},{"type":"text","text":" 处理已到期任务","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#6","attrs":{}}],"attrs":{}},{"type":"text","text":" 移动到下一个格子","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#7","attrs":{}}],"attrs":{}},{"type":"text","text":" 这里是HashedWheelTimer#stop后的逻辑处理,取消任务,停止时间轮","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"迁移任务","attrs":{}},{"type":"text","text":" Worker#transferTimeoutsToBuckets","attrs":{}}]},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"private void transferTimeoutsToBuckets() {\n\t// #1\n\tfor (int i = 0; i < 100000; i++) {\n\t\tHashedWheelTimeout timeout = timeouts.poll();\n\t\tif (timeout == null) {\n\t\t\t// all processed\n\t\t\tbreak;\n\t\t}\n\t\tif (timeout.state() == HashedWheelTimeout.ST_CANCELLED) {\n\t\t\tcontinue;\n\t\t}\n\t\t// #2\n\t\tlong calculated = timeout.deadline / tickDuration;\n\t\t// #3\n\t\ttimeout.remainingRounds = (calculated - tick) / wheel.length;\n\t\t// #4\n\t\tfinal long ticks = Math.max(calculated, tick); // Ensure we don't schedule for past.\n\t\t// #5\n\t\tint stopIndex = (int) (ticks & mask);\n\n\t\tHashedWheelBucket bucket = wheel[stopIndex];\n\t\tbucket.addTimeout(timeout);\n\t}\n}","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#1","attrs":{}}],"attrs":{}},{"type":"text","text":" 注意,每次只迁移100000个任务,以免阻塞线程","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#2","attrs":{}}],"attrs":{}},{"type":"text","text":" 任务延迟时间/每格时间数, 得到该任务需延迟的总格子移动数","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#3","attrs":{}}],"attrs":{}},{"type":"text","text":" (总格子移动数 - 已移动格子数)/每轮格子数,得到轮数","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#4","attrs":{}}],"attrs":{}},{"type":"text","text":" 如果任务在timeouts队列放得太久导致已经过了执行时间,则使用当前tick, 也就是放到当前bucket,以便尽快执行该任务","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#5","attrs":{}}],"attrs":{}},{"type":"text","text":" 计算tick对应格子索引,放到对应的格子位置","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"执行到期任务","attrs":{}},{"type":"text","text":" HashedWheelBucket#expireTimeouts","attrs":{}}]},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"public void expireTimeouts(long deadline) {\n\tHashedWheelTimeout timeout = head;\n\n\twhile (timeout != null) {\n\t\tHashedWheelTimeout next = timeout.next;\n\t\t// #1\n\t\tif (timeout.remainingRounds <= 0) {\n\t\t\t// #2\n\t\t\tnext = remove(timeout);\n\t\t\tif (timeout.deadline <= deadline) {\n\t\t\t\t// #3\n\t\t\t\ttimeout.expire();\n\t\t\t} else {\n\t\t\t\tthrow new IllegalStateException(String.format(\n\t\t\t\t\t\t\"timeout.deadline (%d) > deadline (%d)\", timeout.deadline, deadline));\n\t\t\t}\n\t\t} else if (timeout.isCancelled()) {\n\t\t\tnext = remove(timeout);\n\t\t} else {\n\t\t\t// #4\n\t\t\ttimeout.remainingRounds --;\n\t\t}\n\t\ttimeout = next;\n\t}\n}","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#1","attrs":{}}],"attrs":{}},{"type":"text","text":" 选择轮数小于等于0的任务","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#2","attrs":{}}],"attrs":{}},{"type":"text","text":" 移除任务","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#3","attrs":{}}],"attrs":{}},{"type":"text","text":" 修改状态为过期,并执行任务","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"codeinline","content":[{"type":"text","text":"#4","attrs":{}}],"attrs":{}},{"type":"text","text":" 其他任务轮数减1","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"ScheduledExecutorService使用堆(DelayedWorkQueue)维护任务,新增任务复杂度为O(logN)。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"而 HashedWheelTimer 新增任务复杂度为O(1),所以在任务非常多时, HashedWheelTimer 可以表现出它的优势。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"但是任务较少甚至没有任务时,HashedWheelTimer的执行线程都需要不断移动,也会造成性能消耗。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"注意,HashedWheelTimer使用同一个线程调用和执行任务,如果某些任务执行时间过久,则影响后续定时任务执行。当然,我们也可以考虑在任务中另起线程执行逻辑。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"另外,如果任务过多,也会导致任务长期滞留在HashedWheelTimer#timeouts中而不能及时执行。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"如果您觉得本文不错,欢迎关注我的微信公众号,系列文章持续更新中。您的关注是我坚持的动力!","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/5c/5cb935abd5751b075beefdc1bf4914a5.png","alt":null,"title":"","style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":"","fromPaste":false,"pastePass":false}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}}]}
發表評論
所有評論
還沒有人評論,想成為第一個評論的人麼? 請在上方評論欄輸入並且點擊發布.
相關文章