舉一個Netty服務端創建例子如下：
EventLoopGroup bossGroup = new NioEventLoopGroup(1);
EventLoopGroup workerGroup = new NioEventLoopGroup();
try {
ServerBootstrap b = new ServerBootstrap();
b.group(bossGroup, workerGroup)
.channel(NioServerSocketChannel.class)
.childOption(ChannelOption.TCP_NODELAY, true)
.childAttr(AttributeKey.newInstance("childAttr"), "childAttrValue")
.handler(new ServerHandler())
.childHandler(new ChannelInitializer<SocketChannel>() {
@Override
public void initChannel(SocketChannel ch) {
ch.pipeline().addLast(new AuthHandler());
//.................
}
});
ChannelFuture f = b.bind(8888).sync();
f.channel().closeFuture().sync();
} finally {
bossGroup.shutdownGracefully();
workerGroup.shutdownGracefully();
}
從上面我們首先看到如下代碼：
EventLoopGroup bossGroup = new NioEventLoopGroup(1);
EventLoopGroup workerGroup = new NioEventLoopGroup();
從上可見，一個傳了參數，另一個沒傳，跟蹤代碼如下：
// NioEventLoopGroup.java
public NioEventLoopGroup() {
this(0);
}
繼續跟蹤代碼到如下：
// MultithreadEventLoopGroup.java
protected MultithreadEventLoopGroup(int nThreads, Executor executor, Object... args) {
super(nThreads == 0 ? DEFAULT_EVENT_LOOP_THREADS : nThreads, executor, args);
}
點DEFAULT_EVENT_LOOP_THREADS跟進去看發現 無參 默認初始化爲 （線程數 = CPU核數*2）;
// MultithreadEventLoopGroup.java
static {
DEFAULT_EVENT_LOOP_THREADS = Math.max(1, SystemPropertyUtil.getInt("io.netty.eventLoopThreads", Runtime.getRuntime().availableProcessors() * 2));
........................
}
回到剛纔的MultithreadEventLoopGroup方法，繼續跟蹤到如下方法：
// MultithreadEventExecutorGroup.java
protected MultithreadEventExecutorGroup(int nThreads, Executor executor, EventExecutorChooserFactory chooserFactory, Object... args) {
if (nThreads <= 0) {
throw new IllegalArgumentException(String.format("nThreads: %d (expected: > 0)", nThreads));
}
// 1 初始化線程執行器
if (executor == null) {
executor = new ThreadPerTaskExecutor(newDefaultThreadFactory());
}
children = new EventExecutor[nThreads];
for (int i = 0; i < nThreads; i ++) {
boolean success = false;
try {
// 2 創建NIOEventLoop（裏面會創建一下MpscQueue）
children[i] = newChild(executor, args);
success = true;
} catch (Exception e) {
// TODO: Think about if this is a good exception type
throw new IllegalStateException("failed to create a child event loop", e);
} finally {
// 一堆關閉連接操作
if (!success) {
for (int j = 0; j < i; j ++) {
children[j].shutdownGracefully();
}
for (int j = 0; j < i; j ++) {
EventExecutor e = children[j];
try {
while (!e.isTerminated()) {
e.awaitTermination(Integer.MAX_VALUE, TimeUnit.SECONDS);
}
} catch (InterruptedException interrupted) {
// Let the caller handle the interruption.
Thread.currentThread().interrupt();
break;
}
}
}
}
}
// 3 創建線程選擇器
chooser = chooserFactory.newChooser(children);
final FutureListener<Object> terminationListener = new FutureListener<Object>() {
@Override
public void operationComplete(Future<Object> future) throws Exception {
if (terminatedChildren.incrementAndGet() == children.length) {
terminationFuture.setSuccess(null);
}
}
};
for (EventExecutor e: children) {
e.terminationFuture().addListener(terminationListener);
}
Set<EventExecutor> childrenSet = new LinkedHashSet<EventExecutor>(children.length);
Collections.addAll(childrenSet, children);
readonlyChildren = Collections.unmodifiableSet(childrenSet);
}
一 上面代碼很多，我們首先來看看他是怎麼創建線程執行器的，其創建入口代碼如下：
executor = new ThreadPerTaskExecutor(newDefaultThreadFactory());
首先看他的傳參newDefaultThreadFactory()有什麼東西，跟蹤如下：
// DefaultTreadFactory.java
public DefaultThreadFactory(Class<?> poolType, boolean daemon, int priority) {
this(toPoolName(poolType), daemon, priority);
}
繼續往下跟蹤如下，其分析如註釋所示：
// DefaultThreadFactory.java
public DefaultThreadFactory(String poolName, boolean daemon, int priority, ThreadGroup threadGroup) {
if (poolName == null) {
throw new NullPointerException("poolName");
}
if (priority < Thread.MIN_PRIORITY || priority > Thread.MAX_PRIORITY) {
throw new IllegalArgumentException("priority: " + priority + " (expected: Thread.MIN_PRIORITY <= priority <= Thread.MAX_PRIORITY)");
}
// 聲明待創建線程的一些屬性
// 線程名 前綴(用於後面創建的線程名的拼接)、是否守護線程、優先級、線程組
prefix = poolName + '-' + poolId.incrementAndGet() + '-';
this.daemon = daemon;
this.priority = priority;
this.threadGroup = threadGroup;
}
解析完裏面newDefaultThreadFactory()是什麼樣的，我們來看看new ThreadPerTaskExecutor(newDefaultThreadFacotry())是怎麼樣的？
// ThreadPerTaskExecutor.java
public final class ThreadPerTaskExecutor implements Executor {
private final ThreadFactory threadFactory;
public ThreadPerTaskExecutor(ThreadFactory threadFactory) {
if (threadFactory == null) {
throw new NullPointerException("threadFactory");
}
this.threadFactory = threadFactory;
}
@Override
public void execute(Runnable command) {
threadFactory.newThread(command).start();
}
}
簡析：上面很容易看出ThreadPerTaskExecutor只是對ThreadFactory的包裝並存儲，並提供一個execute用於執行。跟蹤下execute((Runnable command)方法如下：
// DefaultThreadFactory.java
public Thread newThread(Runnable r) {
// 1 這個prefix就是上面的 prefix = poolName + '-' + poolId.incrementAndGet() + '-'，所以最終我們可以看到線程名長這樣的：nioEventLoop-1-xx
Thread t = newThread(new DefaultRunnableDecorator(r), prefix + nextId.incrementAndGet());
try {
if (t.isDaemon()) {
if (!daemon) {
t.setDaemon(false);
}
} else {
if (daemon) {
t.setDaemon(true);
}
}
if (t.getPriority() != priority) {
t.setPriority(priority);
}
} catch (Exception ignored) {
// Doesn't matter even if failed to set.
}
return t;
}
二 解析完線程執行器是怎麼創建的，接下來看看如何創建一個NioEventLoop
try {
children[i] = newChild(executor, args);
...................
跟蹤得到如下代碼：
NioEventLoop(NioEventLoopGroup parent, Executor executor, SelectorProvider selectorProvider, SelectStrategy strategy, RejectedExecutionHandler rejectedExecutionHandler) {
super(parent, executor, false, DEFAULT_MAX_PENDING_TASKS, rejectedExecutionHandler);
if (selectorProvider == null) {
throw new NullPointerException("selectorProvider");
}
if (strategy == null) {
throw new NullPointerException("selectStrategy");
}
// 保存信息，openSelector()下面會分析
provider = selectorProvider;
selector = openSelector();
selectStrategy = strategy;
}
選擇super()跟蹤如下：
protected SingleThreadEventExecutor(EventExecutorGroup parent, Executor executor,
boolean addTaskWakesUp, int maxPendingTasks,
RejectedExecutionHandler rejectedHandler) {
// 保存一些基本信息，如線程執行器，任務隊列(用於執行外部線程任務)
super(parent);
this.addTaskWakesUp = addTaskWakesUp;
this.maxPendingTasks = Math.max(16, maxPendingTasks);
this.executor = ObjectUtil.checkNotNull(executor, "executor");
taskQueue = newTaskQueue(this.maxPendingTasks); // PlatformDependent.newMpscQueue(maxPendingTasks);
rejectedExecutionHandler = ObjectUtil.checkNotNull(rejectedHandler, "rejectedHandler");
}
3 創建線程選擇器
創建選擇器的代碼如下：
// MultithreadEventExecutorGroup.java
chooser = chooserFactory.newChooser(children);
跟蹤源碼如下：
// DefaultEventExecutorChooserFactory.java
public EventExecutorChooser newChooser(EventExecutor[] executors) {
if (isPowerOfTwo(executors.length)) {
return new PowerOfTowEventExecutorChooser(executors);
} else {
return new GenericEventExecutorChooser(executors);
}
}
由上面可見根據isPowerOfTwo(executors.length)判斷使用哪種來創建連接器(由下面可知，當 線程數%2==0時，採用第一種，否則採用第二種普通的)
// DefaultEventExecutorChooserFactory.java
private static boolean isPowerOfTwo(int val) {
// 使用了二進制的方式判斷，效率更高
return (val & -val) == val;
}
看完上面，分別看創建連接器的兩個方法如下：
// DefaultEventExecutorChooserFactory.java
PowerOfTowEventExecutorChooser(EventExecutor[] executors) {
this.executors = executors;
}
@Override
public EventExecutor next() {
// 使用二進制計算的方式，效率更高（線程數%2==0時用這種）
return executors[idx.getAndIncrement() & executors.length - 1];
}
// DefaultEventExecutorChooserFactory.java
GenericEventExecutorChooser(EventExecutor[] executors) {
this.executors = executors;
}
@Override
public EventExecutor next() {
// 求餘方式，效率明顯比不上上面的(PowerOfTowEventExecutorChooser)
return executors[Math.abs(idx.getAndIncrement() % executors.length)];
}
4 NioEventLoop啓動流程
先回憶一下"Netty服務端創建例子"如下：
EventLoopGroup bossGroup = new NioEventLoopGroup(1);
EventLoopGroup workerGroup = new NioEventLoopGroup();
try {
ServerBootstrap b = new ServerBootstrap();
b.group(bossGroup, workerGroup)
.channel(NioServerSocketChannel.class)
.childOption(ChannelOption.TCP_NODELAY, true)
.childAttr(AttributeKey.newInstance("childAttr"), "childAttrValue")
.handler(new ServerHandler())
.childHandler(new ChannelInitializer<SocketChannel>() {
@Override
public void initChannel(SocketChannel ch) {
ch.pipeline().addLast(new AuthHandler());
//.................
}
});
ChannelFuture f = b.bind(8888).sync();
f.channel().closeFuture().sync();
} finally {
bossGroup.shutdownGracefully();
workerGroup.shutdownGracefully();
}
其中我們要分析NioEventLoop的啓動流程入口在b.bind(8888)中，跟蹤代碼如下：
// AbstractBootstrap.java
private static void doBind0(
final ChannelFuture regFuture, final Channel channel,
final SocketAddress localAddress, final ChannelPromise promise) {
// This method is invoked before channelRegistered() is triggered. Give user handlers a chance to set up
// the pipeline in its channelRegistered() implementation.
channel.eventLoop().execute(new Runnable() {
@Override
public void run() {
if (regFuture.isSuccess()) {
channel.bind(localAddress, promise).addListener(ChannelFutureListener.CLOSE_ON_FAILURE);
} else {
promise.setFailure(regFuture.cause());
}
}
});
}
點擊execute()跟蹤到如下：
// SingleThreadEventExecutor.java
public void execute(Runnable task) {
if (task == null) {
throw new NullPointerException("task");
}
boolean inEventLoop = inEventLoop();
if (inEventLoop) {
addTask(task);
} else {
startThread();
addTask(task);
if (isShutdown() && removeTask(task)) {
reject();
}
}
if (!addTaskWakesUp && wakesUpForTask(task)) {
wakeup(inEventLoop);
}
}
選擇startThread()跟蹤代碼如下：
// SingleThreadEventExecutor.java
private void startThread() {
if (STATE_UPDATER.get(this) == ST_NOT_STARTED) {
if (STATE_UPDATER.compareAndSet(this, ST_NOT_STARTED, ST_STARTED)) {
doStartThread();
}
}
}
可見上面使用了CAS 樂觀鎖 來判斷當前是否可啓動，繼續跟蹤doStartThread()代碼如下：
// SingleThreadEventExecutor.java
private void doStartThread() {
assert thread == null;
executor.execute(new Runnable() {
@Override
public void run() {
// 獲取當前線程
thread = Thread.currentThread();
if (interrupted) {
thread.interrupt();
}
boolean success = false;
updateLastExecutionTime();
try {
// 啓動當前NioEventLoop！！！
SingleThreadEventExecutor.this.run();
success = true;
} catch (Throwable t) {
logger.warn("Unexpected exception from an event executor: ", t);
} finally {
for (;;) {
int oldState = STATE_UPDATER.get(SingleThreadEventExecutor.this);
if (oldState >= ST_SHUTTING_DOWN || STATE_UPDATER.compareAndSet(SingleThreadEventExecutor.this, oldState, ST_SHUTTING_DOWN)) {
break;
}
}
// Check if confirmShutdown() was called at the end of the loop.
if (success && gracefulShutdownStartTime == 0) {
logger.error("Buggy " + EventExecutor.class.getSimpleName() + " implementation; " + SingleThreadEventExecutor.class.getSimpleName() + ".confirmShutdown() must be called " +
"before run() implementation terminates.");
}
try {
// Run all remaining tasks and shutdown hooks.
for (;;) {
if (confirmShutdown()) {
break;
}
}
} finally {
try {
cleanup();
} finally {
STATE_UPDATER.set(SingleThreadEventExecutor.this, ST_TERMINATED);
threadLock.release();
if (!taskQueue.isEmpty()) {
logger.warn("An event executor terminated with " + "non-empty task queue (" + taskQueue.size() + ')');
}
terminationFuture.setSuccess(null);
}
}
}
}
});
}
NioEventLoop啓動之後做什麼？請看如下
4 NioEventLoop的執行過程
首先我們找到他的啓動入口SingleThreadEventExecutor.this.run();，點進去跟蹤源碼如下：
protected void run() {
for (;;) {
try {
switch (selectStrategy.calculateStrategy(selectNowSupplier, hasTasks())) {
case SelectStrategy.CONTINUE:
continue;
case SelectStrategy.SELECT:
select(wakenUp.getAndSet(false)); // 檢查IO
if (wakenUp.get()) {
selector.wakeup();
}
default:
// fallthrough
}
cancelledKeys = 0;
needsToSelectAgain = false;
final int ioRatio = this.ioRatio;
if (ioRatio == 100) {
try {
processSelectedKeys(); // 處理IO
} finally {
// Ensure we always run tasks.
runAllTasks(); // 異步任務隊列處理
}
} else {
final long ioStartTime = System.nanoTime();
try {
processSelectedKeys(); // 處理IO
} finally {
// Ensure we always run tasks.
final long ioTime = System.nanoTime() - ioStartTime;
runAllTasks(ioTime * (100 - ioRatio) / ioRatio); // 異步任務隊列處理
}
}
} catch (Throwable t) {
handleLoopException(t);
}
// Always handle shutdown even if the loop processing threw an exception.
try {
if (isShuttingDown()) {
closeAll();
if (confirmShutdown()) {
return;
}
}
} catch (Throwable t) {
handleLoopException(t);
}
}
}
以上源碼中，NioEventLoop的執行流程分爲三部分別如下：
-
4.1 檢查IO select(wakenUp.getAndSet(false));
-
4.2 處理IO processSelectedKeys();
-
4.3 異步任務隊列處理 runAllTasks();
4.1 檢查IO select(wakenUp.getAndSet(false))
private void select(boolean oldWakenUp) throws IOException {
Selector selector = this.selector;
try {
int selectCnt = 0;
long currentTimeNanos = System.nanoTime();
// 獲取 第一個截至時間 = 當前時間 + 截止時間段
long selectDeadLineNanos = currentTimeNanos + delayNanos(currentTimeNanos);
for (;;) {
long timeoutMillis = (selectDeadLineNanos - currentTimeNanos + 500000L) / 1000000L;
// 判斷當前是否超時，如果超時並且selectCnt未處理過一次，就執行非阻塞方法selectNow()，並終止循環
if (timeoutMillis <= 0) {
if (selectCnt == 0) {
selector.selectNow();
selectCnt = 1;
}
break;
}
// 判斷是否有外部線程任務，如果有就執行非阻塞方法selectNow()，並終止循環
if (hasTasks() && wakenUp.compareAndSet(false, true)) {
selector.selectNow();
selectCnt = 1;
break;
}
// 如果上面的非阻塞都沒被調用，就調用這個select()阻塞，等待IO有數據
int selectedKeys = selector.select(timeoutMillis);
selectCnt ++;
if (selectedKeys != 0 || oldWakenUp || wakenUp.get() || hasTasks() || hasScheduledTasks()) {
// - Selected something, 有IO事件
// - waken up by user, or 被喚醒
// - the task queue has a pending task. 有外部線程任務
// - a scheduled task is ready for processing 定時隊列裏有任務
break;
}
// 判斷當前線程是否被中斷
if (Thread.interrupted()) {
if (logger.isDebugEnabled()) {
logger.debug("Selector.select() returned prematurely because " + "Thread.currentThread().interrupt() was called. Use " + "NioEventLoop.shutdownGracefully() to shutdown the NioEventLoop.");
}
selectCnt = 1;
break;
}
long time = System.nanoTime();
// 當前大於超時時間，說明已進行了一次阻塞的select操作。下面會currentTimeNanos = time 重置currentTimeNanos，所以這個判斷條件不會再進去
if (time - TimeUnit.MILLISECONDS.toNanos(timeoutMillis) >= currentTimeNanos) {
// timeoutMillis elapsed without anything selected.
selectCnt = 1;
} else if (SELECTOR_AUTO_REBUILD_THRESHOLD > 0 && selectCnt >= SELECTOR_AUTO_REBUILD_THRESHOLD) {
// 如果上面過了，然後進入了這個if判斷，就算是一次空輪詢，netty設置了一個閾值=512 限制空輪詢的空轉次數，到達這個次數會進行如下處理
// The selector returned prematurely many times in a row.
// Rebuild the selector to work around the problem.
logger.warn("Selector.select() returned prematurely {} times in a row; rebuilding Selector {}.", selectCnt, selector);
// ****!!!重點！！！***********解決這個空輪詢的bug的方法：通過rebuildSelector()將產生bug的selector丟棄掉，重建新的selector
rebuildSelector();
selector = this.selector;
// Select again to populate selectedKeys.
selector.selectNow();
selectCnt = 1;
break;
}
currentTimeNanos = time;
}
if (selectCnt > MIN_PREMATURE_SELECTOR_RETURNS) {
if (logger.isDebugEnabled()) {
logger.debug("Selector.select() returned prematurely {} times in a row for Selector {}.", selectCnt - 1, selector);
}
}
} catch (CancelledKeyException e) {
if (logger.isDebugEnabled()) {
logger.debug(CancelledKeyException.class.getSimpleName() + " raised by a Selector {} - JDK bug?", selector, e);
}
// Harmless exception - log anyway
}
}
備註：netty在static {} 中初始化了空輪詢的最大次數，代碼如下
int selectorAutoRebuildThreshold = SystemPropertyUtil.getInt("io.netty.selectorAutoRebuildThreshold", 512);
4.2 處理IO processSelectedKeys()
分析如下：
// NioEventLoop.java
private void processSelectedKeys() {
if (selectedKeys != null) {
processSelectedKeysOptimized(selectedKeys.flip());
} else {
processSelectedKeysPlain(selector.selectedKeys());
}
}
繼續跟蹤到最後如下(分別爲OP_CONNECT、OP_WRITE、OP_ACCEPT、OP_READ),這裏面selectKey有一個優化的過程(底層結構由set優化爲數組，有興趣可以自行去找)：
private void processSelectedKey(SelectionKey k, AbstractNioChannel ch) {
// 驗證是否合法,不合法的情況省略
.................................
try {
int readyOps = k.readyOps(); //獲取IO事件
// We first need to call finishConnect() before try to trigger a read(...) or write(...) as otherwise
// the NIO JDK channel implementation may throw a NotYetConnectedException.
if ((readyOps & SelectionKey.OP_CONNECT) != 0) {
int ops = k.interestOps();
ops &= ~SelectionKey.OP_CONNECT;
k.interestOps(ops);
unsafe.finishConnect();
}
// Process OP_WRITE first as we may be able to write some queued buffers and so free memory.
if ((readyOps & SelectionKey.OP_WRITE) != 0) {
// Call forceFlush which will also take care of clear the OP_WRITE once there is nothing left to write
ch.unsafe().forceFlush();
}
// Also check for readOps of 0 to workaround possible JDK bug which may otherwise lead
// to a spin loop
if ((readyOps & (SelectionKey.OP_READ | SelectionKey.OP_ACCEPT)) != 0 || readyOps == 0) {
unsafe.read();
if (!ch.isOpen()) {
// Connection already closed - no need to handle write.
return;
}
}
} catch (CancelledKeyException ignored) {
unsafe.close(unsafe.voidPromise());
}
}
4.3 異步任務隊列處理 runAllTasks()
調用如下會執行兩種任務隊列：
-
第一種是外部普通任務隊列(在NioEventLoop啓動的時候就創建了)
-
第二種是定時任務隊列
其中定時任務隊列邏輯跟蹤代碼如下(他會把定時任務的全部移動到 外部普通任務隊列(第一種(taskQueue)))
private boolean fetchFromScheduledTaskQueue() {
long nanoTime = AbstractScheduledEventExecutor.nanoTime();
// 從任務隊列取出第一個
Runnable scheduledTask = pollScheduledTask(nanoTime);
while (scheduledTask != null) {
if (!taskQueue.offer(scheduledTask)) {
// 失敗就重新放回去
scheduledTaskQueue().add((ScheduledFutureTask<?>) scheduledTask);
return false;
}
scheduledTask = pollScheduledTask(nanoTime);
}
return true;
}
定時任務 全部放到第一種(外部普通任務隊列)任務後，就開始執行任務了~~~~~~~~
附加說明：執行任務時，每執行64個，會統計時間是否超過，如果超過會終止任務執行，代碼如下
// Check timeout every 64 tasks because nanoTime() is relatively expensive.
// XXX: Hard-coded value - will make it configurable if it is really a problem.
if ((runTasks & 0x3F) == 0) { // 0x3F -> 0011 1111
lastExecutionTime = ScheduledFutureTask.nanoTime();
if (lastExecutionTime >= deadline) {
break;
}
}