4.1.36這篇netty講懂了-- Netty源碼剖析

NettyServer端源碼

根據代碼演示分析

public static void main(String[] args) throws InterruptedException {
        EventLoopGroup parentGroup = new NioEventLoopGroup();
        EventLoopGroup childGroup = new NioEventLoopGroup();
        try {
            ServerBootstrap bootstrap = new ServerBootstrap();
            bootstrap.group(parentGroup, childGroup)
                    // 指定要創建Channel類型
                     .channel(NioServerSocketChannel.class)
                     .childHandler(new ChannelInitializer<SocketChannel>() {

                        @Override
                        protected void initChannel(SocketChannel ch) throws Exception {
                            // 獲取channel中的Pipeline
                            ChannelPipeline pipeline = ch.pipeline();
                            // StringDecoder：字符串解碼器，將Channel中的ByteBuf數據解碼爲String
                            pipeline.addLast(new StringDecoder());
                            // StringEncoder：字符串編碼器，將String編碼爲將要發送到Channel中的ByteBuf
                            pipeline.addLast(new StringEncoder());
                            pipeline.addLast(new SomeServerHandler());
                        }
                    });
            ChannelFuture future = bootstrap.bind(8080).sync();
            System.out.println("服務器已啓動");
            future.channel().closeFuture().sync();
        } finally {
            parentGroup.shutdownGracefully();
            childGroup.shutdownGracefully();
        }
    }

4.1.36源碼從綁定開始分析:

Netty服務端啓動

創建指定channel

在執行bind之前，先執行了channel(NioServerSocketChannel.class) ，先創建指定的channel

【AbstractBootstrap.class】
public B channel(Class<? extends C> channelClass) {
        if (channelClass == null) {
            throw new NullPointerException("channelClass");
        }
        // 返回一個channelFactory
        return channelFactory(new ReflectiveChannelFactory<C>(channelClass));
}

public ReflectiveChannelFactory(Class<? extends T> clazz) {
        ObjectUtil.checkNotNull(clazz, "clazz");
        try {
            // 初始化NioServerSocketChannel的構造器
            this.constructor = clazz.getConstructor();
        } catch (NoSuchMethodException e) {
            throw new IllegalArgumentException("Class " + StringUtil.simpleClassName(clazz) +
                    " does not have a public non-arg constructor", e);
        }
    }

執行bind流程

private ChannelFuture doBind(final SocketAddress localAddress) {
        // 創建、初始化channel，並將其註冊到Selector
        final ChannelFuture regFuture = initAndRegister();
        // 從異步結果中獲取channel
        final Channel channel = regFuture.channel();
        // 獲取異步操作執行過程中發生的異常
        if (regFuture.cause() != null) {
            return regFuture;
        }

        // 判斷當前異步操作是否完成：或者是成功，或者是異常
        if (regFuture.isDone()) {   // 若異步操作成功
            // At this point we know that the registration was complete and successful.
            // 創建一個可修改的異步結果對象channelFuture
            ChannelPromise promise = channel.newPromise();
            // 綁定端口號
            doBind0(regFuture, channel, localAddress, promise);
            return promise;
        } else {  // 若異步操作未完成
            // Registration future is almost always fulfilled already, but just in case it's not.
            final PendingRegistrationPromise promise = new PendingRegistrationPromise(channel);
            // 爲異步操作添加監聽器
            regFuture.addListener(new ChannelFutureListener() {
                // 當異步操作完成(成功，異常)，就會觸發該方法的執行
                @Override
                public void operationComplete(ChannelFuture future) throws Exception {
                    // 獲取異步操作執行過程中發生的異常
                    Throwable cause = future.cause();
                    if (cause != null) {  // 異步執行過程發生異常
                        // Registration on the EventLoop failed so fail the ChannelPromise directly to not cause an
                        // IllegalStateException once we try to access the EventLoop of the Channel.
                        // 修改異步結果爲：失敗
                        promise.setFailure(cause);
                    } else {
                        // Registration was successful, so set the correct executor to use.
                        // See https://github.com/netty/netty/issues/2586
                        promise.registered();
                        // 綁定端口號
                        doBind0(regFuture, channel, localAddress, promise);
                    }
                }
            });
            return promise;
        }
    }

註冊流程

initAndRegister 流程

final ChannelFuture initAndRegister() {
        Channel channel = null;
        try {
            // 創建一個channel
            channel = channelFactory.newChannel();
            // 初始化channel
            init(channel);
        } catch (Throwable t) {
            if (channel != null) {
                // channel can be null if newChannel crashed (eg SocketException("too many open files"))
                channel.unsafe().closeForcibly();
                // as the Channel is not registered yet we need to force the usage of the GlobalEventExecutor
                return new DefaultChannelPromise(channel, GlobalEventExecutor.INSTANCE).setFailure(t);
            }
            // as the Channel is not registered yet we need to force the usage of the GlobalEventExecutor
            return new DefaultChannelPromise(new FailedChannel(), GlobalEventExecutor.INSTANCE).setFailure(t);
        }

        // 將當前channel註冊給selector
        ChannelFuture regFuture = config().group().register(channel);
        if (regFuture.cause() != null) {
            if (channel.isRegistered()) {
                channel.close();
            } else {
                channel.unsafe().closeForcibly();
            }
        }
		return regFuture;
}

channelFactory.newChannel()流程

創建channel

public T newChannel() {
    try {
        // 使用反射機制，調用其無參構造器，創建channel
        return constructor.newInstance();
    } catch (Throwable t) {
        throw new ChannelException("Unable to create Channel from class " + constructor.getDeclaringClass(), t);
    }
}

最終調用NioServerSocketChannel的無參構造函數

// 獲取到一個全局性的provider
private static final SelectorProvider DEFAULT_SELECTOR_PROVIDER = SelectorProvider.provider();

public NioServerSocketChannel() {
    // 我們Netty的channel實際上是對原生的NIO的channel的封裝
    this(newSocket(DEFAULT_SELECTOR_PROVIDER));
}

// netty最終調用了 jdk原生nio，
private static ServerSocketChannel newSocket(SelectorProvider provider) {
    try {
        // 通過全局性的provider，創建一個原生的NIO的channel
        return provider.openServerSocketChannel();
    } catch (IOException e) {
        throw new ChannelException(
            "Failed to open a server socket.", e);
    }
 }

protected AbstractNioChannel(Channel parent, SelectableChannel ch, int readInterestOp) {
    super(parent);
    this.ch = ch;
    this.readInterestOp = readInterestOp;
    try {
        // 指定channel爲非阻塞
        ch.configureBlocking(false);
    } catch (IOException e) {
        try {
            ch.close();
        } catch (IOException e2) {
            if (logger.isWarnEnabled()) {
                logger.warn(
                    "Failed to close a partially initialized socket.", e2);
            }
        }

        throw new ChannelException("Failed to enter non-blocking mode.", e);
    }
}
protected AbstractChannel(Channel parent) {
        this.parent = parent;
        // 爲Netty的channel生成id
        id = newId();
        // 底層操作對象
        unsafe = newUnsafe();
        // 創建當前channel所綁定的channelPipeline
        pipeline = newChannelPipeline();
}

初始化channel

【ServerBootstrap】#init
@Override
void init(Channel channel) throws Exception {
        // 處理bootstrap中的option設置屬性
        final Map<ChannelOption<?>, Object> options = options0();
        synchronized (options) {
            setChannelOptions(channel, options, logger);
        }

        // 處理bootstrap中的attr設置屬性
        final Map<AttributeKey<?>, Object> attrs = attrs0();
        synchronized (attrs) {
            // 將bootstrap中設置的所有attr屬性配置給channel
            for (Entry<AttributeKey<?>, Object> e: attrs.entrySet()) {
                @SuppressWarnings("unchecked")
                AttributeKey<Object> key = (AttributeKey<Object>) e.getKey();
                channel.attr(key).set(e.getValue());
            }
        }

        // 向pipeline中添加處理器
        ChannelPipeline p = channel.pipeline();

        // 獲取bootstrap中設置的所有child開頭的屬性
        final EventLoopGroup currentChildGroup = childGroup;
        final ChannelHandler currentChildHandler = childHandler;
        final Entry<ChannelOption<?>, Object>[] currentChildOptions;
        final Entry<AttributeKey<?>, Object>[] currentChildAttrs;
        synchronized (childOptions) {
            currentChildOptions = childOptions.entrySet().toArray(newOptionArray(0));
        }
        synchronized (childAttrs) {
            currentChildAttrs = childAttrs.entrySet().toArray(newAttrArray(0));
        }

        // ChannelInitializer是一個處理器，其存在的意義是，爲pipeline添加其它處理器
        p.addLast(new ChannelInitializer<Channel>() {
            @Override
            public void initChannel(final Channel ch) throws Exception {
                final ChannelPipeline pipeline = ch.pipeline();
                // 獲取bootstrap中配置的handler()
                ChannelHandler handler = config.handler();
                if (handler != null) {
                    pipeline.addLast(handler);
                }

                // ch.eventLoop()是獲取到當前channel所綁定的evenLoop
                // 然後再使用該eventLoop所綁定的線程來執行指定的任務
                ch.eventLoop().execute(new Runnable() {
                    @Override
                    public void run() {
                        // 向pipeline中添加ServerBootstrapAcceptor處理器
                        // 該處理器用於處理client的連接
                        pipeline.addLast(new ServerBootstrapAcceptor(
                                ch, currentChildGroup, currentChildHandler, currentChildOptions, currentChildAttrs));
                    }
                });
            }
        });
   }
   
static void setChannelOptions(
    Channel channel, Map<ChannelOption<?>, Object> options, InternalLogger logger) {
    // 遍歷通過bootstrap設置的所有option
    for (Map.Entry<ChannelOption<?>, Object> e: options.entrySet()) {
        setChannelOption(channel, e.getKey(), e.getValue(), logger);
    }
}

ServerBootstrapAcceptor.class

ServerBootstrapAcceptor(
    final Channel channel, EventLoopGroup childGroup, ChannelHandler childHandler,
    Entry<ChannelOption<?>, Object>[] childOptions, Entry<AttributeKey<?>, Object>[] childAttrs) {
    this.childGroup = childGroup;
    this.childHandler = childHandler;
    this.childOptions = childOptions;
    this.childAttrs = childAttrs;

    // Task which is scheduled to re-enable auto-read.
    // It's important to create this Runnable before we try to submit it as otherwise the URLClassLoader may
    // not be able to load the class because of the file limit it already reached.
    //
    // See https://github.com/netty/netty/issues/1328
    enableAutoReadTask = new Runnable() {
        @Override
        public void run() {
            channel.config().setAutoRead(true);
        }
    };
}

// 當client發送來連接請求時，會觸發channelRead()方法的執行
@Override
@SuppressWarnings("unchecked")
public void channelRead(ChannelHandlerContext ctx, Object msg) {
    // 注意，這裏client發送來的就是連接當前Server的子channel
    final Channel child = (Channel) msg;

    // 初始化這個子channel
    // 對用於處理client 讀寫請求的子channel設置handler，以及添加到對應的selector中
    child.pipeline().addLast(childHandler);

    setChannelOptions(child, childOptions, logger);

    for (Entry<AttributeKey<?>, Object> e: childAttrs) {
        child.attr((AttributeKey<Object>) e.getKey()).set(e.getValue());
    }

    try {
        // 將當前子channel註冊到selector
        childGroup.register(child).addListener(new ChannelFutureListener() {
            @Override
            public void operationComplete(ChannelFuture future) throws Exception {
                if (!future.isSuccess()) {
                    forceClose(child, future.cause());
                }
            }
        });
    } catch (Throwable t) {
        forceClose(child, t);
    }
 }

channel註冊到selector上

ChannelFuture regFuture = config().group().register(channel); 將當前channel註冊給selector,這裏指的是parentGroup

【MultithreadEventLoopGroup.class】
@Override
public ChannelFuture register(Channel channel) {
    // 從parentGroup中根據算法選擇一個eventLoop來完成註冊
    return next().register(channel);
}

【AbstractChannel.class】中的【AbstractUnsafe.class】#register
public final void register(EventLoop eventLoop, final ChannelPromise promise) {
            if (eventLoop == null) {
                throw new NullPointerException("eventLoop");
            }
            if (isRegistered()) {
                promise.setFailure(new IllegalStateException("registered to an event loop already"));
                return;
            }
            if (!isCompatible(eventLoop)) {
                promise.setFailure(
                        new IllegalStateException("incompatible event loop type: " + eventLoop.getClass().getName()));
                return;
            }

            // 這裏實現了channel與eventLoop的綁定
            AbstractChannel.this.eventLoop = eventLoop;

            // 判斷當前正在執行的線程是否是當前eventLoop所綁定的線程
            if (eventLoop.inEventLoop()) {
                // 若當前線程是eventLoop綁定線程，則直接讓這個線程來完成註冊操作
                register0(promise);
            } else {
                // 當前線程不是eventLoop綁定線程，則首先會創建一個線程，
                // 然後使用這個新創建的eventLoop線程來完成註冊
                try {
                    eventLoop.execute(new Runnable() {
                        @Override
                        public void run() {
                            register0(promise);
                        }
                    });
                } catch (Throwable t) {
                    logger.warn(
                            "Force-closing a channel whose registration task was not accepted by an event loop: {}",
                            AbstractChannel.this, t);
                    closeForcibly();
                    closeFuture.setClosed();
                    safeSetFailure(promise, t);
                }
            }
}

private void register0(ChannelPromise promise) {
            try {
                // check if the channel is still open as it could be closed in the mean time when the register
                // call was outside of the eventLoop
                if (!promise.setUncancellable() || !ensureOpen(promise)) {
                    return;
                }
                boolean firstRegistration = neverRegistered;
                // 完成註冊
                doRegister();
                // 修改狀態值
                neverRegistered = false;
                registered = true;

                // Ensure we call handlerAdded(...) before we actually notify the promise. This is needed as the
                // user may already fire events through the pipeline in the ChannelFutureListener.
                // 觸發handlerAdded()方法的執行
                pipeline.invokeHandlerAddedIfNeeded();

                safeSetSuccess(promise);
                // 觸發channelRegistered()方法的執行
                pipeline.fireChannelRegistered();
                // Only fire a channelActive if the channel has never been registered. This prevents firing
                // multiple channel actives if the channel is deregistered and re-registered.

                // 若當前channel是激活狀態，且是第一次註冊，
                // 則觸發channelActive()的執行
                if (isActive()) {
                    if (firstRegistration) {
                        pipeline.fireChannelActive();
                    } else if (config().isAutoRead()) {
                        // This channel was registered before and autoRead() is set. This means we need to begin read
                        // again so that we process inbound data.
                        //
                        // See https://github.com/netty/netty/issues/4805
                        beginRead();
                    }
                }
            } catch (Throwable t) {
                // Close the channel directly to avoid FD leak.
                closeForcibly();
                closeFuture.setClosed();
                safeSetFailure(promise, t);
            }
  }

doRegister() 跟進去

【AbstractNioChannel】#doRegister
protected void doRegister() throws Exception {
        boolean selected = false;
        for (;;) {
            try {
                // 其實netty的channel的註冊，本質上是原生的nio的channel的註冊
                selectionKey = javaChannel().register(eventLoop().unwrappedSelector(), 0, this);
                return;
            } catch (CancelledKeyException e) {
                if (!selected) {
                    // Force the Selector to select now as the "canceled" SelectionKey may still be
                    // cached and not removed because no Select.select(..) operation was called yet.
                    eventLoop().selectNow();
                    selected = true;
                } else {
                    // We forced a select operation on the selector before but the SelectionKey is still cached
                    // for whatever reason. JDK bug ?
                    throw e;
                }
            }
        }
 }

在執行bind之前，先執行了channel(NioServerSocketChannel.class)
【AbstractBootstrap】#bind
	|--dobind
		|-- final ChannelFuture regFuture = initAndRegister(); 創建、初始化channel，並將其註冊到selector
			|-- channel = channelFactory.newChannel(); //創建一個channel
				|--【ReflectiveChannelFactory】#newChannel
					|--constructor.newInstance(); 使用反射機制 調用其無參構造器 在調用之前，這裏其實就是先執行了NioServerSocketChannel.class 這個方法，創建了

NioEventLoop分析

NioEventLoopGroup與NioEventLoop

​ 兩個類都最後都繼承自Executor。所以都是一個執行器

​ NioEventLoopGroup本身是個執行器(Executor),裏面還包含了executor，可以調用execute，是線程池實現的execute

​ NioEventLoop本身是個執行器(Executor),裏面還包含了executor，子executor裏面又包含了線程，這個線程是總的executor綁定的factory創建的，可以調用execute，是線程實現的execute

​ EventLoop綁定的線程是在什麼時候創建的？執行註冊的時候創建的

NioEventLoopGroup創建

new NioEventLoopGroup 跟進去

跳轉到 MultithreadEventLoopGroup

private static final int DEFAULT_EVENT_LOOP_THREADS;
static {
        // 該默認值爲當前主機邏輯處理器數量的2倍
        DEFAULT_EVENT_LOOP_THREADS = Math.max(1, SystemPropertyUtil.getInt(
                "io.netty.eventLoopThreads", NettyRuntime.availableProcessors() * 2));

        if (logger.isDebugEnabled()) {
            logger.debug("-Dio.netty.eventLoopThreads: {}", DEFAULT_EVENT_LOOP_THREADS);
        }
}
protected MultithreadEventLoopGroup(int nThreads, Executor executor, Object... args) {
        super(nThreads == 0 ? DEFAULT_EVENT_LOOP_THREADS : nThreads, executor, args);
}


protected MultithreadEventExecutorGroup(int nThreads, Executor executor,
                                            EventExecutorChooserFactory chooserFactory, Object... args) {
        if (nThreads <= 0) {
            throw new IllegalArgumentException(String.format("nThreads: %d (expected: > 0)", nThreads));
        }

        if (executor == null) {
            // 創建一個總executor，這個executor將來會爲每一個EventLoop創建一個子executor，
            // 然後再使用當前這個總executor所綁定的線程Factory爲每個子executor再創建一個線程
            // 與這個子executor綁定
            executor = new ThreadPerTaskExecutor(newDefaultThreadFactory());
        }

        // 使用一個數組來存放當前group中所包含的eventLoop
        children = new EventExecutor[nThreads];

        for (int i = 0; i < nThreads; i ++) {
            boolean success = false;
            try {
                // 創建每一個eventLoop實例，並初始化到相應的數組元素中
                children[i] = newChild(executor, args);
                success = true;
            } catch (Exception e) {
                // TODO: Think about if this is a good exception type
                throw new IllegalStateException("failed to create a child event loop", e);
            } finally {
                if (!success) {
                    for (int j = 0; j < i; j ++) {
                        children[j].shutdownGracefully();
                    }

                    for (int j = 0; j < i; j ++) {
                        EventExecutor e = children[j];
                        try {
                            while (!e.isTerminated()) {
                                e.awaitTermination(Integer.MAX_VALUE, TimeUnit.SECONDS);
                            }
                        } catch (InterruptedException interrupted) {
                            // Let the caller handle the interruption.
                            Thread.currentThread().interrupt();
                            break;
                        }
                    }
                }
            }
        }

        // 創建一個EventLoop數組元素的選擇器
        chooser = chooserFactory.newChooser(children);

        final FutureListener<Object> terminationListener = new FutureListener<Object>() {
            @Override
            public void operationComplete(Future<Object> future) throws Exception {
                if (terminatedChildren.incrementAndGet() == children.length) {
                    terminationFuture.setSuccess(null);
                }
            }
        };

        for (EventExecutor e: children) {
            e.terminationFuture().addListener(terminationListener);
        }

        Set<EventExecutor> childrenSet = new LinkedHashSet<EventExecutor>(children.length);
        Collections.addAll(childrenSet, children);
        readonlyChildren = Collections.unmodifiableSet(childrenSet);
}

newDefaultThreadFactory()跟進去

【DefaultThreadFactory.class】
public DefaultThreadFactory(String poolName, boolean daemon, int priority, ThreadGroup threadGroup) {
        if (poolName == null) {
            throw new NullPointerException("poolName");
        }
        if (priority < Thread.MIN_PRIORITY || priority > Thread.MAX_PRIORITY) {
            throw new IllegalArgumentException(
                    "priority: " + priority + " (expected: Thread.MIN_PRIORITY <= priority <= Thread.MAX_PRIORITY)");
        }
        // 最終獲取到的prefix的值爲    nioEventLoopGroup-線程池id-
        prefix = poolName + '-' + poolId.incrementAndGet() + '-';
        this.daemon = daemon;
        this.priority = priority;
        this.threadGroup = threadGroup;
    }

newChild(executor, args); 跟進去

【NioEventLoopGroup】#newChild
@Override
    protected EventLoop newChild(Executor executor, Object... args) throws Exception {
        return new NioEventLoop(this, executor, (SelectorProvider) args[0],
            ((SelectStrategyFactory) args[1]).newSelectStrategy(), (RejectedExecutionHandler) args[2]);
}

調用NioEventLoop 構造器

【NioEventLoop】
NioEventLoop(NioEventLoopGroup parent, Executor executor, SelectorProvider selectorProvider,
                 SelectStrategy strategy, RejectedExecutionHandler rejectedExecutionHandler) {
        super(parent, executor, false, DEFAULT_MAX_PENDING_TASKS, rejectedExecutionHandler);
        if (selectorProvider == null) {
            throw new NullPointerException("selectorProvider");
        }
        if (strategy == null) {
            throw new NullPointerException("selectStrategy");
        }
        provider = selectorProvider;
        // 創建一個Selector元組
        final SelectorTuple selectorTuple = openSelector();
        selector = selectorTuple.selector;
        unwrappedSelector = selectorTuple.unwrappedSelector;
        selectStrategy = strategy;
}

super 跟進去調用SingleThreadEventLoop 構造器

【SingleThreadEventLoop】
protected SingleThreadEventLoop(EventLoopGroup parent, Executor executor,
                                    boolean addTaskWakesUp, int maxPendingTasks,
                                    RejectedExecutionHandler rejectedExecutionHandler) {
        super(parent, executor, addTaskWakesUp, maxPendingTasks, rejectedExecutionHandler);
        // 創建一個尾部任務隊列，收尾任務存放在這裏
        tailTasks = newTaskQueue(maxPendingTasks);
}

super 跟進去調用SingleThreadEventExecutor構造器

protected SingleThreadEventExecutor(EventExecutorGroup parent, Executor executor,
                                        boolean addTaskWakesUp, int maxPendingTasks,
                                        RejectedExecutionHandler rejectedHandler) {
        super(parent);
        this.addTaskWakesUp = addTaskWakesUp;
        this.maxPendingTasks = Math.max(16, maxPendingTasks);
        // 使用總的executor爲當前eventLoop創建一個executor  this.executor是eventLoop中的executor，executor是總的
        this.executor = ThreadExecutorMap.apply(executor, this);
        // 創建一個任務隊列
        taskQueue = newTaskQueue(this.maxPendingTasks);
        rejectedExecutionHandler = ObjectUtil.checkNotNull(rejectedHandler, "rejectedHandler");
}

ThreadExecutorMap.apply(executor, this);跟進去

【ThreadExecutorMap】
	/**
     *
     * @param executor  總的executor
     * @param eventExecutor  當前正在創建的eventLoop
     * @return
     * 線程套線程，相當於責任鏈 這時候才創建了個匿名的executor，這個new Executor現在不會執行，只有在任務開始的時候纔會執行
     */
public static Executor apply(final Executor executor, final EventExecutor eventExecutor) {
        ObjectUtil.checkNotNull(executor, "executor");
        ObjectUtil.checkNotNull(eventExecutor, "eventExecutor");
        return new Executor() {
            @Override
            public void execute(final Runnable command) {
                // executor.execute()會使用這個總的executor所綁定的線程factory創建一個線程
                executor.execute(apply(command, eventExecutor));
            }
        };
}

public static Runnable apply(final Runnable command, final EventExecutor eventExecutor) {
        ObjectUtil.checkNotNull(command, "command");
        ObjectUtil.checkNotNull(eventExecutor, "eventExecutor");
        return new Runnable() {
            @Override
            public void run() {
                // 爲了保證線程安全，這裏爲當前線程指定其所關聯的eventLoop
                setCurrentEventExecutor(eventExecutor);
                try {
                    // 真正任務的執行是在這裏開始的
                    command.run();
                } finally {
                    setCurrentEventExecutor(null);
                }
            }
        };
}

有任務的時候就去執行上面那個匿名的executor

NioEventLoop添加任務

舉例：register時候線程創建，其他的任務會直接放入隊列中等待執行

// 判斷當前正在執行的線程是否是當前eventLoop所綁定的線程
            if (eventLoop.inEventLoop()) {
                // 若當前線程是eventLoop綁定線程，則直接讓這個線程來完成註冊操作
                register0(promise);
            } else {
                // 當前線程不是eventLoop綁定線程，則首先會創建一個線程，
                // 然後使用這個新創建的eventLoop線程來完成註冊
                try {
                    eventLoop.execute(new Runnable() {
                        @Override
                        public void run() {
                            register0(promise);
                        }
                    });
                } catch (Throwable t) {
                    logger.warn(
                            "Force-closing a channel whose registration task was not accepted by an event loop: {}",
                            AbstractChannel.this, t);
                    closeForcibly();
                    closeFuture.setClosed();
                    safeSetFailure(promise, t);
                }
            }

eventLoop.execute跟進去

【SingleThreadEventExecutor】#execute
 @Override
    public void execute(Runnable task) {
        if (task == null) {
            throw new NullPointerException("task");
        }
        // 若當前線程是當前EventLoop所綁定的線程，則返回true，否則返回false
        boolean inEventLoop = inEventLoop();
        // 將任務添加到taskQueue
        addTask(task);
        if (!inEventLoop) {
            // 創建並啓動線程
            startThread();
            if (isShutdown()) {
                boolean reject = false;
                try {
                    if (removeTask(task)) {
                        reject = true;
                    }
                } catch (UnsupportedOperationException e) {
                    // The task queue does not support removal so the best thing we can do is to just move on and
                    // hope we will be able to pick-up the task before its completely terminated.
                    // In worst case we will log on termination.
                }
                if (reject) {
                    reject();
                }
            }
        }

        if (!addTaskWakesUp && wakesUpForTask(task)) {
            wakeup(inEventLoop);
        }
} startThread();

startThread();方法跟進去

private void startThread() {
        if (state == ST_NOT_STARTED) {
            // 通過CAS將狀態修改爲已啓動
            if (STATE_UPDATER.compareAndSet(this, ST_NOT_STARTED, ST_STARTED)) {
                try {
                    // 啓動線程
                    doStartThread();
                } catch (Throwable cause) {
                    STATE_UPDATER.set(this, ST_NOT_STARTED);
                    PlatformDependent.throwException(cause);
                }
            }
        }
}

private void doStartThread() {
        assert thread == null;
        // 調用當前EventLoop所包含的executor(子executor)
        executor.execute(new Runnable() {
            @Override
            public void run() {
                thread = Thread.currentThread();
                if (interrupted) {
                    thread.interrupt();
                }

                boolean success = false;
                updateLastExecutionTime();
                try {
                    // 進行selector的選擇，然後執行三類任務
                    SingleThreadEventExecutor.this.run();
                    success = true;
                } catch (Throwable t) {
                    logger.warn("Unexpected exception from an event executor: ", t);
                } finally {
                    for (;;) {
                        int oldState = state;
                        if (oldState >= ST_SHUTTING_DOWN || STATE_UPDATER.compareAndSet(
                                SingleThreadEventExecutor.this, oldState, ST_SHUTTING_DOWN)) {
                            break;
                        }
                    }

                    // Check if confirmShutdown() was called at the end of the loop.
                    if (success && gracefulShutdownStartTime == 0) {
                        if (logger.isErrorEnabled()) {
                            logger.error("Buggy " + EventExecutor.class.getSimpleName() + " implementation; " +
                                    SingleThreadEventExecutor.class.getSimpleName() + ".confirmShutdown() must " +
                                    "be called before run() implementation terminates.");
                        }
                    }

                    try {
                        // Run all remaining tasks and shutdown hooks.
                        for (;;) {
                            if (confirmShutdown()) {
                                break;
                            }
                        }
                    } finally {
                        try {
                            cleanup();
                        } finally {
                            // Lets remove all FastThreadLocals for the Thread as we are about to terminate and notify
                            // the future. The user may block on the future and once it unblocks the JVM may terminate
                            // and start unloading classes.
                            // See https://github.com/netty/netty/issues/6596.
                            FastThreadLocal.removeAll();

                            STATE_UPDATER.set(SingleThreadEventExecutor.this, ST_TERMINATED);
                            threadLock.release();
                            if (!taskQueue.isEmpty()) {
                                if (logger.isWarnEnabled()) {
                                    logger.warn("An event executor terminated with " +
                                            "non-empty task queue (" + taskQueue.size() + ')');
                                }
                            }
                            terminationFuture.setSuccess(null);
                        }
                    }
                }
            }
        });
    }

// 調用當前EventLoop所包含的executor(子executor)
executor.execute(new Runnable()

這時候就會去調用剛纔那個匿名創建的 Executor 進來

【ThreadExecutorMap】
	/**
     *
     * @param executor  總的executor
     * @param eventExecutor  當前正在創建的eventLoop
     * @return
     * 線程套線程，相當於責任鏈 這時候才創建了個匿名的executor，這個new Executor現在不會執行，只有在任務開始的時候纔會執行
     */
public static Executor apply(final Executor executor, final EventExecutor eventExecutor) {
        ObjectUtil.checkNotNull(executor, "executor");
        ObjectUtil.checkNotNull(eventExecutor, "eventExecutor");
        return new Executor() {
            @Override
            public void execute(final Runnable command) {
                // executor.execute()會使用這個總的executor所綁定的線程factory創建一個線程
                executor.execute(apply(command, eventExecutor));
            }
        };
}

最終會調用走到

【ThreadPerTaskExecutor】#execute
@Override
    public void execute(Runnable command) {
        // 這裏會創建一個線程，並且啓動這個線程，就是執行command的run()方法
        threadFactory.newThread(command).start();
}

【DefaultThreadFactory】#newThread
@Override
    public Thread newThread(Runnable r) {
        // 創建一個線程，使用線程的名稱爲  nioEventLoopGroup-線程池id-線程id
        Thread t = newThread(FastThreadLocalRunnable.wrap(r), prefix + nextId.incrementAndGet());
        try {
            if (t.isDaemon() != daemon) {
                t.setDaemon(daemon);
            }

            if (t.getPriority() != priority) {
                t.setPriority(priority);
            }
        } catch (Exception ignored) {
            // Doesn't matter even if failed to set.
        }
        return t;
    }

NioEventLoop任務執行

SingleThreadEventExecutor.this.run();

【NioEventLoop】#run
protected void run() {
        // 永久循環
        for (;;) {
            try {
                try {
                    // ------------------------- 1 selector選擇 -------------------
                    // 計算出選擇selector策略
                    switch (selectStrategy.calculateStrategy(selectNowSupplier, hasTasks())) {
                        case SelectStrategy.CONTINUE:  // NioEventLoop不支持
                            continue;

                        case SelectStrategy.BUSY_WAIT:  // Nio不支持
                            // fall-through to SELECT since the busy-wait is not supported with NIO

                        case SelectStrategy.SELECT:   // NioEventLoop支持的唯一策略
                            // 若執行這裏，說明當前任務隊列中沒有任務
                            select(wakenUp.getAndSet(false));

                            // 若當前線程剛被喚醒，selector立即將其選擇的結果返回給我們
                            if (wakenUp.get()) {
                                selector.wakeup();
                            }
                            // fall through
                        default:
                    }
                } catch (IOException e) {
                    // If we receive an IOException here its because the Selector is messed up. Let's rebuild
                    // the selector and retry. https://github.com/netty/netty/issues/8566
                    rebuildSelector0();
                    handleLoopException(e);
                    continue;
                }

                cancelledKeys = 0;
                needsToSelectAgain = false;
                // ioRatio用於控制IO處理與任務隊列中任務的處理所佔時間比例
                final int ioRatio = this.ioRatio;
                if (ioRatio == 100) {
                    try {
                        processSelectedKeys();
                    } finally {
                        // Ensure we always run tasks.
                        runAllTasks();
                    }
                } else {
                    // ------------------------- 2 處理就緒的IO -------------------
                    // IO操作的開始時間
                    final long ioStartTime = System.nanoTime();
                    try {
                        processSelectedKeys();
                    } finally {
                        // ------------------------- 3 執行任務隊列中的任務 -------------------
                        // Ensure we always run tasks.
                        // IO操作總用時
                        final long ioTime = System.nanoTime() - ioStartTime;
                        // ioTime * [(100 - ioRatio) / ioRatio]
                        runAllTasks(ioTime * (100 - ioRatio) / ioRatio);
                    }
                }
            } catch (Throwable t) {
                handleLoopException(t);
            }
            // Always handle shutdown even if the loop processing threw an exception.
            try {
                if (isShuttingDown()) {
                    closeAll();
                    if (confirmShutdown()) {
                        return;
                    }
                }
            } catch (Throwable t) {
                handleLoopException(t);
            }
        }
    }

selectStrategy.calculateStrategy(selectNowSupplier, hasTasks()) 跟進去

先看下selectNowSupplier

private final IntSupplier selectNowSupplier = new IntSupplier() {
        @Override
        public int get() throws Exception {
            // 非阻塞選擇
            return selectNow();
        }
};

int selectNow() throws IOException {
        try {
            // 非阻塞選擇
            return selector.selectNow();
        } finally {
            // restore wakeup state if needed
            // wakenUp爲true，表示當前eventLoop所綁定的線程剛剛被喚醒
            // wakenUp爲false，表示當前eventLoop所綁定的線程即將被阻塞
            if (wakenUp.get()) {
                // 立即將選擇的結果寫入到當前eventLoop的集合
                selector.wakeup();
            }
        }
}

【SingleThreadEventLoop】
@Override
protected boolean hasTasks() {
        // 判斷 taskQueue  或  tailTasks  任務隊列是否爲空
        return super.hasTasks() || !tailTasks.isEmpty();
}

calculateStrategy跟進來

【DefaultSelectStrategy】#calculateStrategy
 @Override
public int calculateStrategy(IntSupplier selectSupplier, boolean hasTasks) throws Exception {
        // 若任務隊列有任務，則馬上進行非阻塞選擇
        return hasTasks ? selectSupplier.get() : SelectStrategy.SELECT;
}

select(wakenUp.getAndSet(false));跟進來，執行這裏，說明當前任務隊列中沒有任務

private void select(boolean oldWakenUp) throws IOException {
        Selector selector = this.selector;
        try {
            // 計數器，記錄當前選擇執行的輪數
            int selectCnt = 0;
            // 獲取當前select()開始的時間點
            long currentTimeNanos = System.nanoTime();
            // delayNanos():從定時任務隊列中取出一個定時任務，計算其還有多久就要執行了
            // selectDeadLineNanos : 表示這個定時任務要開始執行的時間點
            long selectDeadLineNanos = currentTimeNanos + delayNanos(currentTimeNanos);

            for (;;) {
                // --------------------- 1 處理定時任務 ------------------
                // 對於馬上就要到執行時間的定時任務，立即進行選擇
                long timeoutMillis = (selectDeadLineNanos - currentTimeNanos + 500000L) / 1000000L;
                if (timeoutMillis <= 0) {
                    if (selectCnt == 0) {
                        // 非阻塞選擇
                        selector.selectNow();
                        selectCnt = 1;
                    }
                    break;
                }

                // --------------------- 2 在選擇期間，任務隊列中有新任務加入 ------------------
                // If a task was submitted when wakenUp value was true, the task didn't get a chance to call
                // Selector#wakeup. So we need to check task queue again before executing select operation.
                // If we don't, the task might be pended until select operation was timed out.
                // It might be pended until idle timeout if IdleStateHandler existed in pipeline.
                if (hasTasks() && wakenUp.compareAndSet(false, true)) {
                    // 非阻塞選擇 只要有通道就緒就立刻返回
                    selector.selectNow();
                    selectCnt = 1;
                    break;
                }

                // --------------------- 3 阻塞式選擇 ------------------
                // select()方法結束的條件：
                // 1)有channel被選擇
                // 2)seleter.wakeup()被調用
                // 3)當前線程被打斷
                // 4)阻塞時間超時
                // 5)其實這裏還有一個結束的條件：
                // 當長時間沒有就緒的channel時，輪詢會出現長時間空轉，從而會導致CPU佔用率飆升，
                // 此時會使select()結束
                // 注意，timeoutMillis 在這裏是作爲select()的阻塞時長的
                int selectedKeys = selector.select(timeoutMillis);
                selectCnt ++;

                if (selectedKeys != 0 || oldWakenUp || wakenUp.get() || hasTasks() || hasScheduledTasks()) {
                    // - Selected something,         有channel被選擇
                    // - waken up by user, or     或 seleter.wakeup()被調用
                    // - the task queue has a pending task.   // 任務隊列中有掛起的任務
                    // - a scheduled task is ready for processing   // 有定時任務
                    break;
                }

                // --------------------- 4 處理Nio中的Bug ------------------
                if (Thread.interrupted()) {
                    // Thread was interrupted so reset selected keys and break so we not run into a busy loop.
                    // As this is most likely a bug in the handler of the user or it's client library we will
                    // also log it.
                    //
                    // See https://github.com/netty/netty/issues/2426
                    if (logger.isDebugEnabled()) {
                        logger.debug("Selector.select() returned prematurely because " +
                                "Thread.currentThread().interrupt() was called. Use " +
                                "NioEventLoop.shutdownGracefully() to shutdown the NioEventLoop.");
                    }
                    selectCnt = 1;
                    break;
                }


                // 代碼走到這裏，說明select()結束的條件是4)或5)

                // 記錄當前時間
                long time = System.nanoTime();
                // 下面的式子等價於：
                // time - currentTimeNanos >= TimeUnit.MILLISECONDS.toNanos(timeoutMillis)
                // 當前for循環已經執行的時長 >= 阻塞時長
                // 若if的這個條件成立，說明前面的select()方法是通過條件4)結束的
                if (time - TimeUnit.MILLISECONDS.toNanos(timeoutMillis) >= currentTimeNanos) {
                    // timeoutMillis elapsed without anything selected.
                    selectCnt = 1;

                    // 執行else說明  當前for循環已經執行的時長 < 阻塞時長 ，說明前面的select()是通過
                    // 條件5)結束的。若空轉次數大於等於指定的閾值512，則重新構建selector
                } else if (SELECTOR_AUTO_REBUILD_THRESHOLD > 0 &&
                        selectCnt >= SELECTOR_AUTO_REBUILD_THRESHOLD) {
                    // The code exists in an extra method to ensure the method is not too big to inline as this
                    // branch is not very likely to get hit very frequently.
                    selector = selectRebuildSelector(selectCnt);
                    selectCnt = 1;
                    break;
                }

                currentTimeNanos = time;
            }

            if (selectCnt > MIN_PREMATURE_SELECTOR_RETURNS) {
                if (logger.isDebugEnabled()) {
                    logger.debug("Selector.select() returned prematurely {} times in a row for Selector {}.",
                            selectCnt - 1, selector);
                }
            }
        } catch (CancelledKeyException e) {
            if (logger.isDebugEnabled()) {
                logger.debug(CancelledKeyException.class.getSimpleName() + " raised by a Selector {} - JDK bug?",
                        selector, e);
            }
            // Harmless exception - log anyway
        }
}

processSelectedKeys 跟進來

private void processSelectedKeys() {
    // 若selectedKeys是優化過的
    if (selectedKeys != null) {
        // 優化的
        processSelectedKeysOptimized();
    } else {
        // 一般的
        processSelectedKeysPlain(selector.selectedKeys());
    }
}

private void processSelectedKeysOptimized() {
    for (int i = 0; i < selectedKeys.size; ++i) {
        final SelectionKey k = selectedKeys.keys[i];
        // null out entry in the array to allow to have it GC'ed once the Channel close
        // See https://github.com/netty/netty/issues/2363
        // 其就相當於對set集合處理時，要將處理過的key從set集合中刪除是一樣的，
        // 爲了避免對key的重複處理
        selectedKeys.keys[i] = null;

        // 對於NioEventLoop，key中的附件attachement中存放的是當前key所關聯的NioChannel
        final Object a = k.attachment();

        if (a instanceof AbstractNioChannel) {
            // 處理當前遍歷的key
            processSelectedKey(k, (AbstractNioChannel) a);
        } else {
            @SuppressWarnings("unchecked")
            NioTask<SelectableChannel> task = (NioTask<SelectableChannel>) a;
            processSelectedKey(k, task);
        }

        if (needsToSelectAgain) {
            // null out entries in the array to allow to have it GC'ed once the Channel close
            // See https://github.com/netty/netty/issues/2363
            selectedKeys.reset(i + 1);

            selectAgain();
            i = -1;
        }
    }
}

private void processSelectedKey(SelectionKey k, AbstractNioChannel ch) {
    final AbstractNioChannel.NioUnsafe unsafe = ch.unsafe();
    // 處理key失效的情況
    if (!k.isValid()) {
        final EventLoop eventLoop;
        try {
            eventLoop = ch.eventLoop();
        } catch (Throwable ignored) {
            // If the channel implementation throws an exception because there is no event loop, we ignore this
            // because we are only trying to determine if ch is registered to this event loop and thus has authority
            // to close ch.
            return;
        }
        // Only close ch if ch is still registered to this EventLoop. ch could have deregistered from the event loop
        // and thus the SelectionKey could be cancelled as part of the deregistration process, but the channel is
        // still healthy and should not be closed.
        // See https://github.com/netty/netty/issues/5125
        if (eventLoop != this || eventLoop == null) {
            return;
        }
        // close the channel if the key is not valid anymore
        unsafe.close(unsafe.voidPromise());
        return;
    }

    try {
        // 獲取到當前key所有就緒的操作
        int readyOps = k.readyOps();
        // We first need to call finishConnect() before try to trigger a read(...) or write(...) as otherwise
        // the NIO JDK channel implementation may throw a NotYetConnectedException.
        // 若就緒操作中包含連接操作，處理連接就緒
        if ((readyOps & SelectionKey.OP_CONNECT) != 0) {
            // remove OP_CONNECT as otherwise Selector.select(..) will always return without blocking
            // See https://github.com/netty/netty/issues/924
            int ops = k.interestOps();
            ops &= ~SelectionKey.OP_CONNECT;
            k.interestOps(ops);
            // 進行連接
            unsafe.finishConnect();
        }

        // Process OP_WRITE first as we may be able to write some queued buffers and so free memory.
        // 處理寫就緒
        // 當將數據寫入到buffer，那麼當前channel就處於寫就緒
        if ((readyOps & SelectionKey.OP_WRITE) != 0) {
            // Call forceFlush which will also take care of clear the OP_WRITE once there is nothing left to write
            ch.unsafe().forceFlush();
        }

        // Also check for readOps of 0 to workaround possible JDK bug which may otherwise lead
        // to a spin loop
        // 處理讀就緒  或  接收連接就緒
        if ((readyOps & (SelectionKey.OP_READ | SelectionKey.OP_ACCEPT)) != 0 || readyOps == 0) {
            unsafe.read();
        }
    } catch (CancelledKeyException ignored) {
        unsafe.close(unsafe.voidPromise());
    }
}

runAllTasks 跟進來

【SingleThreadEventExecutor】#runAllTasks
protected boolean runAllTasks(long timeoutNanos) {
    // // 將所有定時任務隊列中的任務添加到taskQueue
    fetchFromScheduledTaskQueue();
    // 從taskQueue中獲取一個任務
    Runnable task = pollTask();
    // 若該任務爲null，說明當前任務隊列中沒有任務了，
    // 此時執行tailTasks中的收尾任務
    if (task == null) {
        afterRunningAllTasks();
        return false;
    }

    // 計算taskQueue中所有任務執行完畢的時間
    final long deadline = ScheduledFutureTask.nanoTime() + timeoutNanos;
    long runTasks = 0;
    long lastExecutionTime;
    // 遍歷執行taskQueue中的所有任務
    for (;;) {
        // 執行當前遍歷的任務
        safeExecute(task);

        runTasks ++;

        // Check timeout every 64 tasks because nanoTime() is relatively expensive.
        // XXX: Hard-coded value - will make it configurable if it is really a problem.
        // 每64次任務檢查一次超時
        if ((runTasks & 0x3F) == 0) {
            lastExecutionTime = ScheduledFutureTask.nanoTime();
            if (lastExecutionTime >= deadline) {
                break;
            }
        }

        task = pollTask();
        if (task == null) {
            lastExecutionTime = ScheduledFutureTask.nanoTime();
            break;
        }
    }  // end-for

    // 執行tailTasks中的收尾任務
    afterRunningAllTasks();
    this.lastExecutionTime = lastExecutionTime;
    return true;
}

fetchFromScheduledTaskQueue跟進去

// 將所有定時任務隊列中的任務添加到taskQueue
private boolean fetchFromScheduledTaskQueue() {
    // 獲取當前時間相對於定時任務實例創建時間的時長(定時任務實例已經存活了多久)
    long nanoTime = AbstractScheduledEventExecutor.nanoTime();
    // 從定時任務隊列中取出一個最緊急的任務
    Runnable scheduledTask  = pollScheduledTask(nanoTime);
    while (scheduledTask != null) {
        // 在定時任務不空的前提下，將任務添加到taskQueue
        if (!taskQueue.offer(scheduledTask)) {
            // No space left in the task queue add it back to the scheduledTaskQueue so we pick it up again.
            // 若沒有添加成功，則重新放回到定時任務隊列
            scheduledTaskQueue().add((ScheduledFutureTask<?>) scheduledTask);
            return false;
        }
        // 從定時任務隊列中再取出一個最緊急的任務
        scheduledTask  = pollScheduledTask(nanoTime);
    }
    return true;
}
【AbstractScheduledEventExecutor】#pollScheduledTask
protected final Runnable pollScheduledTask(long nanoTime) {
        assert inEventLoop();
        // 從定時任務隊列中取一個任務
        Queue<ScheduledFutureTask<?>> scheduledTaskQueue = this.scheduledTaskQueue;
        ScheduledFutureTask<?> scheduledTask = scheduledTaskQueue == null ?
                                        null : scheduledTaskQueue.peek();
        if (scheduledTask == null) {
            return null;
        }

        // 若配置的需要推遲的時間比當前的時間還要小，說明這個任務早就應該執行了
        if (scheduledTask.deadlineNanos() <= nanoTime) {
            // 從定時任務隊列中刪除該任務
            scheduledTaskQueue.remove();
            // 返回該任務，以將其添加到taskQUeue去執行
            return scheduledTask;
        }
        return null;
}

pollTask();跟進去

protected static Runnable pollTaskFrom(Queue<Runnable> taskQueue) {
    // 從任務隊列中取出一個任務，只要其不是一個喚醒任務，則直接返回
    for (;;) {
        Runnable task = taskQueue.poll();
        if (task == WAKEUP_TASK) {
            continue;
        }
        return task;
    }
}

afterRunningAllTasks();跟進來

【SingleThreadEventExecutor】#runAllTasks
protected final boolean runAllTasksFrom(Queue<Runnable> taskQueue) {
        // 從任務隊列中獲取一個任務
        Runnable task = pollTaskFrom(taskQueue);
        if (task == null) {
            return false;
        }
        for (;;) {
            safeExecute(task);
            task = pollTaskFrom(taskQueue);
            // task爲null，說明taskQueue中的任務全部執行完畢
            if (task == null) {
                return true;
            }
        }
}
protected static void safeExecute(Runnable task) {
        try {
            // 任務的run()最終在這裏執行了
            task.run();
        } catch (Throwable t) {
            logger.warn("A task raised an exception. Task: {}", task, t);
        }
}