目錄
一、概述
上一篇文章已經分析了Dubbo如何通過netty技術綁定provider服務端口，處理請求的數據invocation，並調用本地的invoker進行響應。接下來我們分析consumer消費端如何通過netty發送invocation請求對象的。
二、Exchangers
ReferenceBean初始化過程中，DubboProtocol的initClient方法中，會調用Exchangers的connect方法創建Client。此方法對url和handler進行校驗，然後調用getExchanger(url)。
public static ExchangeClient connect(URL url, ExchangeHandler handler) throws RemotingException {
if (url == null) {
throw new IllegalArgumentException("url == null");
}
if (handler == null) {
throw new IllegalArgumentException("handler == null");
}
url = url.addParameterIfAbsent(Constants.CODEC_KEY, "exchange");
return getExchanger(url).connect(url, handler);
}
getExchanger方法通過url中的exchange參數，獲取擴展的Exchanger實現類，默認爲HeaderExchanger。
public static Exchanger getExchanger(URL url) {
String type = url.getParameter(Constants.EXCHANGER_KEY, Constants.DEFAULT_EXCHANGER);
return getExchanger(type);
}
public static Exchanger getExchanger(String type) {
return ExtensionLoader.getExtensionLoader(Exchanger.class).getExtension(type);
}
三、HeaderExchanger
HeaderExchanger的connect方法，啓動netty或mina客戶端，連接端口，進行rpc調用。 此connect方法對handler進行HeaderExchange、解碼處理包裝，然後調用Transporters的connect靜態方法 創建Client。
public class HeaderExchanger implements Exchanger {
public static final String NAME = "header";
@Override
public ExchangeClient connect(URL url, ExchangeHandler handler) throws RemotingException {
return new HeaderExchangeClient(Transporters.connect(url, new DecodeHandler(new HeaderExchangeHandler(handler))), true);
}
@Override
public ExchangeServer bind(URL url, ExchangeHandler handler) throws RemotingException {
return new HeaderExchangeServer(Transporters.bind(url, new DecodeHandler(new HeaderExchangeHandler(handler))));
}
}
四、Transporters
Transporters類中都是靜態方法，可以看做是Transporter的工具類。connect方法中如果變參handlers有多個，會組裝成ChannelHandlerDispatcher類進行handler依次循環處理。然後調用getTransporter方法。
public static Client connect(URL url, ChannelHandler... handlers) throws RemotingException {
if (url == null) {
throw new IllegalArgumentException("url == null");
}
ChannelHandler handler;
if (handlers == null || handlers.length == 0) {
handler = new ChannelHandlerAdapter();
} else if (handlers.length == 1) {
handler = handlers[0];
} else {
handler = new ChannelHandlerDispatcher(handlers);
}
return getTransporter().connect(url, handler);
}
getTransporter獲取Transporter的擴展實現類，此處爲NettyTransporter。
public static Transporter getTransporter() {
return ExtensionLoader.getExtensionLoader(Transporter.class).getAdaptiveExtension();
}
五、NettyTransporter
NettyTransporter實現Transporter，connect創建NettyClient。
public class NettyTransporter implements Transporter {
public static final String NAME = "netty";
@Override
public Server bind(URL url, ChannelHandler listener) throws RemotingException {
return new NettyServer(url, listener);
}
@Override
public Client connect(URL url, ChannelHandler listener) throws RemotingException {
return new NettyClient(url, listener);
}
}
六、NettyClient
NettyClient繼承AbstractClient類，構造方法，調用父類的構造方法，啓動初始化netty並連接。
public class NettyClient extends AbstractClient {
private volatile Channel channel;
public NettyClient(final URL url, final ChannelHandler handler) throws RemotingException {
// you can customize name and type of client thread pool by THREAD_NAME_KEY and THREADPOOL_KEY in CommonConstants.
// the handler will be warped: MultiMessageHandler->HeartbeatHandler->handler
super(url, wrapChannelHandler(url, handler));
}
}
AbstractClient構造函數如下
public AbstractClient(URL url, ChannelHandler handler) throws RemotingException {
// 設置url和handler
super(url, handler);
// 初始化是否重連參數
needReconnect = url.getParameter(Constants.SEND_RECONNECT_KEY, false);
try {
// 調用子類方法打開連接
doOpen();
} catch (Throwable t) {
// 報錯調用關閉close方法
close();
throw new RemotingException(url.toInetSocketAddress(), null,
"Failed to start " + getClass().getSimpleName() + " " + NetUtils.getLocalAddress()
+ " connect to the server " + getRemoteAddress() + ", cause: " + t.getMessage(), t);
}
try {
// 連接服務器
connect();
if (logger.isInfoEnabled()) {
logger.info("Start " + getClass().getSimpleName() + " " + NetUtils.getLocalAddress() + " connect to the server " + getRemoteAddress());
}
} catch (RemotingException t) {
if (url.getParameter(Constants.CHECK_KEY, true)) {
// 如果check爲true，則不重試連接，直接關閉，拋出異常
close();
throw t;
} else {
logger.warn("Failed to start " + getClass().getSimpleName() + " " + NetUtils.getLocalAddress()
+ " connect to the server " + getRemoteAddress() + " (check == false, ignore and retry later!), cause: " + t.getMessage(), t);
}
} catch (Throwable t) {
// 報錯調用關閉close方法
close();
throw new RemotingException(url.toInetSocketAddress(), null,
"Failed to start " + getClass().getSimpleName() + " " + NetUtils.getLocalAddress()
+ " connect to the server " + getRemoteAddress() + ", cause: " + t.getMessage(), t);
}
// 初始化executor
executor = (ExecutorService) ExtensionLoader.getExtensionLoader(DataStore.class)
.getDefaultExtension().get(CONSUMER_SIDE, Integer.toString(url.getPort()));
ExtensionLoader.getExtensionLoader(DataStore.class)
.getDefaultExtension().remove(CONSUMER_SIDE, Integer.toString(url.getPort()));
}
下面我們查看NettyClient的doOpen方法
protected void doOpen() throws Throwable {
// 創建NettyClientHandler
final NettyClientHandler nettyClientHandler = new NettyClientHandler(getUrl(), this);
// 創建netty的Bootstrap
bootstrap = new Bootstrap();
// 設置相關參數屬性
bootstrap.group(nioEventLoopGroup)
.option(ChannelOption.SO_KEEPALIVE, true)
.option(ChannelOption.TCP_NODELAY, true)
.option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT)
//.option(ChannelOption.CONNECT_TIMEOUT_MILLIS, getTimeout())
.channel(NioSocketChannel.class);
// 設置連接超時時間
if (getConnectTimeout() < 3000) {
bootstrap.option(ChannelOption.CONNECT_TIMEOUT_MILLIS, 3000);
} else {
bootstrap.option(ChannelOption.CONNECT_TIMEOUT_MILLIS, getConnectTimeout());
}
// 設置netty的ChannelHandler
bootstrap.handler(new ChannelInitializer() {
@Override
protected void initChannel(Channel ch) throws Exception {
// 獲取心跳間隔時間
int heartbeatInterval = UrlUtils.getHeartbeat(getUrl());
NettyCodecAdapter adapter = new NettyCodecAdapter(getCodec(), getUrl(), NettyClient.this);
ch.pipeline()//.addLast("logging",new LoggingHandler(LogLevel.INFO))//for debug
// 添加解碼器
.addLast("decoder", adapter.getDecoder())
// 添加解碼器
.addLast("encoder", adapter.getEncoder())
// 添加連接空閒處理器，如果在心跳間隔指定時間內，chanel處於idle狀態沒有發送心跳，則發送
// IdleState事件，服務器收到事件時就會關閉channel連接
.addLast("client-idle-handler", new IdleStateHandler(heartbeatInterval, 0, 0, MILLISECONDS))
//添加nettyClientHandler處理器
.addLast("handler", nettyClientHandler);
// 如果參數配置了socks代理host,那麼在通道最開始添加socks5ProxyHandler處理器
String socksProxyHost = ConfigUtils.getProperty(SOCKS_PROXY_HOST);
if(socksProxyHost != null) {
int socksProxyPort = Integer.parseInt(ConfigUtils.getProperty(SOCKS_PROXY_PORT, DEFAULT_SOCKS_PROXY_PORT));
Socks5ProxyHandler socks5ProxyHandler = new Socks5ProxyHandler(new InetSocketAddress(socksProxyHost, socksProxyPort));
ch.pipeline().addFirst(socks5ProxyHandler);
}
}
});
}
連接時會調用NettyClient的doConnect方法
protected void doConnect() throws Throwable {
long start = System.currentTimeMillis();
// 調用netty的bootstrap客戶端連接
ChannelFuture future = bootstrap.connect(getConnectAddress());
try {
// 獲取超時時間
boolean ret = future.awaitUninterruptibly(getConnectTimeout(), MILLISECONDS);
// 在超時時間內，返回且future成功了，則創建新的channel，同時關閉oldChannel
if (ret && future.isSuccess()) {
Channel newChannel = future.channel();
try {
// Close old channel
// copy reference
Channel oldChannel = NettyClient.this.channel;
if (oldChannel != null) {
try {
if (logger.isInfoEnabled()) {
logger.info("Close old netty channel " + oldChannel + " on create new netty channel " + newChannel);
}
oldChannel.close();
} finally {
NettyChannel.removeChannelIfDisconnected(oldChannel);
}
}
} finally {
// 每次連接時判斷當前client是否需要關閉，如果需要則關閉新連接
if (NettyClient.this.isClosed()) {
try {
if (logger.isInfoEnabled()) {
logger.info("Close new netty channel " + newChannel + ", because the client closed.");
}
newChannel.close();
} finally {
NettyClient.this.channel = null;
NettyChannel.removeChannelIfDisconnected(newChannel);
}
} else {
NettyClient.this.channel = newChannel;
}
}
} else if (future.cause() != null) {
throw new RemotingException(this, "client(url: " + getUrl() + ") failed to connect to server "
+ getRemoteAddress() + ", error message is:" + future.cause().getMessage(), future.cause());
} else {
throw new RemotingException(this, "client(url: " + getUrl() + ") failed to connect to server "
+ getRemoteAddress() + " client-side timeout "
+ getConnectTimeout() + "ms (elapsed: " + (System.currentTimeMillis() - start) + "ms) from netty client "
+ NetUtils.getLocalHost() + " using dubbo version " + Version.getVersion());
}
} finally {
// just add new valid channel to NettyChannel's cache
if (!isConnected()) {
//future.cancel(true);
}
}
}
接下來我們回到doOpen時在channel通道的最後加上的NettyClientHandler處理器，它繼承netty提供的ChannelDuplexHandler處理器。
public class NettyClientHandler extends ChannelDuplexHandler {
private static final Logger logger = LoggerFactory.getLogger(NettyClientHandler.class);
private final URL url;
private final ChannelHandler handler;
public NettyClientHandler(URL url, ChannelHandler handler) {
if (url == null) {
throw new IllegalArgumentException("url == null");
}
if (handler == null) {
throw new IllegalArgumentException("handler == null");
}
this.url = url;
this.handler = handler;
}
// channel激活時調用，添加至NettyChannel緩存中，調用handler的connected方法
// 如果連接失敗，從緩存中刪除
public void channelActive(ChannelHandlerContext ctx) throws Exception {
NettyChannel channel = NettyChannel.getOrAddChannel(ctx.channel(), url, handler);
try {
handler.connected(channel);
} finally {
NettyChannel.removeChannelIfDisconnected(ctx.channel());
}
}
// channel失聯時調用，調用handler的disconnected方法
public void channelInactive(ChannelHandlerContext ctx) throws Exception {
NettyChannel channel = NettyChannel.getOrAddChannel(ctx.channel(), url, handler);
try {
handler.disconnected(channel);
} finally {
NettyChannel.removeChannelIfDisconnected(ctx.channel());
}
}
@Override
public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
NettyChannel channel = NettyChannel.getOrAddChannel(ctx.channel(), url, handler);
try {
handler.received(channel, msg);
} finally {
NettyChannel.removeChannelIfDisconnected(ctx.channel());
}
}
// channel寫數據時調用
public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) throws Exception {
// 調用父類想chanel寫數據
super.write(ctx, msg, promise);
// 獲取緩存channel或添加至緩存
final NettyChannel channel = NettyChannel.getOrAddChannel(ctx.channel(), url, handler);
// 判斷髮送對象是否是Request
final boolean isRequest = msg instanceof Request;
// 向通道添加監聽，如果寫數據成功，則調用handler的sent方法
promise.addListener(future -> {
try {
if (future.isSuccess()) {
// if our future is success, mark the future to sent.
handler.sent(channel, msg);
return;
}
// 如果失敗且請求對象爲Request，則調用handler的received方法
Throwable t = future.cause();
if (t != null && isRequest) {
Request request = (Request) msg;
Response response = buildErrorResponse(request, t);
handler.received(channel, response);
}
} finally {
NettyChannel.removeChannelIfDisconnected(ctx.channel());
}
});
}
// 當讀到idle時發送heartbeat事件
public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception {
if (evt instanceof IdleStateEvent) {
try {
NettyChannel channel = NettyChannel.getOrAddChannel(ctx.channel(), url, handler);
if (logger.isDebugEnabled()) {
logger.debug("IdleStateEvent triggered, send heartbeat to channel " + channel);
}
Request req = new Request();
req.setVersion(Version.getProtocolVersion());
req.setTwoWay(true);
req.setEvent(Request.HEARTBEAT_EVENT);
channel.send(req);
} finally {
NettyChannel.removeChannelIfDisconnected(ctx.channel());
}
} else {
super.userEventTriggered(ctx, evt);
}
}
// 發送異常時調用handler的caught方法
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause)
throws Exception {
NettyChannel channel = NettyChannel.getOrAddChannel(ctx.channel(), url, handler);
try {
handler.caught(channel, cause);
} finally {
NettyChannel.removeChannelIfDisconnected(ctx.channel());
}
}
/**
* build a bad request's response
*
* @param request the request
* @param t the throwable. In most cases, serialization fails.
* @return the response
*/
private static Response buildErrorResponse(Request request, Throwable t) {
Response response = new Response(request.getId(), request.getVersion());
response.setStatus(Response.BAD_REQUEST);
response.setErrorMessage(StringUtils.toString(t));
return response;
}
}
那麼這個handler是什麼了，我們回到文章最開始的DubboProtocol的initClient方法，Exchangers.connect(url, requestHandler)，傳入的是requestHandler，有關它的相關內容在第一篇文章已經分析了，這裏不再贅述。
七、HeaderExchangeClient
接下來我們回到對nettyClient包裝的HeaderExchangeClient，構造函數會包裝當前client爲HeaderExchangeChannel，send方法調用HeaderExchangeChannel的send方法，並且startTimer爲true，啓動自動重連任務和心跳任務。
public class HeaderExchangeClient implements ExchangeClient {
private final Client client;
private final ExchangeChannel channel;
private static final HashedWheelTimer IDLE_CHECK_TIMER = new HashedWheelTimer(
new NamedThreadFactory("dubbo-client-idleCheck", true), 1, TimeUnit.SECONDS, TICKS_PER_WHEEL);
private HeartbeatTimerTask heartBeatTimerTask;
private ReconnectTimerTask reconnectTimerTask;
public HeaderExchangeClient(Client client, boolean startTimer) {
Assert.notNull(client, "Client can't be null");
this.client = client;
this.channel = new HeaderExchangeChannel(client);
if (startTimer) {
URL url = client.getUrl();
startReconnectTask(url);
startHeartBeatTask(url);
}
}
public void send(Object message) throws RemotingException {
channel.send(message);
}
@Override
public void send(Object message, boolean sent) throws RemotingException {
channel.send(message, sent);
}
// 開啓心跳任務
private void startHeartBeatTask(URL url) {
if (!client.canHandleIdle()) {
AbstractTimerTask.ChannelProvider cp = () -> Collections.singletonList(HeaderExchangeClient.this);
int heartbeat = getHeartbeat(url);
long heartbeatTick = calculateLeastDuration(heartbeat);
this.heartBeatTimerTask = new HeartbeatTimerTask(cp, heartbeatTick, heartbeat);
IDLE_CHECK_TIMER.newTimeout(heartBeatTimerTask, heartbeatTick, TimeUnit.MILLISECONDS);
}
}
// 開啓重連任務
private void startReconnectTask(URL url) {
if (shouldReconnect(url)) {
AbstractTimerTask.ChannelProvider cp = () -> Collections.singletonList(HeaderExchangeClient.this);
int idleTimeout = getIdleTimeout(url);
long heartbeatTimeoutTick = calculateLeastDuration(idleTimeout);
this.reconnectTimerTask = new ReconnectTimerTask(cp, heartbeatTimeoutTick, idleTimeout);
IDLE_CHECK_TIMER.newTimeout(reconnectTimerTask, heartbeatTimeoutTick, TimeUnit.MILLISECONDS);
}
}
// 關閉任務
private void doClose() {
if (heartBeatTimerTask != null) {
heartBeatTimerTask.cancel();
}
if (reconnectTimerTask != null) {
reconnectTimerTask.cancel();
}
}
}
八、HeaderExchangeChannel
final class HeaderExchangeChannel implements ExchangeChannel {
@Override
public void send(Object message) throws RemotingException {
send(message, false);
}
// send方法如果是對象是Request,Response,String類型則直接發送，如果不是包裝成Request發送，並設置爲單向請求模式
// 調用nettyClient發送數據
public void send(Object message, boolean sent) throws RemotingException {
if (closed) {
throw new RemotingException(this.getLocalAddress(), null, "Failed to send message " + message + ", cause: The channel " + this + " is closed!");
}
if (message instanceof Request
|| message instanceof Response
|| message instanceof String) {
channel.send(message, sent);
} else {
Request request = new Request();
request.setVersion(Version.getProtocolVersion());
request.setTwoWay(false);
request.setData(message);
channel.send(request, sent);
}
}
public CompletableFuture<Object> request(Object request) throws RemotingException {
return request(request, channel.getUrl().getPositiveParameter(TIMEOUT_KEY, DEFAULT_TIMEOUT));
}
// request方法，創建Request對象設置數據，設置twoWay爲true，需返回數據
// 調用nettyClient獲取結果
public CompletableFuture<Object> request(Object request, int timeout) throws RemotingException {
if (closed) {
throw new RemotingException(this.getLocalAddress(), null, "Failed to send request " + request + ", cause: The channel " + this + " is closed!");
}
// create request.
Request req = new Request();
req.setVersion(Version.getProtocolVersion());
req.setTwoWay(true);
req.setData(request);
DefaultFuture future = DefaultFuture.newFuture(channel, req, timeout);
try {
channel.send(req);
} catch (RemotingException e) {
future.cancel();
throw e;
}
return future;
}
}
那麼我們跟蹤代碼找到nettyClient的send方法在它的父類AbstractClient中，其中getChannel方法獲取的是子類調用子類nettyClient方法。
public void send(Object message, boolean sent) throws RemotingException {
if (needReconnect && !isConnected()) {
connect();
}
Channel channel = getChannel();
//TODO Can the value returned by getChannel() be null? need improvement.
if (channel == null || !channel.isConnected()) {
throw new RemotingException(this, "message can not send, because channel is closed . url:" + getUrl());
}
channel.send(message, sent);
}
nettyClient獲取的是對io.netty.channel包裝的NettyChannel，模板方法適配器模式， 也可以對mina客戶端的channel進行包裝適配。
protected org.apache.dubbo.remoting.Channel getChannel() {
Channel c = channel;
if (c == null || !c.isActive()) {
return null;
}
return NettyChannel.getOrAddChannel(c, getUrl(), this);
}
九、NettyChannel
nettyClient實際上是對io.netty.channel的包裝適配，轉換成Dubbo包下的Channel它的send方法如下，調用io.netty.channel的writeAndFlush方法寫數據，如果sent爲true，則等待future調用成功。
public void send(Object message, boolean sent) throws RemotingException {
if (needReconnect && !isConnected()) {
connect();
}
Channel channel = getChannel();
//TODO Can the value returned by getChannel() be null? need improvement.
if (channel == null || !channel.isConnected()) {
throw new RemotingException(this, "message can not send, because channel is closed . url:" + getUrl());
}
channel.send(message, sent);
}
nettyClient實際上是對io.netty.channel的包裝適配，轉換成Dubbo包下的Channel
它的send方法如下，調用io.netty.channel的writeAndFlush方法寫數據，如果sent爲true，則等待future調用成功
public void send(Object message, boolean sent) throws RemotingException {
// whether the channel is closed
super.send(message, sent);
boolean success = true;
int timeout = 0;
try {
ChannelFuture future = channel.writeAndFlush(message);
if (sent) {
// wait timeout ms
timeout = getUrl().getPositiveParameter(TIMEOUT_KEY, DEFAULT_TIMEOUT);
success = future.await(timeout);
}
Throwable cause = future.cause();
if (cause != null) {
throw cause;
}
} catch (Throwable e) {
throw new RemotingException(this, "Failed to send message " + message + " to " + getRemoteAddress() + ", cause: " + e.getMessage(), e);
}
if (!success) {
throw new RemotingException(this, "Failed to send message " + message + " to " + getRemoteAddress()
+ "in timeout(" + timeout + "ms) limit");
}
}