本系列的第二篇，加入netty心跳機制，只是簡單地描述瞭如何監聽不活躍的服務端，即自定義一個標記服務是否活躍的規則及空閒鏈接的監聽器，在標記爲不活躍後，客戶端應該怎麼處理，服務端再次活躍了，客戶端又如何處理，先看下這個圖
monitor負責監控server們的是否活躍，如果不活躍，修改zk上對應節點的值，client則監聽zk上的事件，當發生變動的節點的值爲不活躍，則加進不活躍的優先隊列,這裏設置爲優先隊列是用來定時去除不活躍的節的，在執行任務時，如果不活躍時間最長的節點都沒有達到不活躍最長的時長，則不進行刪除,如果重新活躍了，就把他加進活躍節點的list裏面，現在來看看如何實現。
首先看server和monitor之間的交互，新建一個類用來表示serverchannel的狀態
/**
* 用於表示serverchannel的狀態類
*/
@Data
public class ChannelStatus {
//重新活躍的時間
private volatile long reActive;
//是否活躍
private volatile boolean active = true;
//持續重新活躍的次數
private AtomicInteger reActiveCount = new AtomicInteger(0);
//持續不活躍的次數
private AtomicInteger inActiveCount = new AtomicInteger(0);
//對應的channelId
private String channelId;
//不活躍的時間
private volatile long InActive;
public ChannelStatus() {
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
ChannelStatus that = (ChannelStatus) o;
return reActive == that.reActive &&
active == that.active &&
InActive == that.InActive &&
Objects.equals(reActiveCount, that.reActiveCount) &&
Objects.equals(inActiveCount, that.inActiveCount);
}
@Override
public int hashCode() {
return Objects.hash(reActive, active, reActiveCount, inActiveCount, InActive);
}
}
監控server的類，當不活躍時候，通知zk
/**
* @author lulu
* @Date 2019/11/18 22:29
*/
public class HeartbeatHandler extends ChannelInboundHandlerAdapter {
private final static int MAX_IN_ACTIVE_COUNT = 3;
private final static int COUNT_MINUTE = 2;
private final static int MIN_RE_ACTIVE_COUNT = 3;
//維護channelId和具體地址的map，當發生變化時對其進行刪除
private ConcurrentHashMap<String, ChannelStatus> channelUrlMap;
public HeartbeatHandler(ConcurrentHashMap<String, ChannelStatus> map) {
channelUrlMap = map;
}
@Override
public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
String url = msg.toString();
String id = ctx.channel().id().asShortText();
System.out.println("收到channelId：" + id + "發來信息：" + url);
ChannelStatus status;
if ((status = channelUrlMap.get(url)) == null) {
status = new ChannelStatus();
status.setChannelId(id);
channelUrlMap.put(url, status);
} else {
//如果收到不活躍的節點重連發來的信息,
if (!status.isActive()) {
//記錄重連的心跳次數
System.out.println(url + "嘗試重連");
int i = status.getReActiveCount().incrementAndGet();
//第一次重連的話，記錄重連時間
if (i == 1) {
String s = ctx.channel().id().asShortText();
status.setChannelId(s);
status.setReActive(System.currentTimeMillis());
//如果大於最小重連心跳次數
} else if (i >= MIN_RE_ACTIVE_COUNT) {
//計算重連階段的時間
long minute = (System.currentTimeMillis() - status.getReActive()) / (1000 * 60) + 1;
//如果大於要求的時間，則是認爲活躍
if (minute >= COUNT_MINUTE) {
status.setActive(true);
status.setInActiveCount(new AtomicInteger(0));
// 通知連接池重新加入該節點
updateOrRemove(url, ctx, true, ZKConsts.REACTIVE);
}
}
}
}
}
@Override
public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception {
if (evt instanceof IdleStateEvent) {
IdleStateEvent state = (IdleStateEvent) evt;
//在一定時間內讀寫空閒纔會關閉鏈接
if (state.state().equals(IdleState.ALL_IDLE)) {
String s = ctx.channel().id().asShortText();
Integer inActiveCount = 0;
ChannelStatus channelStatus = null;
String url = null;
Object[] objects = getStatusValuesByChannelId(s);
Assert.isTrue(objects != null && objects.length > 0, "該channelId沒有東西");
inActiveCount = (Integer) objects[0];
channelStatus = (ChannelStatus) objects[1];
url = (String) objects[2];
if (inActiveCount == 1) {
channelStatus.setInActive(System.currentTimeMillis());
}
//1分鐘內出現2次以上不活躍現象，有的話就把它去掉
long minute = (System.currentTimeMillis() - channelStatus.getInActive()) / (1000 * 60) + 1;
System.out.printf("第%s次不活躍,當前分鐘%d%n", channelStatus.getInActiveCount().get(), minute);
if (inActiveCount >= MAX_IN_ACTIVE_COUNT && minute <= COUNT_MINUTE) {
System.out.println("移除不活躍的ip" + channelStatus.toString());
//設置不活躍
channelStatus.setActive(false);
updateOrRemove(url, ctx, true, ZKConsts.INACTIVE);
} else {
//重新計算,是活躍的狀態
if (minute > COUNT_MINUTE) {
// System.out.println("新週期開始");
channelStatus.setActive(true);
channelStatus.setInActive(0);
channelStatus.setInActiveCount(new AtomicInteger(0));
}
}
}
}
}
/**
* 通過channelId獲取server的狀態信息
*
* @param channelId
* @return
*/
public Object[] getStatusValuesByChannelId(String channelId) {
Iterator<Map.Entry<String, ChannelStatus>> iterator = channelUrlMap.entrySet().iterator();
Integer inActiveCount = 0;
ChannelStatus channelStatus = null;
String url = null;
System.out.println();
while (iterator.hasNext()) {
Map.Entry<String, ChannelStatus> next = iterator.next();
ChannelStatus status = next.getValue();
if (status.getChannelId().equals(channelId)) {
channelStatus = status;
url = next.getKey();
inActiveCount = channelStatus.getInActiveCount().incrementAndGet();
return new Object[]{inActiveCount, channelStatus, url};
}
}
return null;
}
/**
* 通過ID獲取地址，並刪除zk上相關的，用於心跳監聽的類
*
* @param ctx
*/
private void updateOrRemove(String url, ChannelHandlerContext ctx, Boolean update, String data) {
//移除不活躍的節點
RegisterForClient.getInstance().removeOrUpdate(url, update, data);
//如果不爲重新喚醒，則斷開連接並且做相應的通知
if (!data.equals(ZKConsts.REACTIVE)) {
channelUrlMap.get(url).setChannelId(null);
ctx.channel().close();
}
}
//當出現異常時關閉鏈接
@Override
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
Object[] values = getStatusValuesByChannelId(ctx.channel().id().asShortText());
updateOrRemove((String) values[2], ctx, false, null);
}
}
心跳發送類
/**
* @author lulu
* @Date 2019/11/18 23:30
* 服務端的發送心跳包類
*/
@Getter
public class BeatDataSender {
//狀態
private String activeStatus;
//負責定期發送心跳包的線程池
private ScheduledExecutorService service;
//失敗後重連的線程池
private ScheduledExecutorService retryConnect;
private boolean reconnect = false;
public BeatDataSender(String localAddress, String remoteIp, Integer remotePort, String serviceName) {
service = Executors.newSingleThreadScheduledExecutor();
retryConnect = Executors.newSingleThreadScheduledExecutor();
this.send(localAddress, remoteIp, remotePort, serviceName);
//如果重連了嘗試重新發送心跳包
retryConnect.scheduleAtFixedRate(() -> {
if (activeStatus == ZKConsts.INACTIVE) {
System.out.println("server嘗試重連監控器");
send(localAddress, remoteIp, remotePort, serviceName);
activeStatus = ZKConsts.REACTIVE;
reconnect = true;
}
}, 3, 3, TimeUnit.MINUTES);
}
public void close() {
this.service.shutdown();
this.retryConnect.shutdown();
}
public void send(String localAddress, String remoteIp, Integer remotePort, String serviceName) {
EventLoopGroup eventLoopGroup = new NioEventLoopGroup();
try {
Bootstrap bootstrap = new Bootstrap();
ChannelFuture connect = bootstrap.group(eventLoopGroup).channel(NioSocketChannel.class)
.handler(new ChannelInitializer<SocketChannel>() {
@Override
protected void initChannel(SocketChannel socketChannel) throws Exception {
socketChannel.pipeline().addLast(new StringEncoder())
.addLast(new StringEncoder())
.addLast(new ChannelInboundHandlerAdapter() {
@Override
public void channelInactive(ChannelHandlerContext ctx) throws Exception {
activeStatus = ZKConsts.INACTIVE;
System.out.println("由於不活躍次數在2分鐘內超過3次,鏈接被關閉");
ctx.channel().close();
}
});
}
})
.connect(remoteIp, remotePort).addListener(new ChannelFutureListener() {
@Override
public void operationComplete(ChannelFuture future) throws Exception {
if (future.isSuccess()) {
System.out.println("心跳客戶端綁定" + "hostname:" + remoteIp + "remotePort:" + remotePort);
future.channel().writeAndFlush(serviceName + "@" + localAddress);
//這裏只是演示心跳機制不活躍的情況下重連，普通的做法只需要定時發送本機地址即可
//進入重連狀態後，就穩定發送心跳包
service.scheduleAtFixedRate(() -> {
if (future.channel().isActive()) {
if (reconnect) {
future.channel().writeAndFlush(serviceName + "@" + localAddress);
}
}
}, 30, 30, TimeUnit.SECONDS);
} else {
System.out.println("3s後重連");
TimeUnit.SECONDS.sleep(3);
//重新發送
send(localAddress, remoteIp, remotePort, serviceName);
}
}
});
} catch (Exception e) {
e.printStackTrace();
}
}
}
這裏的心跳發送類主要作用是發送心跳，當處於不活躍狀態時候，就嘗試重新發送
server和monitor的交互大概就到這裏
接下來就是zk和client的交互了，client要爲每個服務下的節點設置監聽並作出相應的處理，這裏採用觀察者模式對zk和client的連接池進行解耦，這裏做了一個代碼的拆分，把client的功能(RegisterForClient)和server(RegisterForServer)的功能分開了
首先看事件發佈者的接口
/**
* @author: lele
* @date: 2019/11/22 下午3:29
*/
public interface NodeChangePublisher {
/**
* 事件標識
*/
int inactive=0;
int remove=1;
int add=2;
int reactive=3;
void addListener(NodeChangeListener listener) ;
void removeListener(NodeChangeListener listener) ;
void notifyListener(int state, String path) ;
}
註冊中心給client方的實現類
自定義線程工廠類，建造者模式
/**
* @author lulu
* @Date 2019/11/22 23:43
*/
public final class RpcThreadFactoryBuilder {
private String namePrefix="default";
private int priority=5;
private boolean daemon=false;
private String groupName="rpc";
public RpcThreadFactoryBuilder setNamePrefix(String namePrefix){
this.namePrefix=namePrefix;
return this;
}
public RpcThreadFactoryBuilder setPriority(int priority){
if(priority>10||priority<0){
throw new UnsupportedOperationException("線程優先級設置不正確");
}
this.priority=priority;
return this;
}
public RpcThreadFactoryBuilder setDaemon(boolean daemon){
this.daemon=daemon;
return this;
}
public RpcThreadFactoryBuilder setGroupName(String groupName){
this.groupName=groupName;
return this;
}
public ThreadFactory build(){
return new BaseThreadFactory(this);
}
/**
* 啓動客戶端鏈接的自定義線程工廠
*/
static class BaseThreadFactory implements ThreadFactory {
private static final AtomicInteger poolNumber = new AtomicInteger(1);
private final ThreadGroup group;
private final AtomicInteger threadNumber = new AtomicInteger(1);
private final String namePrefix;
BaseThreadFactory(RpcThreadFactoryBuilder builder) {
group = new ThreadGroup(builder.groupName);
group.setDaemon(builder.daemon);
group.setMaxPriority(builder.priority);
namePrefix = builder.namePrefix+"-"+
poolNumber.getAndIncrement() +
"-thread-";
}
@Override
public Thread newThread(Runnable r) {
Thread t = new Thread(group, r,
namePrefix + threadNumber.getAndIncrement(),
0);
return t;
}
}
}
,這裏主要是結合線程池來提交通知任務
/**
* @author lulu
* @Date 2019/11/23 22:49
*
*/
public class RegisterForClient implements NodeChangePublisher {
private CuratorFramework client = null;
private List<PathChildrenCache> nodeListenList = new ArrayList<>();
private List<NodeChangeListener> nodeChangeListeners = new ArrayList<>();
private ThreadPoolExecutor notifyPool = new ThreadPoolExecutor(
16, 16, 5, TimeUnit.MINUTES, new ArrayBlockingQueue<>(1024)
, new RpcThreadFactoryBuilder().setNamePrefix("notifyPool").build()
);
private static class Holder {
private static final RegisterForClient j = new RegisterForClient();
}
public static RegisterForClient getInstance() {
return Holder.j;
}
//添加監聽者
private RegisterForClient() {
client = ZkUtils.getClient();
this.addListener(new NodeChangeListener.AddServer());
this.addListener(new NodeChangeListener.RemoveServer());
this.addListener(new NodeChangeListener.InactiveServer());
this.addListener(new NodeChangeListener.ReActiveServer());
}
/**
* 獲取所有的url
*
* @return
*/
public Map<String, List<URL>> getAllURL() {
Map<String, List<URL>> mapList = null;
try {
List<String> servcieList = client.getChildren().forPath("/");
mapList = new HashMap<>(servcieList.size());
for (String s : servcieList) {
//返回對應的service及其可用的url
mapList.put(s, getService(s));
//爲每個服務添加監聽
addListenerForService(s);
}
} catch (Exception e) {
e.printStackTrace();
}
return mapList;
}
private void addListenerForService(String serviceName) throws Exception {
//設置監聽，監聽所有服務下的節點變化，連接管理收到通知後移除相應的節點
final PathChildrenCache childrenCache = new PathChildrenCache(client, ZkUtils.getPath(serviceName), true);
nodeListenList.add(childrenCache);
//同步初始監聽節點
childrenCache.start(PathChildrenCache.StartMode.POST_INITIALIZED_EVENT);
childrenCache.getListenable().addListener(new PathChildrenCacheListener() {
@Override
public void childEvent(CuratorFramework client, PathChildrenCacheEvent event) throws Exception {
if (event.getType().equals(PathChildrenCacheEvent.Type.INITIALIZED)) {
//建立完監聽
return;
}
if (event.getType().equals(PathChildrenCacheEvent.Type.CHILD_REMOVED)) {
String path = event.getData().getPath();
notifyPool.submit(() -> {
System.out.println("刪除遠程服務端節點:" + path);
notifyListener(NodeChangePublisher.remove, path);
});
}
if (event.getType().equals(PathChildrenCacheEvent.Type.CHILD_UPDATED)) {
String path = event.getData().getPath();
notifyPool.submit(() -> {
byte[] status = event.getData().getData();
String serverStatus= new String(status);
if (serverStatus.equals(ACTIVE)) {
notifyPool.submit(() -> {
System.out.println("遠程服務端上線事件:" + NodeChangePublisher.add + path);
notifyListener(NodeChangePublisher.add, path);
});
} else if (serverStatus.equals(ZKConsts.INACTIVE)) {
//失效事件
notifyPool.submit(() -> {
System.out.println("遠程服務端下線事件：" + NodeChangePublisher.inactive + path);
notifyListener(NodeChangePublisher.inactive, path);
});
} else if (serverStatus.equals(ZKConsts.REACTIVE)) {
notifyPool.submit(() -> {
System.out.println("遠程服務端重新上線事件：" + path);
notifyListener(NodeChangePublisher.reactive, path);
});
}
});
}
}
});
}
public List<URL> getService(String serviceName) {
List<URL> urls = null;
try {
List<String> urlList = client.getChildren().forPath(ZkUtils.getPath(serviceName));
if (urlList != null) {
urls = new ArrayList<>(urlList.size());
}
for (String s : urlList) {
String[] url = s.split(":");
urls.add(new URL(url[0], Integer.valueOf(url[1])));
}
} catch (Exception e) {
e.printStackTrace();
}
return urls;
}
//hostname:port,遍歷所有interface節點，把對應的url節點去掉,或者標記爲不活躍的狀態
public void removeOrUpdate(String sl, Boolean update, String data) {
String[] serviceUrl = sl.split("@");
try {
String url = serviceUrl[1];
String anInterface = serviceUrl[0];
List<String> urlList = client.getChildren().forPath(ZkUtils.getPath(anInterface));
for (String s : urlList) {
if (s.equals(url)) {
if (update) {
client.setData().forPath(ZkUtils.getPath(anInterface, url), data.getBytes());
} else {
client.delete().forPath(ZkUtils.getPath(anInterface, url));
}
}
}
} catch (Exception e) {
e.printStackTrace();
}
}
//同步模式下使用，可以當作廢棄
public URL random(String serviceName) {
//通過服務名獲取具體的url
try {
List<String> urlList = client.getChildren().forPath(ZkUtils.getPath(serviceName));
String[] url = urlList.get(0).split(":");
return new URL(url[0], Integer.valueOf(url[1]));
} catch (Exception e) {
e.printStackTrace();
}
return null;
}
public void close() {
ZkUtils.closeZKClient(client);
nodeListenList.forEach(e -> {
try {
e.close();
} catch (IOException e1) {
e1.printStackTrace();
}
});
}
@Override
public void addListener(NodeChangeListener listener) {
nodeChangeListeners.add(listener);
}
@Override
public void removeListener(NodeChangeListener listener) {
nodeChangeListeners.remove(listener);
}
@Override
public void notifyListener(int state, String path) {
int i = path.lastIndexOf("/");
String serviceName = path.substring(1, i);
String[] split = path.substring(i + 1).split(":");
URL url = new URL(split[0], Integer.valueOf(split[1]));
for (NodeChangeListener nodeChangeListener : nodeChangeListeners) {
nodeChangeListener.change(state, url, serviceName);
}
}
}
然後到監聽者類，對應服務上線，服務不活躍移除，服務再次活躍重新加入，下線這四個事件
/**
* @author: lele
* @date: 2019/11/22 下午3:26
*/
public interface NodeChangeListener {
ConnectManager connect=ConnectManager.getInstance();
//相應的處理
void change(int state,URL url,String serviceName);
class AddServer implements NodeChangeListener{
@Override
public void change(int state, URL url,String serviceName) {
if(state==NodeChangePublisher.add){
System.out.println(Thread.currentThread().getName()+"addNode的listern事件被觸發");
connect.addServerAfter(url,serviceName);
}
}
}
class ReActiveServer implements NodeChangeListener{
@Override
public void change(int state, URL url, String serviceName) {
if(state==NodeChangePublisher.reactive){
System.out.println("reActive的listern事件被觸發");
connect.reAddActiveURL(url,serviceName);
}
}
}
class InactiveServer implements NodeChangeListener{
@Override
public void change(int state, URL url, String serviceName) {
if(state==NodeChangePublisher.inactive){
System.out.println("InActive的listern事件被觸發");
connect.addInactiveURL(url,serviceName);
}
}
}
class RemoveServer implements NodeChangeListener{
@Override
public void change(int state, URL url, String serviceName) {
if(state==NodeChangePublisher.remove){
System.out.println("RemovServer的listern事件被觸發");
connect.removeURL(url,serviceName,true);
}
}
}
}
client的連接池類
/**
* @author: lele
* @date: 2019/11/21 上午11:58
* 管理連接池
* 服務端在註冊後，不一定可以獲得，因爲還沒提供服務，需要zk設置節點 狀態爲Active
* //todo 定時更新鏈接
*/
public class ConnectManager {
private Boolean isShutDown = false;
private ScheduledExecutorService removeInactiveTask;
private final Random random = new Random();
/**
* 客戶端鏈接服務端超時時間
*/
private long connectTimeoutMillis = 6000;
/**
* 不活躍的鏈接存活時間，單位ms,這裏表示5分鐘不活躍就去掉
*/
private long maxInActiveTime = 1000 * 60 * 5;
/**
* 自定義6個線程組用於客戶端服務
*/
private EventLoopGroup eventLoopGroup = new NioEventLoopGroup(6);
/**
* 標示非init狀態下，addServerAfter才能起作用
*/
private CountDownLatch serverInitCountDownLatch;
/**
* 存放服務對應的訪問數，用於輪詢
*/
private Map<String, AtomicInteger> pollingMap = new ConcurrentHashMap<>();
/**
* 對於每個服務都有一個鎖，每個鎖都有一個條件隊列，用於控制鏈接獲取以及添加鏈接
*/
private Map<String, Object[]> serviceCondition = new ConcurrentHashMap<>();
/**
* 新增/刪除鏈接時的鎖
*/
private Map<String, ReentrantLock[]> addOrRemoveConnectionLock = new ConcurrentHashMap<>();
/**
* 新增/刪除不活躍鏈接時的鎖
*/
private Map<String, ReentrantLock[]> addOrRemoveInactiveLock = new ConcurrentHashMap<>();
/**
* 存放服務端地址和handler的關係
*/
private Map<String, List<NettyAsynHandler>> serverClientMap = new ConcurrentHashMap<>();
/**
* 存放不活躍的服務端地址和handler的關係，當活躍時添加回正式的handler
*/
private Map<String, PriorityQueue<NettyAsynHandler>> inactiveClientMap = new ConcurrentHashMap<>();
/**
* 用來初始化客戶端
*/
private ThreadPoolExecutor clientBooter = new ThreadPoolExecutor(
16, 16, 600, TimeUnit.SECONDS, new ArrayBlockingQueue<Runnable>(1024)
, new RpcThreadFactoryBuilder().setNamePrefix("clientBooter").build(), new ThreadPoolExecutor.AbortPolicy());
private static class Holder {
private static final ConnectManager j = new ConnectManager();
}
private ConnectManager() {
//初始化時把所有的url加進去,這裏可能沒有可用鏈接，所以需要添加對節點的監聽
Map<String, List<URL>> allURL = RegisterForClient.getInstance().getAllURL();
for (String s : allURL.keySet()) {
//爲每個服務添加鎖和條件隊列，通過條件隊列控制客戶端鏈接獲取
addLockToService(s);
}
addServerInit(allURL);
//定時清理不用的鏈接
removeInactiveTask = Executors.newSingleThreadScheduledExecutor();
removeInactiveTask.scheduleAtFixedRate(() -> removeInactiveURL(), 10, 10, TimeUnit.MINUTES);
}
//爲每個服務添加對應的鎖
private void addLockToService(String serviceName) {
ReentrantLock lock = new ReentrantLock();
Condition getConnection = lock.newCondition();
//獲取可用客戶端的鏈接及條件隊列
serviceCondition.put(serviceName, new Object[]{lock, getConnection});
//爲創建客戶端鏈接添加鎖
ReentrantLock addConnection = new ReentrantLock();
ReentrantLock removeConnection = new ReentrantLock();
addOrRemoveConnectionLock.put(serviceName, new ReentrantLock[]{addConnection, removeConnection});
ReentrantLock addInactive = new ReentrantLock();
ReentrantLock removeInactive = new ReentrantLock();
addOrRemoveInactiveLock.put(serviceName, new ReentrantLock[]{addInactive, removeInactive});
}
public static ConnectManager getInstance() {
return Holder.j;
}
/**
* 添加該服務對應的鏈接和handler
*
* @param serviceName
* @param handler 由於創建客戶端鏈接的線程都會訪問這段代碼，這裏也會存在併發情況，不然會導致多個server上線後，獲取異常
*/
public void addConnection(String serviceName, NettyAsynHandler handler) {
ReentrantLock lock = addOrRemoveConnectionLock.get(serviceName)[0];
lock.lock();
List<NettyAsynHandler> nettyAsynHandlers;
if (!serverClientMap.containsKey(serviceName)) {
nettyAsynHandlers = new ArrayList<>();
} else {
nettyAsynHandlers = serverClientMap.get(serviceName);
}
nettyAsynHandlers.add(handler);
//添加服務名和對應的url:客戶端鏈接
serverClientMap.put(serviceName, nettyAsynHandlers);
//如果處於初始化狀態，則countdown防止新增節點事件再次新增客戶端
if (serverInitCountDownLatch.getCount() != 0) {
System.out.println("連接池初始化新建客戶端鏈接:" + handler.getUrl());
serverInitCountDownLatch.countDown();
} else {
System.out.println("連接池初始化後新建客戶端鏈接:" + handler.getUrl());
}
//喚醒等待客戶端鏈接的線程
signalAvailableHandler(serviceName);
lock.unlock();
}
//通過對應的負載均衡策略挑選可用客戶端連接
public NettyAsynHandler chooseHandler(String serviceName,Integer mode){
List<NettyAsynHandler> handlers = mayWaitBeforeGetConnection(serviceName);
NettyAsynHandler choose = FetchPolicy.getPolicyMap().get(mode).choose(serviceName, handlers);
return choose;
}
//等待可用的客戶端連接
private List<NettyAsynHandler> mayWaitBeforeGetConnection(String serviceName) {
List<NettyAsynHandler> nettyAsynHandlers = serverClientMap.get(serviceName);
int size = 0;
//先嚐試獲取
if (nettyAsynHandlers != null) {
size = nettyAsynHandlers.size();
}
//不行就自選等待
while (!isShutDown && size <= 0) {
try {
//自旋等待可用服務出現，因爲客戶端與服務鏈接需要一定的時間，如果直接返回會出現空指針異常
boolean available = waitingForHandler(serviceName);
if (available) {
nettyAsynHandlers = serverClientMap.get(serviceName);
}
} catch (InterruptedException e) {
throw new RuntimeException("出錯", e);
}
}
return nettyAsynHandlers;
}
/**
* 等待一定時間，等handler和相應的server建立建立鏈接，用條件隊列控制
*
* @param serviceName
* @return
* @throws InterruptedException
*/
private boolean waitingForHandler(String serviceName) throws InterruptedException {
Object[] objects = serviceCondition.get(serviceName);
ReentrantLock lock = (ReentrantLock) objects[0];
lock.lock();
Condition condition = (Condition) objects[1];
try {
return condition.await(this.connectTimeoutMillis, TimeUnit.MILLISECONDS);
} finally {
lock.unlock();
}
}
/**
* 去掉所有與該url鏈接的客戶端,並且關閉客戶端鏈接
*
* @param url
*/
public void removeURL(URL url, String serviceName, boolean close) {
ReentrantLock lock = addOrRemoveConnectionLock.get(serviceName)[1];
lock.lock();
NettyAsynHandler target = null;
//倒序遍歷刪除對應的handler
List<NettyAsynHandler> nettyAsynHandlers = serverClientMap.get(serviceName);
for (int i = nettyAsynHandlers.size() - 1; i >= 0; i--) {
if ((target = nettyAsynHandlers.get(i)).getUrl().equals(url)) {
nettyAsynHandlers.remove(i);
if (close) {
target.close();
}
}
}
System.out.println("active:" + serverClientMap.get(serviceName).toString());
lock.unlock();
}
/**
* 定時清除不活躍的鏈接
*/
public void removeInactiveURL() {
/**
* 移除不活躍列表
*/
Collection<PriorityQueue<NettyAsynHandler>> values = inactiveClientMap.values();
Iterator<PriorityQueue<NettyAsynHandler>> iterator = values.iterator();
while (iterator.hasNext()) {
PriorityQueue<NettyAsynHandler> list = iterator.next();
//遍歷所有客戶端並根據超時時間刪除
NettyAsynHandler target;
long current = System.currentTimeMillis();
while ((current - (target = list.peek()).getInActiveTime()) > maxInActiveTime) {
list.poll();
target.close();
}
}
}
/**
* 去掉可用的服務，把他加入到不活躍的列表
* 由於是通過線程異步操作，可能存在併發問題
*
* @param url
*/
public void addInactiveURL(URL url, String serviceName) {
ReentrantLock lock = addOrRemoveInactiveLock.get(serviceName)[0];
lock.lock();
System.out.println("不活躍鏈接加入_" + url.toString());
List<NettyAsynHandler> nettyAsynHandlers = serverClientMap.get(serviceName);
NettyAsynHandler inActive = null;
for (NettyAsynHandler nettyAsynHandler : nettyAsynHandlers) {
if (nettyAsynHandler.getUrl().equals(url)) {
nettyAsynHandler.setInActiveTime(System.currentTimeMillis());
inActive = nettyAsynHandler;
break;
}
}
PriorityQueue<NettyAsynHandler> inActiveHandlers = null;
if ((inActiveHandlers = inactiveClientMap.get(serviceName)) == null) {
inActiveHandlers = new PriorityQueue<>();
}
inActiveHandlers.offer(inActive);
inactiveClientMap.put(serviceName, inActiveHandlers);
System.out.println("inactive:" + inactiveClientMap.get(serviceName).toString());
lock.unlock();
//刪除url
removeURL(url, serviceName, false);
}
/**
* 重新添加進活躍隊列
*
* @param url
* @param serviceName
*/
public void reAddActiveURL(URL url, String serviceName) {
ReentrantLock lock = addOrRemoveInactiveLock.get(serviceName)[1];
lock.lock();
PriorityQueue<NettyAsynHandler> list;
if ((list = inactiveClientMap.get(serviceName)) != null) {
Iterator<NettyAsynHandler> iterator = list.iterator();
NettyAsynHandler nettyAsynHandler;
while (iterator.hasNext()) {
nettyAsynHandler = iterator.next();
if (nettyAsynHandler.getUrl().equals(url)) {
nettyAsynHandler.setInActiveTime(0);
addConnection(serviceName, nettyAsynHandler);
list.remove(nettyAsynHandler);
System.out.printf("%s服務下的%s重新添加進活躍隊列%n", serviceName, nettyAsynHandler.toString());
break;
}
}
}
lock.unlock();
}
/**
* 釋放對應服務的條件隊列,代表有客戶端鏈接可用了
*
* @param serviceName
*/
private void signalAvailableHandler(String serviceName) {
Object[] objects = serviceCondition.get(serviceName);
ReentrantLock lock = (ReentrantLock) objects[0];
lock.lock();
Condition condition = (Condition) objects[1];
try {
condition.signalAll();
} finally {
lock.unlock();
}
}
/**
* 添加server，並啓動對應的服務器
*
* @param allURL
*/
public void addServerInit(Map<String, List<URL>> allURL) {
Collection<List<URL>> values = allURL.values();
Iterator<List<URL>> iterator = values.iterator();
int res = 0;
while (iterator.hasNext()) {
List<URL> next = iterator.next();
res += next.size();
}
serverInitCountDownLatch = new CountDownLatch(res);
for (String s : allURL.keySet()) {
pollingMap.put(s, new AtomicInteger(0));
List<URL> urls = allURL.get(s);
for (URL url : urls) {
//提交創建任務
clientBooter.submit(new Runnable() {
@Override
public void run() {
createClient(s, eventLoopGroup, url);
}
});
}
}
}
/**
* 當新節點出現後添加，但這裏有個隱患，就是當client尚未監聽完所有節點時
* addServerAfter是不允許操作的
*
* @param url
* @param serviceName
*/
public void addServerAfter(URL url, String serviceName) {
if (serverInitCountDownLatch.getCount() == 0) {
//如果還沒監聽完，就不可以加鏈接
List<NettyAsynHandler> list = null;
if ((list = serverClientMap.get(serviceName)) == null) {
list = new ArrayList<>();
serverClientMap.put(serviceName, list);
addLockToService(serviceName);
} else {
boolean exists = list.stream().filter(e -> e.getUrl().equals(url)).findFirst().isPresent();
if (exists) {
return;
}
}
clientBooter.submit(new Runnable() {
@Override
public void run() {
createClient(serviceName, eventLoopGroup, url);
}
});
}
}
/**
* 創建客戶端,持久化鏈接
*
* @param serviceName
* @param eventLoopGroup
* @param url
*/
public void createClient(String serviceName, EventLoopGroup eventLoopGroup, URL url) {
System.out.println(Thread.currentThread().getName() + "準備新建客戶端");
Bootstrap b = new Bootstrap();
b.group(eventLoopGroup)
.channel(NioSocketChannel.class)
.handler((new ChannelInitializer<SocketChannel>() {
@Override
protected void initChannel(SocketChannel ch) throws Exception {
ch.pipeline()
//把request實體變爲字節
.addLast(new RpcEncoder(RpcRequest.class))
//把返回的response字節變爲對象
.addLast(new RpcDecoder(RpcResponse.class))
.addLast(new NettyAsynHandler(url));
}
}));
ChannelFuture channelFuture = b.connect(url.getHostname(), url.getPort());
channelFuture.addListener(new ChannelFutureListener() {
@Override
public void operationComplete(ChannelFuture channelFuture) throws Exception {
//鏈接成功後的操作，把相應的url地址和客戶端鏈接存入
if (channelFuture.isSuccess()) {
NettyAsynHandler handler = channelFuture.channel().pipeline().get(NettyAsynHandler.class);
addConnection(serviceName, handler);
}
}
});
}
/**
* 關閉方法，關閉每個客戶端鏈接，釋放所有鎖，關掉創建鏈接的線程池，和客戶端的處理器
*/
public void stop() {
isShutDown = true;
serverClientMap.values().forEach(e -> e.forEach(k -> k.close()));
inactiveClientMap.values().forEach(e -> e.forEach(k -> k.close()));
for (String s : serviceCondition.keySet()) {
signalAvailableHandler(s);
}
clientBooter.shutdown();
eventLoopGroup.shutdownGracefully();
}
}
連接池基本都有註釋，所以這裏不做過多講解，下面到負載均衡策略
這裏定義了四個：隨機、輪詢、權重隨機、最少請求，具體實現
/**
* @author: lele
* @date: 2019/11/22 下午3:24
* 獲取鏈接機制,輪詢、隨機、權重
*/
public interface FetchPolicy {
Random RANDOM = new Random();
int random = 1;
int polling = 2;
int weight = 3;
int bestRequest = 4;
//策略類
Map<Integer, FetchPolicy> policyMap = new HashMap<>();
static Map<Integer, FetchPolicy> getPolicyMap() {
policyMap.put(random, new RandomFetch());
policyMap.put(polling, new PollingFetch());
policyMap.put(weight, new WeightFetch());
policyMap.put(bestRequest, new BestRequestFetch());
return policyMap;
}
NettyAsynHandler choose(String serviceName, List<NettyAsynHandler> handlers);
class WeightFetch implements FetchPolicy {
@Override
public NettyAsynHandler choose(String serviceName, List<NettyAsynHandler> handlers) {
int length = handlers.size();
//總權重
int totalWeight = 0;
//是否權重一致
boolean sameWeight = true;
//先把所有權重加起來，並且判斷權重是否一致
for (int i = 0; i < length; i++) {
int weight = handlers.get(i).getWeight();
totalWeight += weight;
if (sameWeight && i > 0
&& weight != handlers.get(i - 1).getWeight()) {
sameWeight = false;
}
}
//不斷減去對應權重所在的區間
if (totalWeight > 0 && !sameWeight) {
int offset = RANDOM.nextInt(totalWeight);
for (int i = 0; i < length; i++) {
offset -= handlers.get(i).getWeight();
if (offset < 0) {
return handlers.get(i);
}
}
}
// 如果權重都一樣，則輪詢返回
return FetchPolicy.getPolicyMap().get(polling).choose(serviceName,handlers);
}
}
/**
* 主要通過NettyAsynHandler的requestCount屬性挑取最小請求的handler進行返回
*/
class BestRequestFetch implements FetchPolicy {
@Override
public NettyAsynHandler choose(String serviceName, List<NettyAsynHandler> handlers) {
int minRequest = Integer.MAX_VALUE;
NettyAsynHandler res = null;
for (NettyAsynHandler handler : handlers) {
if (handler.getRequestCount().get() < minRequest) {
res = handler;
}
}
if(res==null){
// 如果找不到，則輪詢返回
return FetchPolicy.getPolicyMap().get(polling).choose(serviceName,handlers);
}
return res;
}
}
/**
* 記錄每個服務對應的請求次數，並返回對應的handler
*/
class PollingFetch implements FetchPolicy {
private static Map<String, AtomicInteger> pollingMap = new ConcurrentHashMap<>();
@Override
public NettyAsynHandler choose(String serviceName, List<NettyAsynHandler> handlers) {
if (pollingMap.get(serviceName) == null) {
pollingMap.put(serviceName, new AtomicInteger(0));
}
int next = pollingMap.get(serviceName).incrementAndGet();
int index = RANDOM.nextInt(next);
return handlers.get(index);
}
}
/**
* 隨機
*/
class RandomFetch implements FetchPolicy {
@Override
public NettyAsynHandler choose(String serviceName, List<NettyAsynHandler> handlers) {
int index = RANDOM.nextInt(handlers.size());
//取出相應的handler
NettyAsynHandler nettyAsynHandler = handlers.get(index-1);
return nettyAsynHandler;
}
}
}
然後spring掃描相關的註解和代理工廠要做相關的改動
@Retention(RetentionPolicy.RUNTIME)
@Target(ElementType.TYPE)
//用於接口上，name爲服務名，zk則在註冊服務改爲 服務名/ip,服務端通過傳來的接口名通過反射獲取類，或者通過給spring託管獲取其class
public @interface RpcStudyClient {
String name();
//結果返回是異步還是同步模式
int mode() default sync;
int fetch() default FetchPolicy.polling;
int sync=0;
int asyn=1;
}
其他的相關改動
//代理工廠的改動
RpcRequest rpcRequest = new RpcRequest(requestId, interfaceClass.getName(), method.getName(), args, method.getParameterTypes(), annotation.mode());
//發送請求
//這裏的管理連接池通過服務名去訪問zk，獲取可用的url
RpcFuture res = protocol.sendFuture(annotation.fetch(),annotation.name(), rpcRequest);
//發送請求的改動
public RpcFuture sendFuture(int fetch,String serviceName, RpcRequest request) {
NettyAsynHandler handler = ConnectManager.getInstance().chooseHandler(serviceName,fetch);
RpcFuture future = handler.sendRequest(request);
return future;
}
//連接池改動
public NettyAsynHandler chooseHandler(String serviceName,Integer mode){
List<NettyAsynHandler> handlers = mayWaitBeforeGetConnection(serviceName);
NettyAsynHandler choose = FetchPolicy.getPolicyMap().get(mode).choose(serviceName, handlers);
return choose;
}
心跳機制演示
這裏客戶端啓動、監控器啓動、服務端啓動
然後服務端與監控器建立連接後，監控器對不活躍的節點進行記錄，然後發送到zk端，client通過zk對相關的客戶端鏈接加入到不活躍節點，而server則嘗試重連，當監控器覺得該節點活躍了，就發到zk端讓他通知client把他加入到活躍隊列
監控器：
後來活躍了
client:
服務端
然後演示不活躍節點移除，這裏我直接在zk上修改該server爲不活躍，看一段時間後有無移除該服務，這裏把檢查任務的間隙調小一點，
執行 set /register/user/DESKTOP-QU1B3IU:8081 0 後，客戶端把他加入不活躍鏈接，8082也是同樣的操作，因爲8081比8082下線時間早，所以先移除8081
負載均衡的就不演示了，有興趣可以自己嘗試，手寫RPC系列大概就到這裏，後續如果有更新就在github上面提交了，經過這段時間的代碼編寫，從一開始模仿別人的，到根據實際的情況把自己的想法寫進去，還是學到挺多東西的,併發，負載均衡實現，spring相關的註解用法,zk、netty一些簡單應用，項目地址：https://github.com/97lele/rpcstudy
本系列參考資料：
關於netty相關的可以查看相關書籍或博客，書籍比較全一點
整體結構：https://www.bilibili.com/video/av75673208
連接池和異步：https://www.cnblogs.com/luxiaoxun/p/5272384.html
spring仿feign:https://github.com/admin801122/RPCServer
心跳簡介：https://www.cnblogs.com/demingblog/p/9957143.html
負載均衡：https://blog.csdn.net/wudiyong22/article/details/80829808