同步機制
概述
本次項目中數據實時同步沒有使用一些同步插件例如go-mysql-elasticsearch、elasticsearch-jdbc等同步技術,而是根據企業業務和項目的結構的特殊性採用了更適合項目和業務需求的多線程任務調度數據同步的機制。
同步模塊
同步流程
業務數據改變–>修改關係型數據庫數據–>將修改的記錄主鍵、操作等信息放入隊列充當生產者–> 事件調度任務利用多線程獲取數據充當消費者,獲取隊首數據–>ES檢索服務通過隊列數據獲取到相應的記錄數據更新,同步到ES,實現實時同步更新。
主要代碼實現
public class InstantiationTracingBeanPostProcessor implements ApplicationListener<ContextRefreshedEvent> {
private static Logger log = LoggerFactory.getLogger(InstantiationTracingBeanPostProcessor.class);
@Autowired
private EventQueue eventQueue;
@Autowired
private EsSyncTask esSyncTask;
private static final String THREAD_COUNT = SystemConfig.get("es.sync.thread.count");
@Override
public void onApplicationEvent(ContextRefreshedEvent contextRefreshedEvent) {
if (contextRefreshedEvent.getApplicationContext().getParent() == null) {
System.out.println("Application starting ...");
try {
System.out.println("Event start ...");
eventQueue.popQueue();
int threadCount = 1;
try {
threadCount = Integer.parseInt(THREAD_COUNT);
} catch (Exception e) {
log.error("Wrong thread count configuration \"s%\", use default 1 thread to sync.", THREAD_COUNT);
}
System.out.println("Elasticsearch sync thread is starting, current thread count : " + threadCount);
esSyncTask.start(threadCount);
} catch (InterruptedException e) {
log.error("Tasks start error, make sure redis is running : ", e);
}
System.out.println("application start end...");
}
}
}
public void start(int threadCount) {
if (StringUtils.hasLength(ES_SYNC_RETRY_TIMES_CONFIG)) {
ES_SYNC_RETRY_TIMES = Integer.valueOf(ES_SYNC_RETRY_TIMES_CONFIG);
}
ExecutorService executorService = Executors.newCachedThreadPool();
for (int i = 0; i < threadCount; i++) {
executorService.execute(new SyncThread());
}
}
/**
* 同步線程
*/
class SyncThread implements Runnable {
@Override
public void run() {
while (true) {
String threadName = Thread.currentThread().getName() + ":" + Thread.currentThread().getId();
String json = redisService.removeFromHead(RedisConstant.ES_DATA_SYNC_KEY);
if (StringUtils.hasLength(json)) {
long start = System.currentTimeMillis();
log.info("[" + threadName + "]Start to sync data : " + json);
SaveData saveData = JSON.parseObject(json, SaveData.class);
String id = saveData.getId();
if (!StringUtils.hasLength(id)) {
log.warn("No id in saveData be found.");
continue;
}
int type = saveData.getType();
int oper = saveData.getOper();
if (EsSyncConst.SyncType.HOTEL.codeOf().equals(type)) {
syncHotel(id, saveData);
} else if (EsSyncConst.SyncType.HOTEL_SCORE.codeOf().equals(type)) {
syncHotelScore(id, saveData);
} else if (EsSyncConst.SyncType.ROOM_PRICE.codeOf().equals(type)) {
syncRoomPrice(id, saveData);
} else if (EsSyncConst.SyncType.ROOM_PRICE_BY_ROOM.codeOf().equals(type)) {
syncRoomPricesByRoom(id, saveData);
} else if (EsSyncConst.SyncType.NIGHT_USER_BY_ROOM.codeOf().equals(type)) {
syncNightUserByRoom(id, saveData);
} else {
log.warn("Type in saveData is unsupported : " + type);
}
long end = System.currentTimeMillis();
log.info("[" + threadName + "]Sync data finished, spend : " + (end - start) + "ms.");
}
try {
Thread.sleep(500);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
}