分批任務處理

---

方法一: CompletableFuture使用

異步多線程處理任務

//線程池初始化
int threadNum = 5;
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
executor.setCorePoolSize(threadNum);
executor.setMaxPoolSize(threadNum);
executor.setThreadNamePrefix("archive-data-");
executor.setWaitForTasksToCompleteOnShutdown(true);
executor.initialize();
//計算頁碼
Long total = 40000000L;
int batchSize = 1000;
long totalPage = total % batchSize == 0 ? total / batchSize : total / batchSize + 1;
List<CompletableFuture<List<Archive>>> tasks = new ArrayList<>(((int) totalPage));
Archive lastArchive=null;//業務處理需要
//翻頁異步處理任務
LongStream.rangeClosed(1, totalPage).forEach(currentPage -> tasks.add(
      CompletableFuture
              .supplyAsync(() -> {
                  synchronized (lastArchive) {//控制併發
                      List<Archive> archives = getData();//獲取業務數據
                      //需要設置lastArchive,翻頁獲取數據,select
                      return archives;
                  }
              }, executor).whenComplete((archives, throwable) -> {
                  //業務處理,insert,delete
                  archives.clear();//清理處理列表,避免OOM
              }).exceptionally(throwable -> {
                  //異常處理,如:日誌輸出
                  return null;
              })
      )
);
//任務完成後處理,如:業務處理記錄
CompletableFuture.allOf(tasks.toArray(new CompletableFuture[0])).whenComplete((aVoid, throwable) -> {
  System.out.println("==========================all over");
}).join();
executor.destroy();//釋放線程池線程

---

方法二: ForkJoinTask使用

採用遞歸和 fork join算法

1. 任務類

//任務處理類,遞歸處理,即需要確定最小處理單元
public class NoticeArchiveTask extends RecursiveTask<Integer> {
    private static final Logger logger = LoggerFactory.getLogger(NoticeArchiveTask.class);
    private final List<Archive> datas;
    private final int batchSize = 5;
    private final int threadNum = 3;

    private ArchiveDao archiveDao;


    public NoticeArchiveTask(List<Archive> datas,ArchiveDao archiveDao) {
        this.datas = datas;
        this.archiveDao=archiveDao;
    }

    @Override
    protected Integer compute() {
        int dataSize = datas.size();
        if (dataSize <= batchSize) {
            String archiveCollName = "notice-3011";
            if (!CollectionUtils.isEmpty(datas)) {
                archiveDao.insertMany(datas, archiveCollName);
            }
            return dataSize;
        } else {
            int taskCount = dataSize % batchSize == 0 ? dataSize / batchSize : dataSize / batchSize + 1;
            int lastIndex = 0;
            if (threadNum < taskCount) {
                taskCount = threadNum;
                lastIndex = threadNum * batchSize;
            }
            List<NoticeArchiveTask> tasks = new ArrayList<>();
            for (int taskIndex = 0; taskIndex < taskCount; taskIndex++) {
                int fromIndex = taskIndex * batchSize;
                int toIndex = fromIndex + batchSize;
                if (toIndex > dataSize) {
                    toIndex = dataSize;
                }
                List<Archive> subList = datas.subList(fromIndex, toIndex);
                logger.info("taskIndex:{},subList.size:{}", taskIndex, subList.size());
                NoticeArchiveTask noticeArchiveTask = new NoticeArchiveTask(subList,archiveDao);
                tasks.add(noticeArchiveTask);
            }
            if (lastIndex > 0) {
                List<Archive> lastSubList = datas.subList(lastIndex, dataSize);
                NoticeArchiveTask noticeArchiveTask = new NoticeArchiveTask(lastSubList,archiveDao);
                tasks.add(noticeArchiveTask);
                logger.info("lastSubList.size:{}", lastSubList.size());
            }
            invokeAll(tasks);
            int dataSum = tasks.stream().map(task -> task.join()).mapToInt(Integer::intValue).sum();
            return dataSum;
        }
    }

}

2. 調用類

public void archiveNoticeData() {
		Date dateTime = DateUtils.parseDate("2020-03-06");
		// 一次分頁查詢數據量
		// 先查詢第一頁數據
		Page<Archive> page = new Page<>();
		page.setPageSize(pageSize);
		List<Archive> archivePage = archiveDao.getArchivePage(dateTime, page);
		int totalPage = page.getTotalPage();
		int totalSize = page.getTotalSize();
		LOGGER.info("============archive==================pageSize:{},totalPage:{},totalSize:{}",pageSize,totalPage,totalSize);
		//沒有數據
		if (totalSize == 0) {
			LOGGER.info("============archive==================notice表記錄數爲0，無須歸檔");
			// todo 保存歸檔日誌
			return;
		}
		//處理第一批數據 todo
		LOGGER.info("============archive==================第1批數據開始執行");
		ForkJoinPool forkJoinPool=new ForkJoinPool();
		doForkJoinTask(archivePage,forkJoinPool);
		if(totalPage >1){
			for (int pageIndex = 1; pageIndex < totalPage; pageIndex++) {
				String archiveFlag= CacheUtils.get("notice:archive:flag");
				while ("wait".equals(archiveFlag)){//判斷第一批任務未完成,則等待
					try {
						Thread.sleep(500);
						archiveFlag= CacheUtils.get("notice:archive:flag");
						LOGGER.error("============archive==================等待中哦");
					} catch (InterruptedException e) {
						LOGGER.error(e.getMessage(), e);
					}
				}
				//數據處理 todo
				int batchIndex=pageIndex+1;
				LOGGER.info("============archive==================第{}批數據開始執行",batchIndex);
				page.setCurPage(batchIndex);
				archivePage = archiveDao.getArchivePage(dateTime, page);
				doForkJoinTask(archivePage,forkJoinPool);
			}
		}
		//可以用來判斷任務是否已結束
		int activeThreadCount = forkJoinPool.getActiveThreadCount();
		//完成任務後業務處理
	}

	private void doForkJoinTask(List<Archive> archivePage,ForkJoinPool forkJoinPool) {
		try {
			NoticeArchiveTask noticeArchiveTask=new NoticeArchiveTask(archivePage,archiveDao);
			ForkJoinTask<Integer> taskSubmitResult = forkJoinPool.submit(noticeArchiveTask);
			Integer dataSum = taskSubmitResult.get();
			while (Objects.isNull(dataSum)){//沒有結果,則任務沒結束
				try {
					LOGGER.info("============archive==================等待中哦");
					Thread.sleep(500);
				} catch (InterruptedException e) {
					LOGGER.error(e.getMessage(), e);
				}
				CacheUtils.set("notice:archive:flag",1800000,"wait");
			}
			CacheUtils.set("notice:archive:flag",1800000,"goon");
			LOGGER.info("============archive==================該批處理數據總數:{}",dataSum);
		} catch (Exception e) {
			LOGGER.error("============archive==================,該批處理數據異常,錯誤信息:",e);
		}
	}

---