數據批量導入時,加入隊列,分批處理,只是個筆記

1,用到技術點:隊列,線程,單例模式,分批處理

2,添加筆記代碼:

入口:

if(null != set && set.size() > 0){
	//異步,加入隊列
	logger.info(String.format("加入隊列,總共  %s 條數據", set.size()));
	TrackBusinessRunner trackBusinessRunner	= TrackBusinessRunner.getInstance();//獲取單例
	TrackDataDTO trackDataDTO = new TrackDataDTO();//放入處理好的數據
	trackDataDTO.setParam(param);
	trackDataDTO.setSets(set);
	trackBusinessRunner.putQueueOnload(trackDataDTO);//放入隊列
	if(trackBusinessRunner.getThreadTrackBusinessService()==null){
				trackBusinessRunner.setThreadTrackBusinessService(threadTrackBusinessService);
	}
	if(!trackBusinessRunner.isAlive()){
	trackBusinessRunner.start();//判斷是否啓動狀態,如果不是就啓動
	}
				
}

單例模式和隊列的核心類


import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.LinkedBlockingQueue;

import org.apache.commons.collections.CollectionUtils;
import org.apache.log4j.Logger;

import com.sf.iec.common.util.BatchHandlerInterface;
import com.sf.iec.common.util.BatchHandlerList;
import com.sf.iec.customerbusiness.inquiryorder.dto.TrackDataDTO;
import com.sf.iec.customerbusiness.inquiryorder.service.ThreadTrackBusinessService;

public class TrackBusinessRunner extends Thread {

	private final static Logger LOGGER = Logger.getLogger(TrackBusinessRunner.class);
	private ThreadTrackBusinessService threadTrackBusinessService;
	
	public ThreadTrackBusinessService getThreadTrackBusinessService() {
		return threadTrackBusinessService;
	}
	private LinkedBlockingQueue<TrackDataDTO> blockingQueue = new LinkedBlockingQueue<TrackDataDTO>(300);//隊列長度300,非常推薦該隊列(put和take好好用)
	private volatile boolean running = true;//開啓一個線程
	
	private TrackBusinessRunner(){}
	private static TrackBusinessRunner trackBusinessRunner;
	private static Object obj = new Object();
	//單例模式
	public static TrackBusinessRunner getInstance(){
		if(trackBusinessRunner==null){
			synchronized (obj) {
				if(trackBusinessRunner==null){
					trackBusinessRunner = new TrackBusinessRunner();
				}
			}
		}
		return trackBusinessRunner;
	}
	
	public void putQueueOnload(TrackDataDTO trackDataDTO){
		int i= 0;
		try {
			blockingQueue.put(trackDataDTO);//加入隊列
			i = 0;
		} catch (InterruptedException e) {
			LOGGER.error("加入隊列信息異常");
			e.printStackTrace();
			if(i < 2){
				putQueueOnload(trackDataDTO);
				i++;
			}
		}
	}

	@Override
	public void run() {
		while (running) {
			try {
				TrackDataDTO trackDataDTO = blockingQueue.take();
				Set<Map<String,Object>> set = trackDataDTO.getSets();
				final Map<String, String> param = trackDataDTO.getParam();
				//處理 插入  	
				List<Map<String,Object>> lst = new ArrayList<Map<String,Object>>();
				CollectionUtils.addAll(lst, set.iterator());
//分批處理,每次取200條
				BatchHandlerList<Map<String, Object>> handler = new BatchHandlerList<Map<String,Object>>(200,lst) {
					@Override
					public void handler(List<Map<String, Object>> subList) {
						// TODO Auto-generated method stub						
						threadTrackBusinessService.saveMainTainTrajectory(subList, param);
						//休眠	
						try {
							Thread.sleep(12000);
						} catch (InterruptedException e) {
							LOGGER.error("batch track handler thread interrupt excption",e);
						}//12秒
					}
				};
				handler.handlerList();
			} catch (Exception e) {
				LOGGER.error("獲取隊列信息異常",e);
				e.printStackTrace();
			}//取數據,沒有的話會等待
		}
	}		
	
	public void setThreadTrackBusinessService(
			ThreadTrackBusinessService threadTrackBusinessService) {
		this.threadTrackBusinessService = threadTrackBusinessService;
	}

}

分批處理工具類


import java.util.List;

import org.apache.log4j.Logger;

/**
 * @author  
 * @description 分批調用方法接口
 * */
public abstract class BatchHandlerList<T> implements BatchHandlerInterface<T> {
	
	private static final Logger LOGGER = Logger.getLogger(BatchHandlerList.class);
	//每次處理條數
	private Integer perNum;
	
	private List<T> aylist;

	public BatchHandlerList(Integer perNum, List<T> aylist) {
		super();
		this.perNum = perNum;
		this.aylist = aylist;
	}
	
	/**
	 * 分批調用方法
	 * */
	public void handlerList(){
		try{
			if(aylist!=null && aylist.size() > 0){
				int size = aylist.size();
				int startIndex = 0;
				int endIndex = 1;
				int num = 1;
				if (size > perNum) {
					num = size / perNum;
				}
				for (int i = 1; i <= num; i++) {
					endIndex = (i) * perNum > size ? size : (i) * perNum;
					List<T> subList = aylist.subList(startIndex, endIndex);
					
					startIndex = perNum * i;
					if (subList!=null && subList.size() > 0) {
						handler(subList);
					}
					
					if (num == i && perNum * num < size) {
						//最後一批處理
						subList = aylist.subList(perNum * num, size);
						if (subList.size() > 0) {
							handler(subList);
						}
					}
			   }
			}
		}catch(Throwable e){
			LOGGER.error("batchHandlerList handler exception",e);
			//錯誤回調方法可以重寫
			errorHandler();
		}
	}
	
	public void errorHandler(){};
}

 

發表評論
所有評論
還沒有人評論,想成為第一個評論的人麼? 請在上方評論欄輸入並且點擊發布.
相關文章