JAVA 實體類批量插入

依賴這個LogSaveApplication。然後直接save就行了,save的時候做的是插入隊列的操作。
定時或者插入數據數量達標,線程會從隊列拿出一定量的數據(可配置).然後Spring裏面拿出對應class對應的mapper,調用insertList操作。如果mapper不存在,則不會做任何操作。
SpringContextUtil就是獲得applicationContext的類。
ThreadPoolExecutor是使用方自定義的線程池

@Component
@Slf4j
public class LogSaveApplication {

@Resource
private ThreadPoolExecutor executor;

private ConcurrentHashMap<Class, BasePool> pools = new ConcurrentHashMap<>();

public <T> void save(T t) {
    Class<?> clazz = t.getClass();
    BasePool currentPool = pools.computeIfAbsent(clazz, k -> new BasePool<T>(executor) {

        @Override
        /**
         * 實現的存儲邏輯
         */
        public void saveOperation(List<T> dataList) {
            if (dataList != null && dataList.size() > 0) {
                String name = "";
                if (mapper.get() == null) {
                    name = clazz.getSimpleName().substring(0, 1).toLowerCase() + clazz.getSimpleName().substring(1) + "Mapper";
                    /*根據Mapper名從spring中取對應的Mapper Bean對象
                     */
                    MyMapper myMapper = (MyMapper) SpringContextUtil.getAtx().getBean(name);
                    //用cas保證只set一次
                    mapper.compareAndSet(null, myMapper);
                }
                if (mapper.get() == null) {
                    log.error("no mapper for {},drop message", name);
                    return;
                }
                mapper.get().insertListDul(dataList);
                //這是我自己實現的插入重複唯一約束數據更新的操作
                log.info("insert success,size = {}", dataList.size());
            }
        }
    });
    currentPool.addData(t);
}

@Slf4j
static abstract class BasePool<T> {

    private LinkedBlockingQueue<T> dataQueue = new LinkedBlockingQueue<>();

    public int saveBatchSize = 200;

    private Object dataLock = new Object();

    private volatile boolean started = false;

    private ThreadPoolExecutor threadPoolExecutor;

    public AtomicReference<MyMapper> mapper = new AtomicReference<>();

    public BasePool(int saveBatchSize, ThreadPoolExecutor threadPoolExecutor) {
        this.saveBatchSize = saveBatchSize;
        this.threadPoolExecutor = threadPoolExecutor;
    }

    public BasePool(ThreadPoolExecutor threadPoolExecutor) {
        this.threadPoolExecutor = threadPoolExecutor;
    }

    public List<T> getData(int size) {
        log.info("從隊列中獲取數據Step1>>>size="
                + dataQueue.size() + ">>>requestSize:" + size);
        if (dataQueue != null && dataQueue.size() > 0) {
            List<T> datas = new ArrayList<>();
            int realSize = Math.min(size, dataQueue.size());
            dataQueue.drainTo(datas, realSize);
            if (dataQueue != null) {
                log.debug("從隊列中獲取數據Step2>>>size:"
                        + dataQueue.size() + ">>>getSize:" + realSize);
            }
            return datas;
        }
        return null;
    }

    public void addDataList(List<T> data) {
        startIfUnStarted();
        dataQueue.addAll(data);
    }

    public void addData(T data) {
        startIfUnStarted();
        dataQueue.add(data);
    }

    public void startIfUnStarted() {
        if (!started) {
            synchronized (dataLock) {
                if (!started) {
                    execute(threadPoolExecutor);
                    started = true;
                }
            }
        }
    }

    public Integer getDataQueueSize() {
        return dataQueue.size();
    }

    public void execute(ThreadPoolExecutor threadPoolExecutor) {
        threadPoolExecutor.execute(() -> {
            try {
                TimeUnit.SECONDS.sleep(10L);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
            while (true) {
                if (this.getDataQueueSize() <= 100) {
                    synchronized (this.dataLock) {
                        try {
                            this.dataLock.wait(TimeUnit.SECONDS.toMillis(5L));
                        } catch (InterruptedException e) {
                            log.error("循環處理消息報錯,e=", e);
                        }
                    }
                }
                if (this.getDataQueueSize() != 0) {
                    try {
                        saveOperation(getData(saveBatchSize));
                    } catch (Exception e) {
                        log.error("循環處理消息報錯,e=", e);
                    }
                }
            }
        });
    }

    /**
     * 自定義存儲的邏輯,as you wish
     * @param data
     */
    public abstract void saveOperation(List<T> data);

}

}

發表評論
所有評論
還沒有人評論,想成為第一個評論的人麼? 請在上方評論欄輸入並且點擊發布.
相關文章