幾十萬的數據量,無法一個查詢搞定怎麼辦,可以嘗試分段查詢.

之前在項目中處理過數據量比較大的場景.如果數據庫中有幾十萬條數據,一條sql查詢時間太久,可能還會導致整個程序down掉.後來的優化方案就是分段讀取,一次讀取個幾千條,然後把查詢結果都放在一起. 主要的思想就是類似分頁查詢的邏輯.

int size = 10000;
//curDayOrdCount 是通過查詢出來的一個數據量總數,來計算總共要分多少頁
int totalPage = curDayOrdCount % size == 0 ? curDayOrdCount / size : (curDayOrdCount / size) + 1;
List<OrdLogSyn> ordLogSyns = null;
long readStart = System.currentTimeMillis();
log.info(LogHelper.info(baseProcess,"支付交易成功的筆數爲【"+curDayOrdCount+"】筆,開始讀取數據庫.............."));
for (int i = 0; i < totalPage; i++) {
    int start2 = i * size + 1;
    int preEnd2 = size * (i + 1);
    int end2 = preEnd2 > curDayOrdCount ? curDayOrdCount : preEnd2;
    List<OrdLogSyn> ordLogSynList = ordLogSynMapper.batchQuery("AP", yesterday, start2, end2);
    log.info(LogHelper.info(baseProcess,"第【" + start2 + "】條 ======= 第【" + end2 + "】條數據讀取完畢........."));
    if (ordLogSyns == null && ordLogSynList != null){
        ordLogSyns = ordLogSynList;
    }else if (ordLogSynList != null){
        ordLogSyns.addAll(ordLogSynList);
    }
}
long readEnd = System.currentTimeMillis();
log.info(LogHelper.info(baseProcess,"讀取"+curDayOrdCount+"條數據耗時"+(readEnd-readStart)/1000+"秒"));