一次導入2w行的表格，並通過註解校驗字段

背景

最近項目中有一個需求，需要一次通過excel導入大量的數據。之前用的easypoi測試需要大約一小時才能完成，這樣明顯是不行的。
深入瞭解之後，其實只需要將批式處理改爲流式處理便可以解決解析慢的問題。綜合考慮後選用了easyexcel這個框架。
測試2W行的數據從導入到入庫大約需要5s,大大加快了速度。 後續增加了HibernateValidator進行校驗數據。

代碼

web導入：

@RestController
@RequestMapping("easyexcel")
public class EasyExcelTest  {
    @Autowired
    private ExecutiveInfoService executiveInfoService;

    @PostMapping("import_excel")
   public Object importExcel(@RequestParam("file") MultipartFile file) throws IOException {
        UploadDataListener uploadDataListener = new UploadDataListener(executiveInfoService);
        EasyExcel.read(file.getInputStream(), ExecutiveInfoExcel.class,  uploadDataListener).sheet().doRead();
        // 這裏根據自己的返回類型去修改
        if (uploadDataListener.getInsertFlag()){
            return "success";
        }
        return uploadDataListener.getFailList();
    }

    @GetMapping("download")
    public void download(HttpServletResponse response) throws IOException {
        // 這裏注意 有同學反應使用swagger 會導致各種問題，請直接用瀏覽器或者用postman
        response.setContentType("application/vnd.ms-excel");
        response.setCharacterEncoding("utf-8");
        // 這裏URLEncoder.encode可以防止中文亂碼 當然和easyexcel沒有關係
        String fileName = URLEncoder.encode("zms", "UTF-8");
        response.setHeader("Content-disposition", "attachment;filename=" + fileName + ".xlsx");
        EasyExcel.write(response.getOutputStream(), ExecutiveInfoExcel.class).sheet("模板").doWrite(null);
    }
}

解析表格DTO:

@Data
public class ExecutiveInfoExcel  {

    @ExcelProperty("零售客戶名")
    @NotBlank
    private String executiveName;
    @ExcelProperty("企業名稱")
    @NotBlank
    private String enterpriseName;
    @ExcelProperty("統一社會信用代碼")
    @NotNull(message = "不能爲空" )
    private String enterpriseCreditCode;
    @ExcelProperty("詳細地址")
    @NotBlank
    private String enterpriseAddress;
    @ExcelProperty("零售客戶號")
    @NotNull
    private Long executiveId;
    @ExcelProperty("零售客戶身份")
    @NotBlank
    private String executivePosition;
    @ExcelProperty("註冊資金")
    @NotBlank
    private String registerCapital;
    @ExcelProperty("法定代表人")
    @NotBlank
    private String legalRepresentative;
    @ExcelProperty("電話")
    @NotBlank
    private String telephone;
    @ExcelProperty("郵箱")
    @NotBlank
    private String email;
    @ExcelProperty("公司類型")
    @NotBlank
    private String companyType;
    @ExcelProperty("所屬行業")
    @NotBlank
    private String industry;
    @ExcelProperty("卡號")
    @NotNull
    private Integer cardNumber;
    @ExcelProperty("身份證號碼")
    @NotBlank
    private String idCard;
    @ExcelProperty("客戶號開立日期")
    @DateTimeFormat("yyyy-MM-dd")
    private Date customerOpeningDay;
    @ExcelProperty("聯繫電話")
    @NotNull
    private Integer telephoneNumber;
    @ExcelProperty("成立時間")
    @DateTimeFormat("yyyy-MM-dd")
    private Date establishmentTime;
    @ExcelProperty("最高持卡級別")
    @NotBlank
    private String highestCardholderLevel;
    @ExcelProperty("是否辦理過小微貸款業務")
    @NotBlank
    private String flagMicroLoanBusiness;

}

解析監聽類

public class  UploadDataListener extends AnalysisEventListener<ExecutiveInfoExcel> {
    private static final Logger LOGGER =
            LoggerFactory.getLogger(UploadDataListener.class);

    private ExecutiveInfoService executiveInfoService;
    private Boolean insertFlag = true;
    private List<Integer> failList = new ArrayList<>();

    List<ExecutiveInfoExcel> list = new ArrayList<ExecutiveInfoExcel>();

    /**
     * 如果使用了spring,請使用這個構造方法。每次創建Listener的時候需要把spring管理的類傳進來
     *
     * @param
     */
    public UploadDataListener(ExecutiveInfoService executiveInfoService) {
        this.executiveInfoService = executiveInfoService;
    }

    /**
     * 這個每一條數據解析都會來調用
     *
     * @param data
     *            one row value. Is is same as {@link #()}
     * @param context
     */
    @Override
    public void invoke(ExecutiveInfoExcel data, AnalysisContext context) {
        //LOGGER.info("解析到一條數據:{}", JSON.toJSONString(data));
        Validator validator = HibernateValidator.getValidator();
        Set<ConstraintViolation<ExecutiveInfoExcel>> validateSet = validator.validate(data,Default.class);
        if (validateSet != null && !validateSet.isEmpty()) {
            //System.out.println("校驗出錯：" + data);
            LOGGER.info("校驗出錯: {}",data);
            failList.add(context.readRowHolder().getRowIndex());
            insertFlag = false;
        }else {
            list.add(data);
        }

    }

    /**
     * 所有數據解析完成了 都會來調用
     *
     * @param context
     */
    @Override
    public void doAfterAllAnalysed(AnalysisContext context) {
        // 這裏也要保存數據，確保最後遺留的數據也存儲到數據庫
       if (insertFlag){
           saveData();
       }else {
           LOGGER.info("有校驗錯誤不入庫");
       }
    }

    @Override
    public void onException(Exception exception, AnalysisContext context){
        LOGGER.error("exception: {}",exception.toString());
        context.currentReadHolder();
    }

    @Override
    public void invokeHead(Map<Integer, CellData> var1, AnalysisContext var2){

    }

    /**
     * 加上存儲數據庫
     */
    private void saveData() {
       LOGGER.info("{}條數據，開始存儲數據庫！", list.size());
        List<ExecutiveInfoEntity> entityList = ConvertUtils.sourceToTarget(list, ExecutiveInfoEntity.class);
        executiveInfoService.insert(entityList);
       // LOGGER.info("存儲數據庫成功！");
    }

    public List<Integer> getFailList() {
        return failList;
    }

    public Boolean getInsertFlag() {
        return insertFlag;
    }
}

異步插入數據庫，具體的實體類這裏省略

  public void insert(List<ExecutiveInfoEntity> entityList) {
        //一次插入3000條數據到數據庫
        int num = 1;
        asynRedisService.testzms(entityList);
        int toIndex=num;
        int listSize = entityList.size();
        for (int i=0; i< entityList.size(); i+=num){
            if (i+num > listSize){
                toIndex = listSize-i;
            }
            List<ExecutiveInfoEntity> infoEntities = entityList.subList(i, i+toIndex);
            LOGGER.info("入庫{}", i);
            executiveInfoDao.insertOrUpdate(infoEntities);
        }
    }

開啓校驗工具類

public class HibernateValidator {
    private static Validator validator = Validation.byProvider( org.hibernate.validator.HibernateValidator.class )
            .configure()
            .addProperty( "hibernate.validator.fail_fast", "true" )
            .buildValidatorFactory().getValidator();

    private HibernateValidator() {

    }

    public static Validator getValidator() {
        return validator;
    }
}

類型轉換工具類

public class ConvertUtils {
    private static Logger logger = LoggerFactory.getLogger(ConvertUtils.class);

    public static <T> T sourceToTarget(Object source, Class<T> target){
        if(source == null){
            return null;
        }
        T targetObject = null;
        try {
            targetObject = target.newInstance();
            BeanUtils.copyProperties(source, targetObject);
        } catch (Exception e) {
            logger.error("convert error ", e);
        }

        return targetObject;
    }

    public static <T> List<T> sourceToTarget(Collection<?> sourceList, Class<T> target){
        if(sourceList == null){
            return null;
        }

        List targetList = new ArrayList<>(sourceList.size());
        try {
            for(Object source : sourceList){
                T targetObject = target.newInstance();
                BeanUtils.copyProperties(source, targetObject);
                targetList.add(targetObject);
            }
        }catch (Exception e){
            logger.error("convert error ", e);
        }

        return targetList;
    }
}

項目地址：git地址