[JAVA工具類]-大數據Excel生成導出

前言

在實際工作中，經常會需要進行Excel文件的下載導出，並且有時希望通過異步下載來進行實現或者需要下載數據量很大。爲防止各個系統重複造輪子，本文通過註解方式來實現Excel的普通、分片生成。

依賴Jar包

            <dependency>
                <groupId>org.apache.poi</groupId>
                <artifactId>poi-ooxml</artifactId>
                <version>3.17</version>
            </dependency>
            <dependency>
                <groupId>org.apache.poi</groupId>
                <artifactId>poi</artifactId>
                <version>3.17</version>
            </dependency>

實現方案

1、通過自定義註解來定義導出文件的Bean對象。
2、自定義註解可以設定導出字段的標題名稱。
3、創建一個通用的Excel導出工具類，採用SXSSFWorkbook實現大數據量的Excel生成
4、滿足普通生成和分片生成功能，分片上傳通過內存緩存已經生成的ExcelWorkBook，然後後續分片追加，最後完成時刪除緩存的文件。

直接上代碼

1、自定義註解

/**
 * Excel導出字段 
 * @author yukaiji
 * @date 2020-05-20
 */
@Target({ElementType.FIELD, ElementType.TYPE})
@Retention(RetentionPolicy.RUNTIME)
public @interface XlsField {

    String xlsHeaderName() default "";

}

2、自定義註解使用方式

    public class Test {
        @XlsField(xlsHeaderName = "姓名")
        String name;
        @XlsField(xlsHeaderName = "年齡")
        String age;
        @XlsField(xlsHeaderName = "性別")
        String sex;
    }

3、Excel生成工具類

import org.apache.commons.lang3.ArrayUtils;
import org.apache.poi.util.IOUtils;
import org.apache.poi.xssf.streaming.SXSSFCell;
import org.apache.poi.xssf.streaming.SXSSFRow;
import org.apache.poi.xssf.streaming.SXSSFSheet;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;

import java.io.*;
import java.lang.reflect.Field;
import java.util.*;

/**
 * 通過SXSSFWorkbook實現一個大數據excel生成工具類
 * 版本要求excel2007之後版本
 * 擴展名爲.xlsx
 *
 * @author yukaiji
 * @date 2020-05-20
 */
public class ExcelUtil {


    /**
     * 用來做分片上傳，以文件名稱爲key，已經生成過的workBook爲value
     **/
    private static Map<String, LocalWorkbook> FILE_BOOK_MAP = new HashMap<>(64);

    /**
     * 單個Sheet頁最大行數
     **/
    private static final int MAX_ROW_NUM = 1048574;

    /**
     * 根據自定義註解獲取excel表頭
     **/
    private static <T> List<String> genHeader(Class<T> modelClazz) {
        Field[] fields = modelClazz.getDeclaredFields();
        if (ArrayUtils.isEmpty(fields)) {
            return new ArrayList(0);
        } else {
            List<String> headers = new ArrayList(fields.length);
            Field[] arr$ = fields;
            int len$ = fields.length;
            for (int i$ = 0; i$ < len$; ++i$) {
                Field field = arr$[i$];
                boolean isPresent = field.isAnnotationPresent(XlsField.class);
                if (isPresent) {
                    String headerInfo = field.getAnnotation(XlsField.class).xlsHeaderName();
                    headers.add(headerInfo);
                }
            }
            return headers;
        }
    }

    /**
     * 創建一個excel文件（非分片）
     *
     * @param models   數據
     * @param fileName 文件名稱
     * @return 文件
     */
    public static <T> File createExcel(List<T> models, String fileName) throws IllegalAccessException {
        SXSSFWorkbook workbook = createWorkBook(models, fileName);
        File file = new File(fileName);
        OutputStream out = null;
        try {
            if (!file.exists()) {
                file.createNewFile();
            }
            out = new FileOutputStream(file);
            workbook.write(out);
        } catch (Exception e) {
            e.printStackTrace();
            throw new RuntimeException(e);
        } finally {
            IOUtils.closeQuietly(out);
        }
        return file;
    }

    public static <T> File multipartCreateExcel(List<T> models, String fileName, boolean isFinish) throws IllegalAccessException {
        return multipartCreateExcel(models, fileName, MAX_ROW_NUM, isFinish);
    }

    /**
     * 分片生成excel
     *
     * @param models   數據
     * @param fileName 文件名稱
     * @param sheetNum 每個Sheet頁最大行數
     * @param isFinish 是否生成完成(最後一片)
     * @return 流，可以直接上傳S3
     */
    public static <T> File multipartCreateExcel(List<T> models, String fileName, int sheetNum, boolean isFinish) throws IllegalAccessException {
        if (sheetNum > MAX_ROW_NUM) {
            throw new IllegalAccessException("sheet rows num More than " + MAX_ROW_NUM + " rows ");
        }
        SXSSFWorkbook workbook = null;
        try {
            workbook = multipartCreateWorkBook(models, fileName, sheetNum);
        } catch (IllegalAccessException e) {
            FILE_BOOK_MAP.remove(fileName);
            throw e;
        }
        OutputStream out = null;
        File file = new File(fileName);
        if (isFinish) {
            try {
                out = new FileOutputStream(file);
                //臨時緩衝區
                //創建臨時文件
                workbook.write(out);
            } catch (Exception e) {
                e.printStackTrace();
            } finally {
                FILE_BOOK_MAP.remove(fileName);
            }
        }
        return file;
    }

    /**
     * 分片寫入SXSSFWorkbook
     *
     * @param models      數據
     * @param fileName    文件名稱
     * @param sheetRowNum 一個sheet頁多少行
     * @return SXSSFWorkbook excel文件
     */
    private static <T> SXSSFWorkbook multipartCreateWorkBook(List<T> models, String fileName, int sheetRowNum) throws IllegalAccessException {
        List<String> header = genHeader(models.get(0).getClass());
        Field[] fields = models.get(0).getClass().getDeclaredFields();
        SXSSFWorkbook workbook;
        SXSSFSheet sheet;
        SXSSFRow row;
        int rowIndex = 0;
        if (!FILE_BOOK_MAP.containsKey(fileName)) {
            workbook = new SXSSFWorkbook(1000);
            sheet = workbook.createSheet();
            row = sheet.createRow(0);
            for (int i = 0; i < header.size(); i++) {
                SXSSFCell cell = row.createCell(i);
                cell.setCellValue(header.get(i));
            }
            FILE_BOOK_MAP.put(fileName, new LocalWorkbook(workbook, rowIndex));
        } else {
            workbook = FILE_BOOK_MAP.get(fileName).getSxssfWorkbook();
            sheet = workbook.getSheetAt(0);
            rowIndex = FILE_BOOK_MAP.get(fileName).getRowIndex();
        }

        Iterator<T> it = models.iterator();
        while (it.hasNext()) {
            if (rowIndex == sheetRowNum) {
                rowIndex = 0;
                sheet = workbook.createSheet();
                row = sheet.createRow(0);
                for (int i = 0; i < header.size(); i++) {
                    SXSSFCell cell = row.createCell(i);
                    cell.setCellValue(header.get(i));
                }
                FILE_BOOK_MAP.get(fileName).setRowIndex(rowIndex);
            }

            rowIndex++;
            row = sheet.createRow(rowIndex);
            T t = (T) it.next();
            int cellIndex = 0;
            for (Field f : fields) {
                SXSSFCell cell = row.createCell(cellIndex);
                f.setAccessible(true);
                boolean isPresent = f.isAnnotationPresent(XlsField.class);
                if (!isPresent) {
                    continue;
                }
                String value = Objects.toString(f.get(t));
                cell.setCellValue(value);
                cellIndex++;
            }
        }
        FILE_BOOK_MAP.get(fileName).setRowIndex(rowIndex);
        return workbook;
    }

    private static <T> SXSSFWorkbook createWorkBook(List<T> models, String fileName) throws IllegalAccessException {
        List<String> header = genHeader(models.get(0).getClass());
        Field[] fields = models.get(0).getClass().getDeclaredFields();
        SXSSFWorkbook workbook = new SXSSFWorkbook(1000);
        SXSSFSheet sheet = workbook.createSheet();
        SXSSFRow row = sheet.createRow(0);
        for (int i = 0; i < header.size(); i++) {
            SXSSFCell cell = row.createCell(i);
            cell.setCellValue(header.get(i));
        }
        Iterator<T> it = models.iterator();
        int index = 0;
        while (it.hasNext()) {
            index++;
            row = sheet.createRow(index);
            T t = (T) it.next();
            int cellIndex = 0;
            for (Field f : fields) {
                SXSSFCell cell = row.createCell(cellIndex);
                f.setAccessible(true);
                boolean isPresent = f.isAnnotationPresent(XlsField.class);
                if (!isPresent) {
                    continue;
                }
                String value = Objects.toString(f.get(t));
                cell.setCellValue(value);
                cellIndex++;
            }
        }
        return workbook;
    }

    /**
     * 分片文件上傳文件類
     */
    static class LocalWorkbook {

        private LocalWorkbook(SXSSFWorkbook sxssfWorkbook, int rowIndex) {
            this.sxssfWorkbook = sxssfWorkbook;
            this.rowIndex = rowIndex;
            this.totalRowNum = 0;
        }

        /**
         * 未完成的workBook
         **/
        private SXSSFWorkbook sxssfWorkbook;
        /**
         * 當前sheet頁row指針
         **/
        private int rowIndex;
        /**
         * 文件整體的行數
         **/
        private int totalRowNum;

        public SXSSFWorkbook getSxssfWorkbook() {
            return sxssfWorkbook;
        }

        public void setSxssfWorkbook(SXSSFWorkbook sxssfWorkbook) {
            this.sxssfWorkbook = sxssfWorkbook;
        }

        public int getRowIndex() {
            return rowIndex;
        }

        public void setRowIndex(int rowIndex) {
            this.rowIndex = rowIndex;
        }

        public int getTotalRowNum() {
            return totalRowNum;
        }

        public void setTotalRowNum(int totalRowNum) {
            this.totalRowNum = totalRowNum;
        }
    }
}

4、百萬數據量測試

public static void main(String[] args) throws IllegalAccessException {
        String fileName = "testexcel.xlsx";
        List<Test> list = new ArrayList<>(1234345);
        for (int i = 0; i < 1234345; i++) {
            list.add(new Test(String.valueOf(i), String.valueOf(i), String.valueOf(i)));
        }
        // 按照200000分片
        List<List<Test>> ss = Lists.partition(list, 200000);
        File file = null;
        for (int i = 0; i < ss.size(); i++) {
            file =  ExcelUtil.multipartCreateExcel(ss.get(i), fileName, 100000, i == ss.size() - 1);
        }
    }

百萬數據量的Excel生成大概幾秒鐘的時間。

總結

就簡簡單單通過poi包來實現文件的生成，但是隻支持excel2007之後的版本，其中通過自定義註解獲取要生成的列和表頭，生成方式通過傳入一個List列表方式。