java多線程方式導入excel數據入數據庫

需求:
最近項目寫了poi導入excel數據到數據庫,在代碼上已經算是很優了,雖然領導沒有要求我優化導入接口,但是本着技術而言,想把學到的知識用於實踐,於是使用多線程方式導入excel。

所需pow依賴:

		<dependency>
			<groupId>org.apache.poi</groupId>
			<artifactId>poi</artifactId>
			<version>3.17</version>
		</dependency>
		<dependency>
			<groupId>org.apache.poi</groupId>
			<artifactId>poi-ooxml</artifactId>
			<version>3.17</version>
		</dependency>

導入的service實現類:

/**
     * 多線程導入
     * @param file
     * @return
     * @throws Exception
     */
    @Override
    public Map<String,Object> importData(MultipartFile file) throws Exception{
        final Date now = new Date();
        SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
        logger.info("{},開始導入數據...", format.format(now));
        //設置一個信號量爲5的信號量,限制同時運行的線程數量最大爲5
        Semaphore semaphore = new Semaphore(10);

        Map<String,Object> map = new HashMap<>();
        //多線程編程需要一個線程安全的ArrayList
        List<ErrorInfo> list = Collections.synchronizedList(new ArrayList<ErrorInfo>());
        Workbook workbook = null;
        String filename = file.getOriginalFilename();
        if(filename.endsWith("xls")){
            workbook = new HSSFWorkbook(file.getInputStream());
        }else if(filename.endsWith("xlsx")){
            workbook = new XSSFWorkbook(file.getInputStream());
        }else {
            ErrorInfo errorInfo = new ErrorInfo();
            errorInfo.setErrorMsg("請上傳xlx或xlsx格式的文件");
            list.add(errorInfo);
            map.put("code",501);
            map.put("data",list);
            return map;
        }
        Sheet sheet = workbook.getSheetAt(0);
        int physicalNumberOfRows = sheet.getPhysicalNumberOfRows();
        logger.info("獲取到workbook中的總行數:{}" ,physicalNumberOfRows);
        //第一行是表頭,實際行數要減1
        int rows = physicalNumberOfRows - 1;
        //一個線程讓他處理200個row,也許可以處理更多吧
        int threadNum = rows/200 + 1; //線程數量

        //設置一個倒計時門閂,用來處理主線程等待螞蟻線程執行完成工作之後再運行
        CountDownLatch countDownLatch = new CountDownLatch(threadNum);
        //查詢是否重名
        Set<String> names = this.findAllUser().stream().map(User::getUsername).collect(Collectors.toSet());
        //創建一個定長的線程池
        ExecutorService executorService = Executors.newFixedThreadPool(threadNum);

        logger.info("開始創建線程,數據總行數:{},線程數量:{}",rows,threadNum);

        List<Future<Integer>> futures = new ArrayList<>();
        int successCount = 0;

        for(int i = 1; i <= threadNum; i++){

            int startRow = (i-1)*200 +1;
            int endRow = i*200;
            if(i == threadNum){
                endRow = rows;
            }
            logger.info("開始執行線程方法,線程ID:<{}>,線程名稱:<{}>",Thread.currentThread().getId(),Thread.currentThread().getName());
            Future<Integer> future = executorService.submit(new UserThread(semaphore,workbook, startRow, endRow, list, names,this,countDownLatch));
            futures.add(future);
            logger.info("結束線程執行方法,返回結果:<{}>,當前線程ID:<{}>,當前線程名稱:<{}>",JSON.toJSONString(future),Thread.currentThread().getId(),Thread.currentThread().getName());
            //get方法中可以設置超時時間,即規定時間內沒有返回結果,則繼續運行
            //get方法是線程阻塞的,調用get方法會導致後續線程因主線程阻塞而沒有創建,達不到效果。
            //successCount += future.get();
        }
        //主線程等待子線程完成任務,60秒還沒執行完成就繼續執行

        for(Future<Integer> future : futures){
                successCount += future.get();
        }
		//主線程等待子線程全部跑完才繼續運行。設置60秒等待時間,超時後繼續執行。
        countDownLatch.await(60,TimeUnit.SECONDS);
        executorService.shutdown();

        Date endDate = new Date();
        long difference = endDate.getTime() - now.getTime();
        String duration = DurationFormatUtils.formatDuration(difference, "HH:mm:ss");
        logger.info("執行完成,錯誤信息:{}", JSON.toJSONString(list));
        logger.info("{},結束導入,共{}條數據,導入成功:{},耗時={}", format.format(endDate), rows,successCount,duration);
        map.put("code",200);
        map.put("msg","結束導入,共" + rows + "條數據,導入成功" + successCount + "條,耗時:" +duration);
        map.put("data",list);
        return map;
    }

導入線程類:

package com.thread.demo.thread;

import com.thread.demo.common.ErrorInfo;
import com.thread.demo.entity.User;
import com.thread.demo.service.UserService;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.List;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Semaphore;

/**
 * @Author Honey
 * @Date 2019/11/15 10:31
 * @Description
 **/
public class UserThread implements Callable<Integer>{

    private Logger logger = LoggerFactory.getLogger(UserThread.class);

    private Workbook workbook;

    private Integer startRow;

    private Integer endRow;

    private List<ErrorInfo> errorInfoList;

    private Set<String> names;

    private UserService userService;

    private Semaphore semaphore;

    private CountDownLatch latch;

    public UserThread(Semaphore semaphore,Workbook workbook,Integer startRow,Integer endRow,List<ErrorInfo> errorInfoList,Set<String> names,UserService userService,CountDownLatch latch){
        this.workbook = workbook;
        this.startRow = startRow;
        this.endRow = endRow;
        this.errorInfoList = errorInfoList;
        this.names = names;
        this.userService = userService;
        this.semaphore = semaphore;
        this.latch = latch;
    }

    @Override
    public Integer call() throws Exception {
        logger.info("線程ID:<{}>開始運行,startRow:{},endRow:{}",Thread.currentThread().getId(),startRow,endRow);
        semaphore.acquire();
        logger.info("消耗了一個信號量,剩餘信號量爲:{}",semaphore.availablePermits());
        latch.countDown();
        Sheet sheet = workbook.getSheetAt(0);
        int count = 0;
        for(int i = startRow; i <= endRow; i++){
            User user = new User();
            Row row = sheet.getRow(i);
            Cell cell1 = row.getCell(0);
            String username = cell1.getStringCellValue();
            user.setUsername(username);
            user.setPassword("123456");
            Cell cell2 = row.getCell(1);
            String realname = cell2.getStringCellValue();
            user.setRealName(realname);
            if(names.contains(username)){
                ErrorInfo errorInfo = new ErrorInfo();
                errorInfo.setRow(startRow);
                errorInfo.setColumn(1);
                errorInfo.setErrorMsg("第" + startRow + "行用戶賬號已存在");
                errorInfoList.add(errorInfo);
                break;
            }
            count += userService.addUser(user);
        }
        semaphore.release();
        return count;
    }
}

controller也貼一下吧。沒什麼東西

package com.thread.demo.controller;

import com.thread.demo.service.UserService;
import org.apache.poi.ss.usermodel.Workbook;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile;

import javax.servlet.http.HttpServletResponse;
import java.util.Map;

/**
 * @Author Honey
 * @Date 2019/11/15 10:27
 * @Description
 **/
@RestController
public class UserController {

    @Autowired
    private UserService userService;

    /**
     * 多線程導入
     * @param file
     * @return
     */
    @PostMapping("/importManyThread")
    public Map importData(MultipartFile file){
        Map<String, Object> map = null;
        try {
            map = userService.importData(file);
            return map;
        } catch (Exception e) {
            e.printStackTrace();
            map.put("code",501);
            map.put("msg","數據出錯");
            return map;
        }
    }

    /**
     * 單線程導入
     * @param file
     * @return
     */
    @PostMapping("/importSingleThread")
    public Map importData2(MultipartFile file){
        Map<String, Object> map = null;
        try {
            map = userService.importDataYiBan(file);
            return map;
        } catch (Exception e) {
            e.printStackTrace();
            map.put("code",501);
            map.put("msg","數據出錯");
            return map;
        }
    }

    /**
     * 導出excel
     * @param response
     * @throws Exception
     */
    @GetMapping("/export")
    public void exportData(HttpServletResponse response) throws Exception{
        Workbook workbook = userService.exportData();
        response.setContentType("application/vnd.ms-excel;charset=utf-8");
        response.setCharacterEncoding("UTF-8");
        //test.xls是彈出下載對話框的文件名,不能爲中文,中文請自行編碼
        response.setHeader("Content-Disposition", "attachment;filename=user.xlsx");
        workbook.write(response.getOutputStream());
    }

}

執行結果:
在這裏插入圖片描述

使用多線程方式導入5000條數據花費時間14秒,而單線程導入則需1分鐘14秒。
可見多線程方式運行程序是可以達到空間換時間的目的的。

發表評論
所有評論
還沒有人評論,想成為第一個評論的人麼? 請在上方評論欄輸入並且點擊發布.
相關文章