多線程處理Excel導入數據入庫

本次示例是使用springboot版本：2.2.6.RELEASE 數據庫使用：Mysql

springboot整合mybatisPlus，模擬導入2.4萬條數據。

目錄

一、目錄結構如下圖

二、依賴

三、實體類

四、Dao

五、service

五、controller

六、線程類

七、測試

八、測試結果

---

---

一、目錄結構如下圖

 

二、依賴

        <dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter-data-jpa</artifactId>
		</dependency>
		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter-web</artifactId>
		</dependency>

		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-devtools</artifactId>
			<scope>runtime</scope>
			<optional>true</optional>
		</dependency>
		<!--<dependency>
			<groupId>com.oracle.ojdbc</groupId>
			<artifactId>ojdbc8</artifactId>

		</dependency>-->
		<dependency>
			<groupId>mysql</groupId>
			<artifactId>mysql-connector-java</artifactId>

		</dependency>

		<dependency>
			<groupId>org.projectlombok</groupId>
			<artifactId>lombok</artifactId>
			<optional>false</optional>
		</dependency>
		<dependency>
			<groupId>org.apache.poi</groupId>
			<artifactId>poi</artifactId>
			<version>3.14</version>
		</dependency>
		<dependency>
			<groupId>org.apache.poi</groupId>
			<artifactId>poi-ooxml</artifactId>
			<version>3.14</version>
		</dependency>
		<dependency>
			<groupId>com.baomidou</groupId>
			<artifactId>mybatis-plus-boot-starter</artifactId>
			<version>3.1.1</version>
		</dependency>
		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter-test</artifactId>
			<scope>test</scope>
			<exclusions>
				<exclusion>
					<groupId>org.junit.vintage</groupId>
					<artifactId>junit-vintage-engine</artifactId>
				</exclusion>
			</exclusions>
		</dependency>

三、實體類

由於我idea沒有lombok插件，雖然使用了lombok包，get和set方法依然不好使。

package com.springboot.po;

import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.springframework.context.annotation.Bean;

import java.util.Date;

@Data
@TableName("TB_USER")
@NoArgsConstructor
public class User {

    @TableId(type = IdType.UUID)
    private String id;
    private String name;
    private String sex;
    private String address;
    private String telephone;
    private String type;

    private String idNumber;
    private Date createTime;

    public String getId() {
        return id;
    }

    public void setId(String id) {
        this.id = id;
    }

    public String getName() {
        return name;
    }

    public void setName(String name) {
        this.name = name;
    }

    public String getSex() {
        return sex;
    }

    public void setSex(String sex) {
        this.sex = sex;
    }

    public String getAddress() {
        return address;
    }

    public void setAddress(String address) {
        this.address = address;
    }

    public String getTelephone() {
        return telephone;
    }

    public void setTelephone(String telephone) {
        this.telephone = telephone;
    }

    public String getType() {
        return type;
    }

    public void setType(String type) {
        this.type = type;
    }

    public String getIdNumber() {
        return idNumber;
    }

    public void setIdNumber(String idNumber) {
        this.idNumber = idNumber;
    }

    public Date getCreateTime() {
        return createTime;
    }

    public void setCreateTime(Date createTime) {
        this.createTime = createTime;
    }
}

四、Dao

package com.springboot.dao;

import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.springboot.po.User;
import org.apache.ibatis.annotations.Mapper;


@Mapper
public interface UserDao extends BaseMapper<User> {

}

五、service

package com.springboot.service;


import com.springboot.dao.UserDao;
import com.springboot.po.User;
import com.springboot.thread.ImportThread;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;


import javax.persistence.criteria.CriteriaBuilder;
import java.util.List;
import java.util.concurrent.*;


@Service
public class UserService {

    @Autowired
    private UserDao userDao;

    public void insert(User user){
        userDao.insert(user);
    }

    public void saveUser(List<User> list) throws Exception {
        //一個線程處理100條數據
        int count = 100;
        //數據集合大小
        int listSize = list.size();
        //開啓的線程數
        int runSize = (listSize / count) + 1;
        //存放每個線程的執行數據
        List<User> newlist = null;
        Integer mun = 0;
        ExecutorService executor = Executors.newFixedThreadPool(runSize);
        CountDownLatch begin = new CountDownLatch(1);
        CountDownLatch end = new CountDownLatch(runSize);
        //循環創建線程
        for (int i = 0; i < runSize; i++) {
            //計算每個線程執行的數據
            if ((i + 1) == runSize) {
                int startIndex = (i * count);
                int endIndex = list.size();
                newlist = list.subList(startIndex, endIndex);
            } else {
                int startIndex = (i * count);
                int endIndex = (i + 1) * count;
                newlist = list.subList(startIndex, endIndex);
            }
            //線程類
            ImportThread mythead = new ImportThread(newlist, begin, end, userDao);
            executor.execute(mythead);
            mun = mythead.getCount();
        }
        while (count == mun){
            break;
        }
        begin.countDown();
        end.await();
        //執行完關閉線程池
        executor.shutdown();
    }
}

五、controller

package com.springboot.controller;

import com.springboot.po.User;
import com.springboot.service.UserService;
import com.springboot.utils.FileUtil;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RestController;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;

@RequestMapping("/userController")
@RestController
public class UserController {

    @Autowired
    private UserService userService;

    @RequestMapping(value = "/import",method = RequestMethod.GET)
    public Integer importData() throws FileNotFoundException {

        long  start =  System.currentTimeMillis();
        InputStream in = new FileInputStream("F:\\temp\\測試數據.xls");
        List<Object> datas = FileUtil.formatDataByExcel(in, "測試數據.xls", true, User.class, setCloums());
        long  end  =  System.currentTimeMillis();
        System.out.println("讀取Excel消耗時間："+(end-start)+"毫秒");
        List<User> list = new ArrayList<User>();

        for (Object object:datas) {
           User user =  (User)object;
           user.setCreateTime(new Date());
            list.add(user);
        }
        try {
            userService.saveUser(list);
        } catch (Exception e) {
            e.printStackTrace();
            System.out.println("多線程異常");
        }
        long  end2  =  System.currentTimeMillis();
        System.out.println("入庫消耗時間："+(end2-end)+"毫秒");
        System.out.println("消耗總時間："+(end2-start)+"毫秒");

        return list.size();
    }

    private List<String> setCloums() {
        List<String> strings = new ArrayList<>();
        strings.add("name");
        strings.add("sex");
        strings.add("address");
        strings.add("telephone");
        strings.add("type");
        strings.add("idNumber");
        return strings;
    }
    @RequestMapping(value = "/test")
    public String welcome(){
        return "Crud Spring Boot Project ! ";
    }

}

六、線程類

package com.springboot.thread;

import com.springboot.dao.UserDao;
import com.springboot.po.User;

import java.util.List;
import java.util.concurrent.CountDownLatch;


public class ImportThread implements Runnable {


    public ImportThread() {
    }
    UserDao userDao;
    private List<User> list;
    private CountDownLatch begin;
    private CountDownLatch end;
    public Integer count = 0;


    public Integer getCount() {
        return count;
    }

    public void setCount(Integer count) {
        this.count = count;
    }

    /**
     * @param list 入庫數據
     * @param begin 計時器
     * @param end   計時器
     * @param userDao  數據庫連接
     */
    public ImportThread(List<User> list, CountDownLatch begin, CountDownLatch end, UserDao userDao) {
        this.list = list;
        this.begin = begin;
        this.end = end;
        this.userDao=userDao;
    }

    @Override
    public void run() {

        try {
            for (User user :list) {
                userDao.insert(user);
            }
            count = 1;
            begin.await();
        } catch (InterruptedException e) {
            e.printStackTrace();
        } finally {
            end.countDown();
        }
    }
}

 

七、測試

八、測試結果

2.4萬條數據，用了236秒，時間有點久，各路大神有沒有更好的優化方案，歡迎來指導。