需求:
最近項目寫了poi導入excel數據到數據庫,在代碼上已經算是很優了,雖然領導沒有要求我優化導入接口,但是本着技術而言,想把學到的知識用於實踐,於是使用多線程方式導入excel。
所需pow依賴:
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi</artifactId>
<version>3.17</version>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-ooxml</artifactId>
<version>3.17</version>
</dependency>
導入的service實現類:
/**
* 多線程導入
* @param file
* @return
* @throws Exception
*/
@Override
public Map<String,Object> importData(MultipartFile file) throws Exception{
final Date now = new Date();
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
logger.info("{},開始導入數據...", format.format(now));
//設置一個信號量爲5的信號量,限制同時運行的線程數量最大爲5
Semaphore semaphore = new Semaphore(10);
Map<String,Object> map = new HashMap<>();
//多線程編程需要一個線程安全的ArrayList
List<ErrorInfo> list = Collections.synchronizedList(new ArrayList<ErrorInfo>());
Workbook workbook = null;
String filename = file.getOriginalFilename();
if(filename.endsWith("xls")){
workbook = new HSSFWorkbook(file.getInputStream());
}else if(filename.endsWith("xlsx")){
workbook = new XSSFWorkbook(file.getInputStream());
}else {
ErrorInfo errorInfo = new ErrorInfo();
errorInfo.setErrorMsg("請上傳xlx或xlsx格式的文件");
list.add(errorInfo);
map.put("code",501);
map.put("data",list);
return map;
}
Sheet sheet = workbook.getSheetAt(0);
int physicalNumberOfRows = sheet.getPhysicalNumberOfRows();
logger.info("獲取到workbook中的總行數:{}" ,physicalNumberOfRows);
//第一行是表頭,實際行數要減1
int rows = physicalNumberOfRows - 1;
//一個線程讓他處理200個row,也許可以處理更多吧
int threadNum = rows/200 + 1; //線程數量
//設置一個倒計時門閂,用來處理主線程等待螞蟻線程執行完成工作之後再運行
CountDownLatch countDownLatch = new CountDownLatch(threadNum);
//查詢是否重名
Set<String> names = this.findAllUser().stream().map(User::getUsername).collect(Collectors.toSet());
//創建一個定長的線程池
ExecutorService executorService = Executors.newFixedThreadPool(threadNum);
logger.info("開始創建線程,數據總行數:{},線程數量:{}",rows,threadNum);
List<Future<Integer>> futures = new ArrayList<>();
int successCount = 0;
for(int i = 1; i <= threadNum; i++){
int startRow = (i-1)*200 +1;
int endRow = i*200;
if(i == threadNum){
endRow = rows;
}
logger.info("開始執行線程方法,線程ID:<{}>,線程名稱:<{}>",Thread.currentThread().getId(),Thread.currentThread().getName());
Future<Integer> future = executorService.submit(new UserThread(semaphore,workbook, startRow, endRow, list, names,this,countDownLatch));
futures.add(future);
logger.info("結束線程執行方法,返回結果:<{}>,當前線程ID:<{}>,當前線程名稱:<{}>",JSON.toJSONString(future),Thread.currentThread().getId(),Thread.currentThread().getName());
//get方法中可以設置超時時間,即規定時間內沒有返回結果,則繼續運行
//get方法是線程阻塞的,調用get方法會導致後續線程因主線程阻塞而沒有創建,達不到效果。
//successCount += future.get();
}
//主線程等待子線程完成任務,60秒還沒執行完成就繼續執行
for(Future<Integer> future : futures){
successCount += future.get();
}
//主線程等待子線程全部跑完才繼續運行。設置60秒等待時間,超時後繼續執行。
countDownLatch.await(60,TimeUnit.SECONDS);
executorService.shutdown();
Date endDate = new Date();
long difference = endDate.getTime() - now.getTime();
String duration = DurationFormatUtils.formatDuration(difference, "HH:mm:ss");
logger.info("執行完成,錯誤信息:{}", JSON.toJSONString(list));
logger.info("{},結束導入,共{}條數據,導入成功:{},耗時={}", format.format(endDate), rows,successCount,duration);
map.put("code",200);
map.put("msg","結束導入,共" + rows + "條數據,導入成功" + successCount + "條,耗時:" +duration);
map.put("data",list);
return map;
}
導入線程類:
package com.thread.demo.thread;
import com.thread.demo.common.ErrorInfo;
import com.thread.demo.entity.User;
import com.thread.demo.service.UserService;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.List;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Semaphore;
/**
* @Author Honey
* @Date 2019/11/15 10:31
* @Description
**/
public class UserThread implements Callable<Integer>{
private Logger logger = LoggerFactory.getLogger(UserThread.class);
private Workbook workbook;
private Integer startRow;
private Integer endRow;
private List<ErrorInfo> errorInfoList;
private Set<String> names;
private UserService userService;
private Semaphore semaphore;
private CountDownLatch latch;
public UserThread(Semaphore semaphore,Workbook workbook,Integer startRow,Integer endRow,List<ErrorInfo> errorInfoList,Set<String> names,UserService userService,CountDownLatch latch){
this.workbook = workbook;
this.startRow = startRow;
this.endRow = endRow;
this.errorInfoList = errorInfoList;
this.names = names;
this.userService = userService;
this.semaphore = semaphore;
this.latch = latch;
}
@Override
public Integer call() throws Exception {
logger.info("線程ID:<{}>開始運行,startRow:{},endRow:{}",Thread.currentThread().getId(),startRow,endRow);
semaphore.acquire();
logger.info("消耗了一個信號量,剩餘信號量爲:{}",semaphore.availablePermits());
latch.countDown();
Sheet sheet = workbook.getSheetAt(0);
int count = 0;
for(int i = startRow; i <= endRow; i++){
User user = new User();
Row row = sheet.getRow(i);
Cell cell1 = row.getCell(0);
String username = cell1.getStringCellValue();
user.setUsername(username);
user.setPassword("123456");
Cell cell2 = row.getCell(1);
String realname = cell2.getStringCellValue();
user.setRealName(realname);
if(names.contains(username)){
ErrorInfo errorInfo = new ErrorInfo();
errorInfo.setRow(startRow);
errorInfo.setColumn(1);
errorInfo.setErrorMsg("第" + startRow + "行用戶賬號已存在");
errorInfoList.add(errorInfo);
break;
}
count += userService.addUser(user);
}
semaphore.release();
return count;
}
}
controller也貼一下吧。沒什麼東西
package com.thread.demo.controller;
import com.thread.demo.service.UserService;
import org.apache.poi.ss.usermodel.Workbook;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile;
import javax.servlet.http.HttpServletResponse;
import java.util.Map;
/**
* @Author Honey
* @Date 2019/11/15 10:27
* @Description
**/
@RestController
public class UserController {
@Autowired
private UserService userService;
/**
* 多線程導入
* @param file
* @return
*/
@PostMapping("/importManyThread")
public Map importData(MultipartFile file){
Map<String, Object> map = null;
try {
map = userService.importData(file);
return map;
} catch (Exception e) {
e.printStackTrace();
map.put("code",501);
map.put("msg","數據出錯");
return map;
}
}
/**
* 單線程導入
* @param file
* @return
*/
@PostMapping("/importSingleThread")
public Map importData2(MultipartFile file){
Map<String, Object> map = null;
try {
map = userService.importDataYiBan(file);
return map;
} catch (Exception e) {
e.printStackTrace();
map.put("code",501);
map.put("msg","數據出錯");
return map;
}
}
/**
* 導出excel
* @param response
* @throws Exception
*/
@GetMapping("/export")
public void exportData(HttpServletResponse response) throws Exception{
Workbook workbook = userService.exportData();
response.setContentType("application/vnd.ms-excel;charset=utf-8");
response.setCharacterEncoding("UTF-8");
//test.xls是彈出下載對話框的文件名,不能爲中文,中文請自行編碼
response.setHeader("Content-Disposition", "attachment;filename=user.xlsx");
workbook.write(response.getOutputStream());
}
}
執行結果:
使用多線程方式導入5000條數據花費時間14秒,而單線程導入則需1分鐘14秒。
可見多線程方式運行程序是可以達到空間換時間的目的的。