1、ListUtils工具類(提供將List均分成n個list和將List分成每份count數量的List操作)
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;
public class ListUtils {
/**
* 將一個list均分成n個list:
* @param source
* @param n
* @return
*/
public static <T> List<List<T>> avgList(List<T> source, int n) {
List<List<T>> result = new ArrayList<>();
//(先計算出餘數)
int remainder = source.size() % n;
//然後是商
int number = source.size() / n;
//偏移量
int offset = 0;
for (int i = 0; i < n; i++) {
List<T> value;
if (remainder > 0) {
value = source.subList(i * number + offset, (i + 1) * number + offset + 1);
remainder--;
offset++;
} else {
value = source.subList(i * number + offset, (i + 1) * number + offset);
}
result.add(value);
}
return result;
}
/**
* 將list分成每份count的數量的list:
* @param resList
* @param count
* @return
*/
public static <T> List<List<T>> splitList(List<T> resList, int count) {
if (resList == null || count < 1) {
return null;
}
//long t1 = System.currentTimeMillis();
List<List<T>> result = new ArrayList<List<T>>();
int size = resList.size();
if (size <= count) {
// 數據量不足count指定的大小
result.add(resList);
} else {
int pre = size / count;
int last = size % count;
// 前面pre個集合,每個大小都是count個元素
for (int i = 0; i < pre; i++) {
List<T> itemList = new ArrayList<T>();
for (int j = 0; j < count; j++) {
itemList.add(resList.get(i * count + j));
}
result.add(itemList);
}
// last的進行處理
if (last > 0) {
List<T> itemList = new ArrayList<T>();
for (int i = 0; i < last; i++) {
itemList.add(resList.get(pre * count + i));
}
result.add(itemList);
}
}
/*long t2 = System.currentTimeMillis();
System.out.println("splitList====>>> resList.size:" + resList.size()
+ ", count:" +count + ", costs time:" + (t2 - t1) + " ms");*/
return result;
}
public static <T> List<List<T>> splitList2(List<T> list, int count) { //效率低不推薦使用
long t1 = System.currentTimeMillis();
int limit = (list.size() + count - 1) / count;
// 方法一:使用流遍歷操作
/*List<List<T>> result = new ArrayList<>();
Stream.iterate(0, n -> n + 1).limit(limit).forEach(i -> {
result.add(list.stream().skip(i * count).limit(count).collect(Collectors.toList()));
});*/
//方法二:獲取分割後的集合
List<List<T>> result = Stream.iterate(0, n -> n + 1).limit(limit).parallel().map(a -> list.stream().skip(a * count).limit(count).parallel().collect(Collectors.toList())).collect(Collectors.toList());
long t2 = System.currentTimeMillis();
System.out.println("splitList====>>> resList.size:" + list.size()
+ ", count:" +count + ", costs time:" + (t2 - t1) + " ms");
return result;
}
public static void main(String[] args) {
/*List<Integer> list=new ArrayList<>();
for (int i = 0; i < 17; i++) {
list.add(i);
}
List<List<Integer>> avgList = avgList(list, 5);
System.out.println("avgList: " + avgList);
List<List<Integer>> splitList = splitList(list, 5);
System.out.println("splitList: " + splitList);
ArrayList<Integer> arr_list = new ArrayList<>();
for (int i = 0; i < splitList.size(); i++) {
List<Integer> subList = splitList.get(i);
arr_list.addAll(subList);
}
System.out.println("arr_list: " + arr_list);*/
}
}
2、模擬一千萬請求數據,創建一個線程池10線程個控制最大併發數,每個線程一次處理10萬條,開啓100個線程。(用於控制每個批次線程的數據量業務場景)
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Semaphore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.test.utils.ListUtils;
public class ConcurrencyTest {
public static void main(String[] args) {
int count = 10000000;
List<Long> listKey = new ArrayList<>();
for (int i = 0; i < count; i++) {
listKey.add(114560315500000000L + i);
}
ConcurrencyTest concurrencyTest = new ConcurrencyTest();
List<Long> valueList1 = concurrencyTest.getValueList1(listKey);
System.out.println("====>> getValueList1 valueList.size: " + valueList1.size());
}
private final static Logger log = LoggerFactory.getLogger(ConcurrencyTest.class);
/**
* 模擬一千萬請求數據,創建一個線程池10個線程控制最大併發數,每個線程一次處理10萬條,開啓100個線程。(用於控制每批次線程的數據量業務場景)
* @param listKey 請求處理的總數據量
* @return
*/
public List<Long> getValueList1(List<Long> listKey) {
/**
(1)newCachedThreadPool 創建一個可緩存線程池,如果線程池長度超過處理需要,可靈活回收空閒線程,若無可回收,則新建線程。
(2)newFixedThreadPool 創建一個定長線程池,可控制線程最大併發數,超出的線程會在隊列中等待。
(3)newScheduledThreadPool 創建一個定長線程池,支持定時及週期性任務執行。
(4)newSingleThreadExecutor 創建一個單線程化的線程池,它只會用唯一的工作線程來執行任務,保證所有任務按照指定順序(FIFO, LIFO, 優先級)執行。
*/
//創建一個線程池
final ExecutorService executorService = Executors.newFixedThreadPool(10); //用於控制同時併發執行的線程數,使用線程池創建資源提高效率
List<Long> list_val = Collections.synchronizedList(new ArrayList<>()); //保證多線程操作的是同一個List
try {
long t1 = System.currentTimeMillis();
int max_one_batch = 100000; // 一批次最多處理100000條數據
List<List<Long>> newList = ListUtils.splitList(listKey, max_one_batch);
int runSize = newList.size(); // 開啓的線程數
/**
* CountDownLanch 只需要在子線程執行之前, 賦予初始化countDownLanch, 並賦予線程數量爲初始值。
* 每個線程執行完畢的時候, 就countDown一下。主線程只需要調用await方法, 可以等待所有子線程執行結束。
*/
final CountDownLatch countDownLatch = new CountDownLatch(runSize); //計數器
/**
* Semaphore(信號量)是用來控制同時訪問特定資源的線程數量,拿到信號量的線程可以進入,否則就等待。
* 通過acquire()和release()獲取和釋放訪問許可。
*/
final Semaphore semaphore = new Semaphore(runSize); //信號量
// 循環創建線程
for (int j = 0; j < runSize; j++) {
final int i = j;
executorService.execute(new Runnable() {
@Override
public void run() {
try {
semaphore.acquire();
// 執行程序
List<Long> subList = newList.get(i);
List<Long> sub_ret = getValue(subList);
list_val.addAll(sub_ret);
log.info(Thread.currentThread().getName() + ": 當前線程/總線程: [" + i + "/" + runSize + "]"
+ ", 處理的數據條數:" + subList.size());
semaphore.release();
} catch (Exception e) {
log.error(e.getMessage(), e);
} finally {
// 計數器減一
countDownLatch.countDown();
}
}
});
}
// 阻塞主線程等待所有線程執行完成
countDownLatch.await();
// 所有線程執行完,之後才能執行的部分
long t2 = System.currentTimeMillis();
log.info("Call getValueList1 success... ret: {} size, threadCount: {}, costs time: {} ms", list_val.size(), runSize, (t2 - t1));
return list_val;
} catch (Exception e) {
return null;
} finally {
// 關閉線程池
executorService.shutdown();
}
}
private List<Long> getValue(List<Long> listKey) {
//具體操作省略
return listKey;
}
}
測試日誌記錄:
01:11:17.515 [pool-1-thread-4] INFO com.test.thread.ConcurrencyTest - pool-1-thread-4: 當前線程/總線程: [3/100], 處理的數據條數:100000
01:11:17.515 [pool-1-thread-8] INFO com.test.thread.ConcurrencyTest - pool-1-thread-8: 當前線程/總線程: [7/100], 處理的數據條數:100000
01:11:17.515 [pool-1-thread-1] INFO com.test.thread.ConcurrencyTest - pool-1-thread-1: 當前線程/總線程: [0/100], 處理的數據條數:100000
01:11:17.515 [pool-1-thread-10] INFO com.test.thread.ConcurrencyTest - pool-1-thread-10: 當前線程/總線程: [9/100], 處理的數據條數:100000
01:11:17.515 [pool-1-thread-2] INFO com.test.thread.ConcurrencyTest - pool-1-thread-2: 當前線程/總線程: [1/100], 處理的數據條數:100000
01:11:17.516 [pool-1-thread-9] INFO com.test.thread.ConcurrencyTest - pool-1-thread-9: 當前線程/總線程: [8/100], 處理的數據條數:100000
01:11:17.515 [pool-1-thread-5] INFO com.test.thread.ConcurrencyTest - pool-1-thread-5: 當前線程/總線程: [4/100], 處理的數據條數:100000
01:11:17.515 [pool-1-thread-3] INFO com.test.thread.ConcurrencyTest - pool-1-thread-3: 當前線程/總線程: [2/100], 處理的數據條數:100000
01:11:17.516 [pool-1-thread-6] INFO com.test.thread.ConcurrencyTest - pool-1-thread-6: 當前線程/總線程: [5/100], 處理的數據條數:100000
01:11:17.516 [pool-1-thread-7] INFO com.test.thread.ConcurrencyTest - pool-1-thread-7: 當前線程/總線程: [6/100], 處理的數據條數:100000
01:11:17.520 [pool-1-thread-4] INFO com.test.thread.ConcurrencyTest - pool-1-thread-4: 當前線程/總線程: [10/100], 處理的數據條數:100000
01:11:17.520 [pool-1-thread-7] INFO com.test.thread.ConcurrencyTest - pool-1-thread-7: 當前線程/總線程: [19/100], 處理的數據條數:100000
01:11:17.520 [pool-1-thread-4] INFO com.test.thread.ConcurrencyTest - pool-1-thread-4: 當前線程/總線程: [20/100], 處理的數據條數:100000
01:11:17.520 [pool-1-thread-6] INFO com.test.thread.ConcurrencyTest - pool-1-thread-6: 當前線程/總線程: [18/100], 處理的數據條數:100000
01:11:17.521 [pool-1-thread-3] INFO com.test.thread.ConcurrencyTest - pool-1-thread-3: 當前線程/總線程: [17/100], 處理的數據條數:100000
01:11:17.522 [pool-1-thread-5] INFO com.test.thread.ConcurrencyTest - pool-1-thread-5: 當前線程/總線程: [16/100], 處理的數據條數:100000
01:11:17.523 [pool-1-thread-9] INFO com.test.thread.ConcurrencyTest - pool-1-thread-9: 當前線程/總線程: [15/100], 處理的數據條數:100000
01:11:17.523 [pool-1-thread-8] INFO com.test.thread.ConcurrencyTest - pool-1-thread-8: 當前線程/總線程: [14/100], 處理的數據條數:100000
01:11:17.523 [pool-1-thread-2] INFO com.test.thread.ConcurrencyTest - pool-1-thread-2: 當前線程/總線程: [13/100], 處理的數據條數:100000
01:11:17.523 [pool-1-thread-10] INFO com.test.thread.ConcurrencyTest - pool-1-thread-10: 當前線程/總線程: [12/100], 處理的數據條數:100000
01:11:17.523 [pool-1-thread-1] INFO com.test.thread.ConcurrencyTest - pool-1-thread-1: 當前線程/總線程: [11/100], 處理的數據條數:100000
01:11:17.523 [pool-1-thread-10] INFO com.test.thread.ConcurrencyTest - pool-1-thread-10: 當前線程/總線程: [29/100], 處理的數據條數:100000
01:11:17.526 [pool-1-thread-2] INFO com.test.thread.ConcurrencyTest - pool-1-thread-2: 當前線程/總線程: [28/100], 處理的數據條數:100000
01:11:17.527 [pool-1-thread-8] INFO com.test.thread.ConcurrencyTest - pool-1-thread-8: 當前線程/總線程: [27/100], 處理的數據條數:100000
01:11:17.527 [pool-1-thread-2] INFO com.test.thread.ConcurrencyTest - pool-1-thread-2: 當前線程/總線程: [32/100], 處理的數據條數:100000
01:11:17.527 [pool-1-thread-9] INFO com.test.thread.ConcurrencyTest - pool-1-thread-9: 當前線程/總線程: [26/100], 處理的數據條數:100000
01:11:17.527 [pool-1-thread-5] INFO com.test.thread.ConcurrencyTest - pool-1-thread-5: 當前線程/總線程: [25/100], 處理的數據條數:100000
01:11:17.527 [pool-1-thread-3] INFO com.test.thread.ConcurrencyTest - pool-1-thread-3: 當前線程/總線程: [24/100], 處理的數據條數:100000
01:11:17.528 [pool-1-thread-6] INFO com.test.thread.ConcurrencyTest - pool-1-thread-6: 當前線程/總線程: [23/100], 處理的數據條數:100000
01:11:17.528 [pool-1-thread-4] INFO com.test.thread.ConcurrencyTest - pool-1-thread-4: 當前線程/總線程: [22/100], 處理的數據條數:100000
01:11:17.528 [pool-1-thread-7] INFO com.test.thread.ConcurrencyTest - pool-1-thread-7: 當前線程/總線程: [21/100], 處理的數據條數:100000
01:11:17.528 [pool-1-thread-4] INFO com.test.thread.ConcurrencyTest - pool-1-thread-4: 當前線程/總線程: [39/100], 處理的數據條數:100000
01:11:17.528 [pool-1-thread-6] INFO com.test.thread.ConcurrencyTest - pool-1-thread-6: 當前線程/總線程: [38/100], 處理的數據條數:100000
01:11:17.528 [pool-1-thread-3] INFO com.test.thread.ConcurrencyTest - pool-1-thread-3: 當前線程/總線程: [37/100], 處理的數據條數:100000
01:11:17.645 [pool-1-thread-5] INFO com.test.thread.ConcurrencyTest - pool-1-thread-5: 當前線程/總線程: [36/100], 處理的數據條數:100000
01:11:17.645 [pool-1-thread-9] INFO com.test.thread.ConcurrencyTest - pool-1-thread-9: 當前線程/總線程: [35/100], 處理的數據條數:100000
01:11:17.646 [pool-1-thread-2] INFO com.test.thread.ConcurrencyTest - pool-1-thread-2: 當前線程/總線程: [34/100], 處理的數據條數:100000
01:11:17.646 [pool-1-thread-8] INFO com.test.thread.ConcurrencyTest - pool-1-thread-8: 當前線程/總線程: [33/100], 處理的數據條數:100000
01:11:17.646 [pool-1-thread-10] INFO com.test.thread.ConcurrencyTest - pool-1-thread-10: 當前線程/總線程: [31/100], 處理的數據條數:100000
01:11:17.646 [pool-1-thread-1] INFO com.test.thread.ConcurrencyTest - pool-1-thread-1: 當前線程/總線程: [30/100], 處理的數據條數:100000
01:11:17.647 [pool-1-thread-10] INFO com.test.thread.ConcurrencyTest - pool-1-thread-10: 當前線程/總線程: [48/100], 處理的數據條數:100000
01:11:17.647 [pool-1-thread-8] INFO com.test.thread.ConcurrencyTest - pool-1-thread-8: 當前線程/總線程: [47/100], 處理的數據條數:100000
01:11:17.647 [pool-1-thread-2] INFO com.test.thread.ConcurrencyTest - pool-1-thread-2: 當前線程/總線程: [46/100], 處理的數據條數:100000
01:11:17.647 [pool-1-thread-9] INFO com.test.thread.ConcurrencyTest - pool-1-thread-9: 當前線程/總線程: [45/100], 處理的數據條數:100000
01:11:17.647 [pool-1-thread-5] INFO com.test.thread.ConcurrencyTest - pool-1-thread-5: 當前線程/總線程: [44/100], 處理的數據條數:100000
01:11:17.648 [pool-1-thread-3] INFO com.test.thread.ConcurrencyTest - pool-1-thread-3: 當前線程/總線程: [43/100], 處理的數據條數:100000
01:11:17.648 [pool-1-thread-6] INFO com.test.thread.ConcurrencyTest - pool-1-thread-6: 當前線程/總線程: [42/100], 處理的數據條數:100000
01:11:17.648 [pool-1-thread-4] INFO com.test.thread.ConcurrencyTest - pool-1-thread-4: 當前線程/總線程: [41/100], 處理的數據條數:100000
01:11:17.648 [pool-1-thread-7] INFO com.test.thread.ConcurrencyTest - pool-1-thread-7: 當前線程/總線程: [40/100], 處理的數據條數:100000
01:11:17.648 [pool-1-thread-4] INFO com.test.thread.ConcurrencyTest - pool-1-thread-4: 當前線程/總線程: [57/100], 處理的數據條數:100000
01:11:17.649 [pool-1-thread-6] INFO com.test.thread.ConcurrencyTest - pool-1-thread-6: 當前線程/總線程: [56/100], 處理的數據條數:100000
01:11:17.655 [pool-1-thread-3] INFO com.test.thread.ConcurrencyTest - pool-1-thread-3: 當前線程/總線程: [55/100], 處理的數據條數:100000
01:11:17.656 [pool-1-thread-9] INFO com.test.thread.ConcurrencyTest - pool-1-thread-9: 當前線程/總線程: [53/100], 處理的數據條數:100000
01:11:17.656 [pool-1-thread-9] INFO com.test.thread.ConcurrencyTest - pool-1-thread-9: 當前線程/總線程: [62/100], 處理的數據條數:100000
01:11:17.655 [pool-1-thread-5] INFO com.test.thread.ConcurrencyTest - pool-1-thread-5: 當前線程/總線程: [54/100], 處理的數據條數:100000
01:11:17.656 [pool-1-thread-2] INFO com.test.thread.ConcurrencyTest - pool-1-thread-2: 當前線程/總線程: [52/100], 處理的數據條數:100000
01:11:17.657 [pool-1-thread-2] INFO com.test.thread.ConcurrencyTest - pool-1-thread-2: 當前線程/總線程: [65/100], 處理的數據條數:100000
01:11:17.657 [pool-1-thread-8] INFO com.test.thread.ConcurrencyTest - pool-1-thread-8: 當前線程/總線程: [51/100], 處理的數據條數:100000
01:11:17.657 [pool-1-thread-10] INFO com.test.thread.ConcurrencyTest - pool-1-thread-10: 當前線程/總線程: [50/100], 處理的數據條數:100000
01:11:17.657 [pool-1-thread-1] INFO com.test.thread.ConcurrencyTest - pool-1-thread-1: 當前線程/總線程: [49/100], 處理的數據條數:100000
01:11:17.657 [pool-1-thread-10] INFO com.test.thread.ConcurrencyTest - pool-1-thread-10: 當前線程/總線程: [68/100], 處理的數據條數:100000
01:11:17.657 [pool-1-thread-8] INFO com.test.thread.ConcurrencyTest - pool-1-thread-8: 當前線程/總線程: [67/100], 處理的數據條數:100000
01:11:17.658 [pool-1-thread-2] INFO com.test.thread.ConcurrencyTest - pool-1-thread-2: 當前線程/總線程: [66/100], 處理的數據條數:100000
01:11:17.658 [pool-1-thread-5] INFO com.test.thread.ConcurrencyTest - pool-1-thread-5: 當前線程/總線程: [64/100], 處理的數據條數:100000
01:11:17.658 [pool-1-thread-9] INFO com.test.thread.ConcurrencyTest - pool-1-thread-9: 當前線程/總線程: [63/100], 處理的數據條數:100000
01:11:17.658 [pool-1-thread-3] INFO com.test.thread.ConcurrencyTest - pool-1-thread-3: 當前線程/總線程: [61/100], 處理的數據條數:100000
01:11:17.659 [pool-1-thread-6] INFO com.test.thread.ConcurrencyTest - pool-1-thread-6: 當前線程/總線程: [60/100], 處理的數據條數:100000
01:11:17.659 [pool-1-thread-4] INFO com.test.thread.ConcurrencyTest - pool-1-thread-4: 當前線程/總線程: [59/100], 處理的數據條數:100000
01:11:17.659 [pool-1-thread-7] INFO com.test.thread.ConcurrencyTest - pool-1-thread-7: 當前線程/總線程: [58/100], 處理的數據條數:100000
01:11:17.659 [pool-1-thread-4] INFO com.test.thread.ConcurrencyTest - pool-1-thread-4: 當前線程/總線程: [77/100], 處理的數據條數:100000
01:11:17.659 [pool-1-thread-6] INFO com.test.thread.ConcurrencyTest - pool-1-thread-6: 當前線程/總線程: [76/100], 處理的數據條數:100000
01:11:17.659 [pool-1-thread-3] INFO com.test.thread.ConcurrencyTest - pool-1-thread-3: 當前線程/總線程: [75/100], 處理的數據條數:100000
01:11:17.660 [pool-1-thread-9] INFO com.test.thread.ConcurrencyTest - pool-1-thread-9: 當前線程/總線程: [74/100], 處理的數據條數:100000
01:11:17.660 [pool-1-thread-5] INFO com.test.thread.ConcurrencyTest - pool-1-thread-5: 當前線程/總線程: [73/100], 處理的數據條數:100000
01:11:17.660 [pool-1-thread-2] INFO com.test.thread.ConcurrencyTest - pool-1-thread-2: 當前線程/總線程: [72/100], 處理的數據條數:100000
01:11:17.660 [pool-1-thread-8] INFO com.test.thread.ConcurrencyTest - pool-1-thread-8: 當前線程/總線程: [71/100], 處理的數據條數:100000
01:11:17.794 [pool-1-thread-10] INFO com.test.thread.ConcurrencyTest - pool-1-thread-10: 當前線程/總線程: [70/100], 處理的數據條數:100000
01:11:17.795 [pool-1-thread-1] INFO com.test.thread.ConcurrencyTest - pool-1-thread-1: 當前線程/總線程: [69/100], 處理的數據條數:100000
01:11:17.795 [pool-1-thread-10] INFO com.test.thread.ConcurrencyTest - pool-1-thread-10: 當前線程/總線程: [86/100], 處理的數據條數:100000
01:11:17.795 [pool-1-thread-8] INFO com.test.thread.ConcurrencyTest - pool-1-thread-8: 當前線程/總線程: [85/100], 處理的數據條數:100000
01:11:17.796 [pool-1-thread-2] INFO com.test.thread.ConcurrencyTest - pool-1-thread-2: 當前線程/總線程: [84/100], 處理的數據條數:100000
01:11:17.796 [pool-1-thread-2] INFO com.test.thread.ConcurrencyTest - pool-1-thread-2: 當前線程/總線程: [90/100], 處理的數據條數:100000
01:11:17.797 [pool-1-thread-2] INFO com.test.thread.ConcurrencyTest - pool-1-thread-2: 當前線程/總線程: [91/100], 處理的數據條數:100000
01:11:17.797 [pool-1-thread-2] INFO com.test.thread.ConcurrencyTest - pool-1-thread-2: 當前線程/總線程: [92/100], 處理的數據條數:100000
01:11:17.797 [pool-1-thread-5] INFO com.test.thread.ConcurrencyTest - pool-1-thread-5: 當前線程/總線程: [83/100], 處理的數據條數:100000
01:11:17.797 [pool-1-thread-9] INFO com.test.thread.ConcurrencyTest - pool-1-thread-9: 當前線程/總線程: [82/100], 處理的數據條數:100000
01:11:17.798 [pool-1-thread-3] INFO com.test.thread.ConcurrencyTest - pool-1-thread-3: 當前線程/總線程: [81/100], 處理的數據條數:100000
01:11:17.798 [pool-1-thread-6] INFO com.test.thread.ConcurrencyTest - pool-1-thread-6: 當前線程/總線程: [80/100], 處理的數據條數:100000
01:11:17.798 [pool-1-thread-4] INFO com.test.thread.ConcurrencyTest - pool-1-thread-4: 當前線程/總線程: [79/100], 處理的數據條數:100000
01:11:17.798 [pool-1-thread-7] INFO com.test.thread.ConcurrencyTest - pool-1-thread-7: 當前線程/總線程: [78/100], 處理的數據條數:100000
01:11:17.798 [pool-1-thread-4] INFO com.test.thread.ConcurrencyTest - pool-1-thread-4: 當前線程/總線程: [98/100], 處理的數據條數:100000
01:11:17.799 [pool-1-thread-6] INFO com.test.thread.ConcurrencyTest - pool-1-thread-6: 當前線程/總線程: [97/100], 處理的數據條數:100000
01:11:17.799 [pool-1-thread-3] INFO com.test.thread.ConcurrencyTest - pool-1-thread-3: 當前線程/總線程: [96/100], 處理的數據條數:100000
01:11:17.799 [pool-1-thread-5] INFO com.test.thread.ConcurrencyTest - pool-1-thread-5: 當前線程/總線程: [94/100], 處理的數據條數:100000
01:11:17.799 [pool-1-thread-9] INFO com.test.thread.ConcurrencyTest - pool-1-thread-9: 當前線程/總線程: [95/100], 處理的數據條數:100000
01:11:17.799 [pool-1-thread-2] INFO com.test.thread.ConcurrencyTest - pool-1-thread-2: 當前線程/總線程: [93/100], 處理的數據條數:100000
01:11:17.800 [pool-1-thread-8] INFO com.test.thread.ConcurrencyTest - pool-1-thread-8: 當前線程/總線程: [89/100], 處理的數據條數:100000
01:11:17.800 [pool-1-thread-10] INFO com.test.thread.ConcurrencyTest - pool-1-thread-10: 當前線程/總線程: [88/100], 處理的數據條數:100000
01:11:17.800 [pool-1-thread-1] INFO com.test.thread.ConcurrencyTest - pool-1-thread-1: 當前線程/總線程: [87/100], 處理的數據條數:100000
01:11:17.800 [pool-1-thread-7] INFO com.test.thread.ConcurrencyTest - pool-1-thread-7: 當前線程/總線程: [99/100], 處理的數據條數:100000
01:11:17.800 [main] INFO com.test.thread.ConcurrencyTest - Call getValueList1 success... ret: 10000000 size, threadCount: 100, costs time: 629 ms
====>> getValueList1 valueList.size: 10000000
3、模擬一千萬請求數據,創建一個線程池10個線程控制最大併發數,設置請求的線程數爲100,平均處理這一千萬請求數據。(用於控制請求線程數的業務場景)
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Semaphore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.test.utils.ListUtils;
public class ConcurrencyTest {
public static void main(String[] args) {
int count = 10000000;
List<Long> listKey = new ArrayList<>();
for (int i = 0; i < count; i++) {
listKey.add(114560315500000000L + i);
}
ConcurrencyTest concurrencyTest = new ConcurrencyTest();
List<Long> valueList2 = concurrencyTest.getValueList2(listKey);
System.out.println("====>> getValueList2 valueList.size: " + valueList2.size());
}
private final static Logger log = LoggerFactory.getLogger(ConcurrencyTest.class);
/**
* 模擬一千萬請求數據,創建一個線程池10個線程控制最大併發數,設置請求的線程數爲100,平均處理這一千萬請求數據。(用於控制請求線程數的業務場景)
* @param listKey 請求處理的總數據量
* @return
*/
public List<Long> getValueList2(List<Long> listKey) {
/**
(1)newCachedThreadPool 創建一個可緩存線程池,如果線程池長度超過處理需要,可靈活回收空閒線程,若無可回收,則新建線程。
(2)newFixedThreadPool 創建一個定長線程池,可控制線程最大併發數,超出的線程會在隊列中等待。
(3)newScheduledThreadPool 創建一個定長線程池,支持定時及週期性任務執行。
(4)newSingleThreadExecutor 創建一個單線程化的線程池,它只會用唯一的工作線程來執行任務,保證所有任務按照指定順序(FIFO, LIFO, 優先級)執行。
*/
//創建一個線程池
final ExecutorService executorService = Executors.newFixedThreadPool(10); //用於控制同時併發執行的線程數,使用線程池創建資源提高效率
List<Long> list_val = Collections.synchronizedList(new ArrayList<>()); //保證多線程操作的是同一個List
try {
long t1 = System.currentTimeMillis();
int threadCount = 100; //請求的線程數
List<List<Long>> newList = ListUtils.avgList(listKey, threadCount);
int runSize = threadCount; // 請求開啓的線程數
/**
* CountDownLanch 只需要在子線程執行之前, 賦予初始化countDownLanch, 並賦予線程數量爲初始值。
* 每個線程執行完畢的時候, 就countDown一下。主線程只需要調用await方法, 可以等待所有子線程執行結束。
*/
final CountDownLatch countDownLatch = new CountDownLatch(runSize); //計數器
/**
* Semaphore(信號量)是用來控制同時訪問特定資源的線程數量,拿到信號量的線程可以進入,否則就等待。
* 通過acquire()和release()獲取和釋放訪問許可。
*/
final Semaphore semaphore = new Semaphore(runSize); //信號量
// 循環創建線程
for (int j = 0; j < runSize; j++) {
final int i = j;
executorService.execute(new Runnable() {
@Override
public void run() {
try {
semaphore.acquire();
// 執行程序
List<Long> subList = newList.get(i);
List<Long> sub_ret = getValue(subList);
list_val.addAll(sub_ret);
log.info(Thread.currentThread().getName() + ": 當前線程/總線程: [" + i + "/" + runSize + "]"
+ ", 處理的數據條數:" + subList.size());
semaphore.release();
} catch (Exception e) {
log.error(e.getMessage(), e);
} finally {
// 計數器減一
countDownLatch.countDown();
}
}
});
}
// 阻塞主線程等待所有線程執行完成
countDownLatch.await();
// 所有線程執行完,之後才能執行的部分
long t2 = System.currentTimeMillis();
log.info("Call getValueList2 success... ret: {} size, threadCount: {}, costs time: {} ms", list_val.size(), runSize, (t2 - t1));
return list_val;
} catch (Exception e) {
return null;
} finally {
// 關閉線程池
executorService.shutdown();
}
}
private List<Long> getValue(List<Long> listKey) {
//具體操作省略
return listKey;
}
}
測試日誌記錄:
01:16:05.076 [pool-1-thread-9] INFO com.test.thread.ConcurrencyTest - pool-1-thread-9: 當前線程/總線程: [8/100], 處理的數據條數:100000
01:16:05.076 [pool-1-thread-10] INFO com.test.thread.ConcurrencyTest - pool-1-thread-10: 當前線程/總線程: [9/100], 處理的數據條數:100000
01:16:05.079 [pool-1-thread-5] INFO com.test.thread.ConcurrencyTest - pool-1-thread-5: 當前線程/總線程: [4/100], 處理的數據條數:100000
01:16:05.075 [pool-1-thread-1] INFO com.test.thread.ConcurrencyTest - pool-1-thread-1: 當前線程/總線程: [0/100], 處理的數據條數:100000
01:16:05.076 [pool-1-thread-8] INFO com.test.thread.ConcurrencyTest - pool-1-thread-8: 當前線程/總線程: [7/100], 處理的數據條數:100000
01:16:05.078 [pool-1-thread-7] INFO com.test.thread.ConcurrencyTest - pool-1-thread-7: 當前線程/總線程: [6/100], 處理的數據條數:100000
01:16:05.081 [pool-1-thread-4] INFO com.test.thread.ConcurrencyTest - pool-1-thread-4: 當前線程/總線程: [3/100], 處理的數據條數:100000
01:16:05.082 [pool-1-thread-6] INFO com.test.thread.ConcurrencyTest - pool-1-thread-6: 當前線程/總線程: [5/100], 處理的數據條數:100000
01:16:05.082 [pool-1-thread-2] INFO com.test.thread.ConcurrencyTest - pool-1-thread-2: 當前線程/總線程: [1/100], 處理的數據條數:100000
01:16:05.083 [pool-1-thread-3] INFO com.test.thread.ConcurrencyTest - pool-1-thread-3: 當前線程/總線程: [2/100], 處理的數據條數:100000
01:16:05.085 [pool-1-thread-2] INFO com.test.thread.ConcurrencyTest - pool-1-thread-2: 當前線程/總線程: [18/100], 處理的數據條數:100000
01:16:05.086 [pool-1-thread-6] INFO com.test.thread.ConcurrencyTest - pool-1-thread-6: 當前線程/總線程: [17/100], 處理的數據條數:100000
01:16:05.086 [pool-1-thread-4] INFO com.test.thread.ConcurrencyTest - pool-1-thread-4: 當前線程/總線程: [16/100], 處理的數據條數:100000
01:16:05.086 [pool-1-thread-7] INFO com.test.thread.ConcurrencyTest - pool-1-thread-7: 當前線程/總線程: [15/100], 處理的數據條數:100000
01:16:05.087 [pool-1-thread-8] INFO com.test.thread.ConcurrencyTest - pool-1-thread-8: 當前線程/總線程: [14/100], 處理的數據條數:100000
01:16:05.088 [pool-1-thread-1] INFO com.test.thread.ConcurrencyTest - pool-1-thread-1: 當前線程/總線程: [13/100], 處理的數據條數:100000
01:16:05.089 [pool-1-thread-5] INFO com.test.thread.ConcurrencyTest - pool-1-thread-5: 當前線程/總線程: [12/100], 處理的數據條數:100000
01:16:05.089 [pool-1-thread-10] INFO com.test.thread.ConcurrencyTest - pool-1-thread-10: 當前線程/總線程: [11/100], 處理的數據條數:100000
01:16:05.089 [pool-1-thread-9] INFO com.test.thread.ConcurrencyTest - pool-1-thread-9: 當前線程/總線程: [10/100], 處理的數據條數:100000
01:16:05.090 [pool-1-thread-10] INFO com.test.thread.ConcurrencyTest - pool-1-thread-10: 當前線程/總線程: [27/100], 處理的數據條數:100000
01:16:05.090 [pool-1-thread-5] INFO com.test.thread.ConcurrencyTest - pool-1-thread-5: 當前線程/總線程: [26/100], 處理的數據條數:100000
01:16:05.090 [pool-1-thread-1] INFO com.test.thread.ConcurrencyTest - pool-1-thread-1: 當前線程/總線程: [25/100], 處理的數據條數:100000
01:16:05.093 [pool-1-thread-8] INFO com.test.thread.ConcurrencyTest - pool-1-thread-8: 當前線程/總線程: [24/100], 處理的數據條數:100000
01:16:05.093 [pool-1-thread-7] INFO com.test.thread.ConcurrencyTest - pool-1-thread-7: 當前線程/總線程: [23/100], 處理的數據條數:100000
01:16:05.094 [pool-1-thread-4] INFO com.test.thread.ConcurrencyTest - pool-1-thread-4: 當前線程/總線程: [22/100], 處理的數據條數:100000
01:16:05.094 [pool-1-thread-6] INFO com.test.thread.ConcurrencyTest - pool-1-thread-6: 當前線程/總線程: [21/100], 處理的數據條數:100000
01:16:05.094 [pool-1-thread-2] INFO com.test.thread.ConcurrencyTest - pool-1-thread-2: 當前線程/總線程: [20/100], 處理的數據條數:100000
01:16:05.095 [pool-1-thread-3] INFO com.test.thread.ConcurrencyTest - pool-1-thread-3: 當前線程/總線程: [19/100], 處理的數據條數:100000
01:16:05.095 [pool-1-thread-2] INFO com.test.thread.ConcurrencyTest - pool-1-thread-2: 當前線程/總線程: [36/100], 處理的數據條數:100000
01:16:05.096 [pool-1-thread-6] INFO com.test.thread.ConcurrencyTest - pool-1-thread-6: 當前線程/總線程: [35/100], 處理的數據條數:100000
01:16:05.096 [pool-1-thread-4] INFO com.test.thread.ConcurrencyTest - pool-1-thread-4: 當前線程/總線程: [34/100], 處理的數據條數:100000
01:16:05.096 [pool-1-thread-7] INFO com.test.thread.ConcurrencyTest - pool-1-thread-7: 當前線程/總線程: [33/100], 處理的數據條數:100000
01:16:05.097 [pool-1-thread-8] INFO com.test.thread.ConcurrencyTest - pool-1-thread-8: 當前線程/總線程: [32/100], 處理的數據條數:100000
01:16:05.097 [pool-1-thread-1] INFO com.test.thread.ConcurrencyTest - pool-1-thread-1: 當前線程/總線程: [31/100], 處理的數據條數:100000
01:16:05.101 [pool-1-thread-5] INFO com.test.thread.ConcurrencyTest - pool-1-thread-5: 當前線程/總線程: [30/100], 處理的數據條數:100000
01:16:05.101 [pool-1-thread-10] INFO com.test.thread.ConcurrencyTest - pool-1-thread-10: 當前線程/總線程: [29/100], 處理的數據條數:100000
01:16:05.102 [pool-1-thread-9] INFO com.test.thread.ConcurrencyTest - pool-1-thread-9: 當前線程/總線程: [28/100], 處理的數據條數:100000
01:16:05.102 [pool-1-thread-10] INFO com.test.thread.ConcurrencyTest - pool-1-thread-10: 當前線程/總線程: [45/100], 處理的數據條數:100000
01:16:05.102 [pool-1-thread-5] INFO com.test.thread.ConcurrencyTest - pool-1-thread-5: 當前線程/總線程: [44/100], 處理的數據條數:100000
01:16:05.103 [pool-1-thread-1] INFO com.test.thread.ConcurrencyTest - pool-1-thread-1: 當前線程/總線程: [43/100], 處理的數據條數:100000
01:16:05.103 [pool-1-thread-8] INFO com.test.thread.ConcurrencyTest - pool-1-thread-8: 當前線程/總線程: [42/100], 處理的數據條數:100000
01:16:05.103 [pool-1-thread-7] INFO com.test.thread.ConcurrencyTest - pool-1-thread-7: 當前線程/總線程: [41/100], 處理的數據條數:100000
01:16:05.104 [pool-1-thread-4] INFO com.test.thread.ConcurrencyTest - pool-1-thread-4: 當前線程/總線程: [40/100], 處理的數據條數:100000
01:16:05.104 [pool-1-thread-6] INFO com.test.thread.ConcurrencyTest - pool-1-thread-6: 當前線程/總線程: [39/100], 處理的數據條數:100000
01:16:05.105 [pool-1-thread-2] INFO com.test.thread.ConcurrencyTest - pool-1-thread-2: 當前線程/總線程: [38/100], 處理的數據條數:100000
01:16:05.105 [pool-1-thread-3] INFO com.test.thread.ConcurrencyTest - pool-1-thread-3: 當前線程/總線程: [37/100], 處理的數據條數:100000
01:16:05.106 [pool-1-thread-2] INFO com.test.thread.ConcurrencyTest - pool-1-thread-2: 當前線程/總線程: [54/100], 處理的數據條數:100000
01:16:05.106 [pool-1-thread-6] INFO com.test.thread.ConcurrencyTest - pool-1-thread-6: 當前線程/總線程: [53/100], 處理的數據條數:100000
01:16:05.106 [pool-1-thread-4] INFO com.test.thread.ConcurrencyTest - pool-1-thread-4: 當前線程/總線程: [52/100], 處理的數據條數:100000
01:16:05.107 [pool-1-thread-7] INFO com.test.thread.ConcurrencyTest - pool-1-thread-7: 當前線程/總線程: [51/100], 處理的數據條數:100000
01:16:05.107 [pool-1-thread-8] INFO com.test.thread.ConcurrencyTest - pool-1-thread-8: 當前線程/總線程: [50/100], 處理的數據條數:100000
01:16:05.246 [pool-1-thread-1] INFO com.test.thread.ConcurrencyTest - pool-1-thread-1: 當前線程/總線程: [49/100], 處理的數據條數:100000
01:16:05.247 [pool-1-thread-5] INFO com.test.thread.ConcurrencyTest - pool-1-thread-5: 當前線程/總線程: [48/100], 處理的數據條數:100000
01:16:05.247 [pool-1-thread-1] INFO com.test.thread.ConcurrencyTest - pool-1-thread-1: 當前線程/總線程: [61/100], 處理的數據條數:100000
01:16:05.248 [pool-1-thread-10] INFO com.test.thread.ConcurrencyTest - pool-1-thread-10: 當前線程/總線程: [47/100], 處理的數據條數:100000
01:16:05.248 [pool-1-thread-9] INFO com.test.thread.ConcurrencyTest - pool-1-thread-9: 當前線程/總線程: [46/100], 處理的數據條數:100000
01:16:05.249 [pool-1-thread-10] INFO com.test.thread.ConcurrencyTest - pool-1-thread-10: 當前線程/總線程: [64/100], 處理的數據條數:100000
01:16:05.249 [pool-1-thread-1] INFO com.test.thread.ConcurrencyTest - pool-1-thread-1: 當前線程/總線程: [63/100], 處理的數據條數:100000
01:16:05.249 [pool-1-thread-5] INFO com.test.thread.ConcurrencyTest - pool-1-thread-5: 當前線程/總線程: [62/100], 處理的數據條數:100000
01:16:05.250 [pool-1-thread-8] INFO com.test.thread.ConcurrencyTest - pool-1-thread-8: 當前線程/總線程: [60/100], 處理的數據條數:100000
01:16:05.250 [pool-1-thread-7] INFO com.test.thread.ConcurrencyTest - pool-1-thread-7: 當前線程/總線程: [59/100], 處理的數據條數:100000
01:16:05.251 [pool-1-thread-4] INFO com.test.thread.ConcurrencyTest - pool-1-thread-4: 當前線程/總線程: [58/100], 處理的數據條數:100000
01:16:05.251 [pool-1-thread-6] INFO com.test.thread.ConcurrencyTest - pool-1-thread-6: 當前線程/總線程: [57/100], 處理的數據條數:100000
01:16:05.251 [pool-1-thread-2] INFO com.test.thread.ConcurrencyTest - pool-1-thread-2: 當前線程/總線程: [56/100], 處理的數據條數:100000
01:16:05.252 [pool-1-thread-3] INFO com.test.thread.ConcurrencyTest - pool-1-thread-3: 當前線程/總線程: [55/100], 處理的數據條數:100000
01:16:05.252 [pool-1-thread-2] INFO com.test.thread.ConcurrencyTest - pool-1-thread-2: 當前線程/總線程: [73/100], 處理的數據條數:100000
01:16:05.252 [pool-1-thread-6] INFO com.test.thread.ConcurrencyTest - pool-1-thread-6: 當前線程/總線程: [72/100], 處理的數據條數:100000
01:16:05.253 [pool-1-thread-4] INFO com.test.thread.ConcurrencyTest - pool-1-thread-4: 當前線程/總線程: [71/100], 處理的數據條數:100000
01:16:05.253 [pool-1-thread-7] INFO com.test.thread.ConcurrencyTest - pool-1-thread-7: 當前線程/總線程: [70/100], 處理的數據條數:100000
01:16:05.254 [pool-1-thread-8] INFO com.test.thread.ConcurrencyTest - pool-1-thread-8: 當前線程/總線程: [69/100], 處理的數據條數:100000
01:16:05.254 [pool-1-thread-5] INFO com.test.thread.ConcurrencyTest - pool-1-thread-5: 當前線程/總線程: [68/100], 處理的數據條數:100000
01:16:05.254 [pool-1-thread-1] INFO com.test.thread.ConcurrencyTest - pool-1-thread-1: 當前線程/總線程: [67/100], 處理的數據條數:100000
01:16:05.255 [pool-1-thread-10] INFO com.test.thread.ConcurrencyTest - pool-1-thread-10: 當前線程/總線程: [66/100], 處理的數據條數:100000
01:16:05.256 [pool-1-thread-9] INFO com.test.thread.ConcurrencyTest - pool-1-thread-9: 當前線程/總線程: [65/100], 處理的數據條數:100000
01:16:05.256 [pool-1-thread-10] INFO com.test.thread.ConcurrencyTest - pool-1-thread-10: 當前線程/總線程: [82/100], 處理的數據條數:100000
01:16:05.257 [pool-1-thread-1] INFO com.test.thread.ConcurrencyTest - pool-1-thread-1: 當前線程/總線程: [81/100], 處理的數據條數:100000
01:16:05.266 [pool-1-thread-5] INFO com.test.thread.ConcurrencyTest - pool-1-thread-5: 當前線程/總線程: [80/100], 處理的數據條數:100000
01:16:05.267 [pool-1-thread-8] INFO com.test.thread.ConcurrencyTest - pool-1-thread-8: 當前線程/總線程: [79/100], 處理的數據條數:100000
01:16:05.267 [pool-1-thread-7] INFO com.test.thread.ConcurrencyTest - pool-1-thread-7: 當前線程/總線程: [78/100], 處理的數據條數:100000
01:16:05.267 [pool-1-thread-4] INFO com.test.thread.ConcurrencyTest - pool-1-thread-4: 當前線程/總線程: [77/100], 處理的數據條數:100000
01:16:05.268 [pool-1-thread-6] INFO com.test.thread.ConcurrencyTest - pool-1-thread-6: 當前線程/總線程: [76/100], 處理的數據條數:100000
01:16:05.268 [pool-1-thread-2] INFO com.test.thread.ConcurrencyTest - pool-1-thread-2: 當前線程/總線程: [75/100], 處理的數據條數:100000
01:16:05.269 [pool-1-thread-3] INFO com.test.thread.ConcurrencyTest - pool-1-thread-3: 當前線程/總線程: [74/100], 處理的數據條數:100000
01:16:05.269 [pool-1-thread-2] INFO com.test.thread.ConcurrencyTest - pool-1-thread-2: 當前線程/總線程: [91/100], 處理的數據條數:100000
01:16:05.270 [pool-1-thread-6] INFO com.test.thread.ConcurrencyTest - pool-1-thread-6: 當前線程/總線程: [90/100], 處理的數據條數:100000
01:16:05.270 [pool-1-thread-4] INFO com.test.thread.ConcurrencyTest - pool-1-thread-4: 當前線程/總線程: [89/100], 處理的數據條數:100000
01:16:05.270 [pool-1-thread-7] INFO com.test.thread.ConcurrencyTest - pool-1-thread-7: 當前線程/總線程: [88/100], 處理的數據條數:100000
01:16:05.271 [pool-1-thread-8] INFO com.test.thread.ConcurrencyTest - pool-1-thread-8: 當前線程/總線程: [87/100], 處理的數據條數:100000
01:16:05.271 [pool-1-thread-5] INFO com.test.thread.ConcurrencyTest - pool-1-thread-5: 當前線程/總線程: [86/100], 處理的數據條數:100000
01:16:05.272 [pool-1-thread-1] INFO com.test.thread.ConcurrencyTest - pool-1-thread-1: 當前線程/總線程: [85/100], 處理的數據條數:100000
01:16:05.272 [pool-1-thread-10] INFO com.test.thread.ConcurrencyTest - pool-1-thread-10: 當前線程/總線程: [84/100], 處理的數據條數:100000
01:16:05.272 [pool-1-thread-9] INFO com.test.thread.ConcurrencyTest - pool-1-thread-9: 當前線程/總線程: [83/100], 處理的數據條數:100000
01:16:05.273 [pool-1-thread-1] INFO com.test.thread.ConcurrencyTest - pool-1-thread-1: 當前線程/總線程: [99/100], 處理的數據條數:100000
01:16:05.273 [pool-1-thread-5] INFO com.test.thread.ConcurrencyTest - pool-1-thread-5: 當前線程/總線程: [98/100], 處理的數據條數:100000
01:16:05.274 [pool-1-thread-8] INFO com.test.thread.ConcurrencyTest - pool-1-thread-8: 當前線程/總線程: [97/100], 處理的數據條數:100000
01:16:05.274 [pool-1-thread-7] INFO com.test.thread.ConcurrencyTest - pool-1-thread-7: 當前線程/總線程: [96/100], 處理的數據條數:100000
01:16:05.276 [pool-1-thread-4] INFO com.test.thread.ConcurrencyTest - pool-1-thread-4: 當前線程/總線程: [95/100], 處理的數據條數:100000
01:16:05.276 [pool-1-thread-6] INFO com.test.thread.ConcurrencyTest - pool-1-thread-6: 當前線程/總線程: [94/100], 處理的數據條數:100000
01:16:05.277 [pool-1-thread-2] INFO com.test.thread.ConcurrencyTest - pool-1-thread-2: 當前線程/總線程: [93/100], 處理的數據條數:100000
01:16:05.277 [pool-1-thread-3] INFO com.test.thread.ConcurrencyTest - pool-1-thread-3: 當前線程/總線程: [92/100], 處理的數據條數:100000
01:16:05.277 [main] INFO com.test.thread.ConcurrencyTest - Call getValueList2 success... ret: 10000000 size, threadCount: 100, costs time: 366 ms
====>> getValueList2 valueList.size: 10000000