場景：多個任務提交到redis的list中，多機器部署，每臺機器都是在啓動項目的時候啓動這個從redis的list中pop任務進行處理，如果redis的list中等待timeout=2s沒有數據就會再次去list中去取task；任務處理比較耗時，大概將近60s處理完畢
- 基於redis的分佈式任務提交處理的線程池
public abstract class AbstractDistributedTaskHandler extends ThreadPoolExecutor {
private final Logger logger = LoggerFactory.getLogger(getClass());
//根據自己的redis進行相應的調整
private ListCommands listCommands;
private HashCommands hashCommands;
private boolean running;
private final HashMap<Integer, ProjectFutureTask> tasks = new HashMap<>(100);
private TimeUnit taskTimeoutTimeUnit = TimeUnit.MINUTES;
private long taskTimeout = 30;
public AbstractDistributedTaskHandler() {
this(5, "task");
}
/**
* 構造當前節點線程池
*
* @param poolSize 用於提交用戶的線程池大小，另外增加2個內部線程用於提交和檢測超時
*/
public AbstractDistributedTaskHandler(int poolSize, String poolThreadName) {
super(poolSize + 2, poolSize + 2, 10, TimeUnit.SECONDS, new ArrayBlockingQueue<Runnable>(100), new NamedThreadFactory(poolThreadName), new AbortPolicy());
}
/**
* 初始化方法
*/
public final void init() {
listCommands = RedisFactory.getClusterListCommands(getRedisGroupName());
hashCommands = RedisFactory.getClusterHashCommands(getRedisGroupName());
}
/**
* 開始啓動接收分佈式線程
*
* @param taskTimeoutTimeUnit 每個線程超時時間單位
* @param taskTimeout 每個線程超時時間
*/
public final void start(TimeUnit taskTimeoutTimeUnit, long taskTimeout) {
this.taskTimeoutTimeUnit = taskTimeoutTimeUnit;
this.taskTimeout = taskTimeout;
this.running = true;
execute(() -> fetchNewTask());
execute(() -> checkTaskTimeout());
}
@Override
public void shutdown() {
running = false;
super.shutdown();
}
/**
* 開始從redis獲取任務並提交到線程池
*/
private void fetchNewTask() {
while (running) {
if (getActiveCount() >= getCorePoolSize()) {
try {
TimeUnit.SECONDS.sleep(3);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
continue;
}
Task task = listCommands.rightPop(getRedisNamespace(), getTaskQueueRedisKey(), 2, TimeUnit.SECONDS, true);
if (task != null) {
ProjectFutureTask future = new ProjectFutureTask(task);
execute(future);
tasks.put(task.getTaskId(), future);
}
}
}
/**
* 開始線程池中的線程超時檢查
*/
private void checkTaskTimeout() {
while (running) {
Iterator<Map.Entry<Integer, ProjectFutureTask>> it = tasks.entrySet().iterator();
while (it.hasNext()) {
ProjectFutureTask futureTask = it.next().getValue();
if (futureTask.isDone() || futureTask.isCancelled()) {
it.remove();
} else if (System.currentTimeMillis() - futureTask.getStartTime() > taskTimeoutTimeUnit.toMillis(taskTimeout)) {
logger.warn("force cancel task: {}", futureTask.getTask());
futureTask.cancel(true);
}
}
try {
TimeUnit.MINUTES.sleep(1);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
}
/**
* 封裝的提交線程池的FutureTask
*/
private class ProjectFutureTask extends FutureTask {
private Task task;
private long startTime = System.currentTimeMillis();
Task getTask() {
return task;
}
public long getStartTime() {
return startTime;
}
public ProjectFutureTask(Task task) {
super(new Callable<String>() {
@Override
public String call() throws Exception {
return taskHandler(task);
}
});
this.task = task;
}
}
/**
* 獲取記錄任務信息的redis key名稱
*
* @param taskId 任務ID
* @return redis key名稱
*/
public static final String getTaskRedisKey(Integer taskId) {
return String.format("project:tasks:flinktasks:%s", taskId);
}
/**
* 獲取任務隊列的redis key名稱
*
* @return redis key名稱
*/
private String getTaskQueueRedisKey() {
return String.format("project:tasks:%s", getRedisTaskQueueName());
}
/**
* 提交到任務，分佈式提交，不一定當前節點執行該任務
*
* @param task 任務
*/
public final void submitTask(Task task) {
String taskKey = getTaskRedisKey(task.getTaskId());
Boolean succeed = hashCommands.putIfAbsent(getRedisNamespace(), taskKey, "taskId", task.getTaskId(), true);
if (!succeed) {
throw new TaskExistException(String.format("task: %s exist", task.getTaskId()));
}
Map<String, Object> map = new HashMap<>(10);
map.put("taskId", task.getTaskId());
map.put("taskName", task.getTaskName());
map.put("taskStatus", task.getActionEnum().getInitial());
map.put("taskSubmitTime", System.currentTimeMillis());
long keyExpire = 12;
hashCommands.expire(getRedisNamespace(), taskKey, keyExpire, TimeUnit.HOURS, true);
hashCommands.putAll(getRedisNamespace(), taskKey, map, true);
String taskQueueKey = getTaskQueueRedisKey();
listCommands.leftPush(true, getRedisNamespace(), taskQueueKey, task);
listCommands.expire(getRedisNamespace(), taskQueueKey, keyExpire, TimeUnit.HOURS, true);
logger.info("submit task: {}", task);
}
/**
* 獲取在排隊等待執行的任務，不包括正在執行中的
*
* @return
*/
public final List<Task> getQueueTasks() {
return listCommands.range(getRedisNamespace(), getTaskQueueRedisKey(), 0, -1, true);
}
/**
* 獲取任務的信息和執行狀態
*
* @param taskIds
* @return
*/
public final Map<Integer, Integer> getTaskStatus(List<Integer> taskIds) {
List<String> taskKeys = new ArrayList<>();
for (Integer taskId : taskIds) {
taskKeys.add(getTaskRedisKey(taskId));
}
Map<Integer, Integer> map = new HashMap<>(50);
List<Map<String, Object>> entries = hashCommands.multiEntries(getRedisNamespace(), taskKeys, true, false);
if (entries != null) {
for (Map<String, Object> entry : entries) {
if (entry.isEmpty()) {
continue;
}
map.put((Integer) entry.get("taskId"), (Integer) entry.get("taskStatus"));
}
}
return map;
}
/**
* 獲取任務的信息和執行狀態
*
* @param taskId
* @return
*/
public final Integer getTaskStatus(Integer taskId) {
String taskKey = getTaskRedisKey(taskId);
return hashCommands.get(getRedisNamespace(), taskKey, "taskStatus", true);
}
/**
* 取消任務
*
* @param task 任務
* @param force 是否取消正在執行的
*/
public final void cancelTask(Task task, boolean force) {
String taskKey = getTaskRedisKey(task.getTaskId());
if (!hashCommands.persist(getRedisNamespace(), taskKey, true)) {
throw new TaskNotExistException(String.format("task: %s not exist", task.getTaskId()));
}
hashCommands.delete(getRedisNamespace(), taskKey, true);
listCommands.remove(getRedisNamespace(), getTaskQueueRedisKey(), 0, task, true);
if (force && tasks.containsKey(task.getTaskId())) {
tasks.get(task.getTaskId()).cancel(true);
}
}
/**
* 任務線程開始執行前
*
* @param t
* @param r
*/
@Override
protected void beforeExecute(Thread t, Runnable r) {
super.beforeExecute(t, r);
if (!(r instanceof ProjectFutureTask)) {
return;
}
ProjectFutureTask futureTask = (ProjectFutureTask) r;
logger.debug("beforeExecute task: {}", futureTask.getTask());
String taskKey = getTaskRedisKey(futureTask.getTask().getTaskId());
hashCommands.put(getRedisNamespace(), taskKey, "taskStatus", futureTask.getTask().getActionEnum().getProcessing(), true);
hashCommands.expire(getRedisNamespace(), taskKey, taskTimeout, taskTimeoutTimeUnit, true);
}
/**
* 任務線程開始執行後
*
* @param r
* @param t
*/
@Override
protected void afterExecute(Runnable r, Throwable t) {
super.afterExecute(r, t);
if (t != null) {
logger.error("Execute Error:", t);
}
if (!(r instanceof ProjectFutureTask)) {
return;
}
ProjectFutureTask futureTask = (ProjectFutureTask) r;
logger.debug("afterExecute task: {}", futureTask.getTask());
String taskKey = getTaskRedisKey(futureTask.getTask().getTaskId());
hashCommands.delete(getRedisNamespace(), taskKey, true);
}
/**
* 獲取redis的GroupName
*
* @return
*/
public abstract String getRedisGroupName();
/**
* 獲取redis的Namespace
*
* @return
*/
public abstract int getRedisNamespace();
/**
* 獲取redis的任務隊列名稱
*
* @return
*/
public abstract String getRedisTaskQueueName();
/**
* 線程中執行任務的具體方法
*
* @param task 任務
* @return 任務執行結果
* @throws Exception 執行異常
*/
public abstract String taskHandler(Task task) throws Exception;
}
- 分佈式任務對象接口
public interface Task extends Serializable {
/**
* 任務ID
* @return 任務ID
*/
public Integer getTaskId();
/**
* 任務名稱
* @return 任務名稱
*/
public String getTaskName();
/**
* 任務的動作枚舉
* @return 任務的動作枚舉
*/
public TaskActionEnum getActionEnum();
}
- 分佈式任務行爲類型
public enum TaskActionEnum {
/**
* 準備啓動、啓動中
*/
START(TaskStatusEnum.PRESTART, TaskStatusEnum.STARTING),
/**
* 準備停止、停止中
*/
STOP(TaskStatusEnum.PRESTOP, TaskStatusEnum.STOPING),
/**
* 準備遷移、遷移中
*/
MIGRATE(TaskStatusEnum.PREMIGRATE,TaskStatusEnum.MIGRATEING);
private TaskStatusEnum initial;
private TaskStatusEnum processing;
TaskActionEnum(TaskStatusEnum initial, TaskStatusEnum processing) {
this.initial = initial;
this.processing = processing;
}
public Integer getInitial() {
return initial.getValue();
}
public Integer getProcessing() {
return processing.getValue();
}
}
- 分佈式任務狀態
public enum TaskStatusEnum {
/**
* 未開始
*/
WAIT(0, "未開始"),
/**
* 運行中
*/
RUNNING(1, "運行中"),
/**
* 異常中止
*/
ERROR(2, "異常中止"),
/**
* 終止運行
*/
STOP(3, "終止運行"),
/**
* 等待啓動
*/
PRESTART(4, "等待啓動"),
/**
* 啓動中
*/
STARTING(5, "啓動中"),
/**
* 等待停止
*/
PRESTOP(6, "等待停止"),
/**
* 停止中
*/
STOPING(7, "停止中"),
/**
* 等待遷移
*/
PREMIGRATE(8, "等待遷移"),
/**
* 遷移中
*/
MIGRATEING(9, "遷移中");
private int value;
private String description;
private TaskStatusEnum(int value, String desc) {
this.value = value;
this.description = desc;
}
public int getValue() {
return this.value;
}
public String getDescription() {
return this.description;
}
public static String getDescription(int index) {
for (TaskStatusEnum i : TaskStatusEnum.values()) {
if (i.getValue() == index) {
return i.description;
}
}
return "";
}
}
- 任務已經存在異常
public class TaskExistException extends RuntimeException implements Serializable {
public TaskExistException() {
super();
}
public TaskExistException(String message) {
super(message);
}
public TaskExistException(String message, Throwable cause) {
super(message, cause);
}
}
- 任務不存在異常
public class TaskNotExistException extends RuntimeException implements Serializable {
public TaskNotExistException() {
super();
}
public TaskNotExistException(String message) {
super(message);
}
public TaskNotExistException(String message, Throwable cause) {
super(message, cause);
}
}
- 具體的任務處理， 對task進行邏輯處理，如果提交完了任務，需要定時的檢查參見
延遲和定期執行的線程池分佈式並行處理模板代碼
@Component
@Slf4j
public class FlinkSubmitTaskHandler extends AbstractDistributedTaskHandler implements CommandLineRunner {
public FlinkSubmitTaskHandler() {
super(5, "submittask");
}
/**
* 實現spring的CommandLineRunner接口，在項目啓動之後會spring會自動調用run方法
*/
@Override
public void run(String... strings) throws Exception {
init();
//刪除平臺文件夾，防止該文件夾磁盤滿了
deleteLocalFlinkHomeAndFlinkSqlAndUserJobs();
//開發環境不啓動
if (!EnvironmentConstant.DEVELOPMENT.equals(poseidonxConfig.getEnvironment())) {
start(TimeUnit.MINUTES, poseidonxConfig.getFlinkTaskStartThreadTimeoutMinutes());
}
}
/**
* 刪除平臺本地flinkHome和flinkSql和userJobs文件夾
*/
private void deleteLocalFlinkHomeAndFlinkSqlAndUserJobs() {
// String flinkHomeLocation = poseidonxConfig.getFlinkHomeLocation();
// FileService.deleteAllFile(new File(flinkHomeLocation));
// String flinkSqlLocation = poseidonxConfig.getFlinkSqlLocation();
// FileService.deleteAllFile(new File(flinkSqlLocation));
String userJobsLocation = poseidonxConfig.getUserJobsLocation();
FileService.deleteAllFile(new File(userJobsLocation));
}
@Override
public String getRedisGroupName() {
return poseidonxConfig.getRedisGroupName();
}
@Override
public int getRedisNamespace() {
return poseidonxConfig.getRedisNamespace();
}
@Override
public String getRedisTaskQueueName() {
return "flinksubmittasks";
}
@Override
public String taskHandler(Task task) throws Exception {
log.info("$$$$$$$$$$$$$ begin submit task, task: {}", task);
//todo 對task進行邏輯處理，如果提交完了任務，需要定時的檢查參見
log.info("$$$$$$$$$$$$$ end submit task, task: {}", task);
}
}
- 線程池命名工廠類
public class NamedThreadFactory implements ThreadFactory {
private final AtomicInteger mThreadNum = new AtomicInteger(1);
private final String mPrefix;
private final boolean mDaemo;
private final ThreadGroup mGroup;
public NamedThreadFactory(String prefix) {
this(prefix, false);
}
public NamedThreadFactory(String prefix, boolean daemo) {
mPrefix = prefix + "-thread-";
mDaemo = daemo;
SecurityManager s = System.getSecurityManager();
mGroup = (s == null) ? Thread.currentThread().getThreadGroup() : s.getThreadGroup();
}
@Override
public Thread newThread(Runnable runnable) {
String name = mPrefix + mThreadNum.getAndIncrement();
Thread ret = new Thread(mGroup, runnable, name, 0);
ret.setDaemon(mDaemo);
return ret;
}
}
