job的执行过程
首先从demo开始看
//com.dangdang.ddframe.job.example.JavaMain
public static void main(final String[] args) throws IOException {
// CHECKSTYLE:ON
//zk创建
CoordinatorRegistryCenter regCenter = setUpRegistryCenter();
//作业事件配置
JobEventConfiguration jobEventConfig =
new JobEventRdbConfiguration(setUpEventTraceDataSource());
setUpSimpleJob(regCenter, jobEventConfig);//-->方法在下面
}
//com.dangdang.ddframe.job.example.JavaMain
//overwrite(true) 是否覆盖zk的配置,如果为false,则改动后重启也不会覆盖zk已有的配置
private static void setUpSimpleJob(final CoordinatorRegistryCenter regCenter, final JobEventConfiguration jobEventConfig) {
//作业的核心配置
JobCoreConfiguration coreConfig = JobCoreConfiguration.newBuilder("javaSimpleJob", "0/1 * * * * ?", 3).shardingItemParameters("0=Beijing,1=Shanghai,2=Guangzhou").build();
SimpleJobConfiguration simpleJobConfig = new SimpleJobConfiguration(coreConfig, JavaSimpleJob.class.getCanonicalName());
//创建一个job,并且初始化
new JobScheduler(regCenter, LiteJobConfiguration.newBuilder(simpleJobConfig).overwrite(true).build(), jobEventConfig).init();
}
构造一个JobScheduler
//io.elasticjob.lite.api.JobScheduler
private JobScheduler(final CoordinatorRegistryCenter regCenter, final LiteJobConfiguration liteJobConfig, final JobEventBus jobEventBus, final ElasticJobListener... elasticJobListeners) {
//将job任务统一交由注册器JobRegistry统一管理
JobRegistry.getInstance().addJobInstance(liteJobConfig.getJobName(), new JobInstance());
this.liteJobConfig = liteJobConfig;
this.regCenter = regCenter;
//弹性化分布式作业监听器集合
List<ElasticJobListener> elasticJobListenerList = Arrays.asList(elasticJobListeners);
setGuaranteeServiceForElasticJobListeners(regCenter, elasticJobListenerList);
schedulerFacade = new SchedulerFacade(regCenter, liteJobConfig.getJobName(), elasticJobListenerList);
jobFacade = new LiteJobFacade(regCenter, liteJobConfig.getJobName(), Arrays.asList(elasticJobListeners), jobEventBus);
}
在init方法初始化作业
//io.elasticjob.lite.api.JobScheduler
/**
* 初始化作业.
*/
public void init() {
//1.更新作业配置.会判断,是否存在节点,或者是否LiteJobConfiguration里面的overwrite是否为true,如果满足一个条件就更新配置
//然后直接从注册中心而非本地缓存获取作业节点最新数据.
LiteJobConfiguration liteJobConfigFromRegCenter = schedulerFacade.updateJobConfiguration(liteJobConfig);
//2.设置当前分片总数.和作业名称
JobRegistry.getInstance().setCurrentShardingTotalCount(liteJobConfigFromRegCenter.getJobName(), liteJobConfigFromRegCenter.getTypeConfig().getCoreConfig().getShardingTotalCount());
//3.创建一个作业调度控制器.
JobScheduleController jobScheduleController = new JobScheduleController(
//3.1根据配置文件创建一个调度器
createScheduler(),
//3.2根据执行任务的class创建一个调度器详情实例
createJobDetail(liteJobConfigFromRegCenter.getTypeConfig().getJobClass()),
//3.3获取job名称
liteJobConfigFromRegCenter.getJobName());
//4.job实例设置job的名称,作业调度控制器,和用于协调分布式服务的注册中心.
JobRegistry.getInstance().registerJob(liteJobConfigFromRegCenter.getJobName(), jobScheduleController, regCenter);
//5.注册并且启动信息,启动各种监听器
schedulerFacade.registerStartUpInfo(!liteJobConfigFromRegCenter.isDisabled());
//6.根据cron表达式调度作业
jobScheduleController.scheduleJob(liteJobConfigFromRegCenter.getTypeConfig().getCoreConfig().getCron());
}
1.1
/**
* 更新作业配置.
* SchedulerFacade.java
* @param liteJobConfig 作业配置
* @return 更新后的作业配置
*/
public LiteJobConfiguration updateJobConfiguration(final LiteJobConfiguration liteJobConfig) {
//更新或者添加作业配置
configService.persist(liteJobConfig);
//从zk去获取更新后的作业配置
return configService.load(false);
}
1.2.1
/**
* 读取作业配置.
*
* @param fromCache 是否从缓存中读取
* @return 作业配置
*/
public LiteJobConfiguration load(final boolean fromCache) {
String result;
if (fromCache) {
result = jobNodeStorage.getJobNodeData(ConfigurationNode.ROOT);
if (null == result) {
result = jobNodeStorage.getJobNodeDataDirectly(ConfigurationNode.ROOT);
}
} else {
//直接从注册中心而非本地缓存获取作业节点数据.
result = jobNodeStorage.getJobNodeDataDirectly(ConfigurationNode.ROOT);
}
return LiteJobConfigurationGsonFactory.fromJson(result);
}
2.1
/**
* 设置当前分片总数.
*
* @param jobName 作业名称
* @param currentShardingTotalCount 当前分片总数
*/
public void setCurrentShardingTotalCount(final String jobName, final int currentShardingTotalCount) {
currentShardingTotalCountMap.put(jobName, currentShardingTotalCount);
}
3.1.1
//io.elasticjob.lite.api.JobScheduler
/**
* 创建调度程序
* @return
*/
private Scheduler createScheduler() {
Scheduler result;
try {
StdSchedulerFactory factory = new StdSchedulerFactory();
//获取基本Quartz的属性
factory.initialize(getBaseQuartzProperties());
//从工厂获取一个调度程序
result = factory.getScheduler();
//往调度程序添加一个作业触发监听器
result.getListenerManager().addTriggerListener(schedulerFacade.newJobTriggerListener());
} catch (final SchedulerException ex) {
throw new JobSystemException(ex);
}
return result;
}
/**
* 获取基本Quartz的属性
* @return
*/
private Properties getBaseQuartzProperties() {
Properties result = new Properties();
result.put("org.quartz.threadPool.class", org.quartz.simpl.SimpleThreadPool.class.getName());
result.put("org.quartz.threadPool.threadCount", "1");
result.put("org.quartz.scheduler.instanceName", liteJobConfig.getJobName());
result.put("org.quartz.jobStore.misfireThreshold", "1");
result.put("org.quartz.plugin.shutdownhook.class", JobShutdownHookPlugin.class.getName());
result.put("org.quartz.plugin.shutdownhook.cleanShutdown", Boolean.TRUE.toString());
return result;
}
3.2.1
//io.elasticjob.lite.api.JobScheduler
/**
* 创建一个job详情
* @param jobClass
* @return
*/
private JobDetail createJobDetail(final String jobClass) {
//3.2.1.1通过建造者模式,创建一个调度作业
JobDetail result = JobBuilder.newJob(LiteJob.class).withIdentity(liteJobConfig.getJobName()).build();
result.getJobDataMap().put(JOB_FACADE_DATA_MAP_KEY, jobFacade);
Optional<ElasticJob> elasticJobInstance = createElasticJobInstance();
//flase
if (elasticJobInstance.isPresent()) {
result.getJobDataMap().put(ELASTIC_JOB_DATA_MAP_KEY, elasticJobInstance.get());
} else if (!jobClass.equals(ScriptJob.class.getCanonicalName())) {
try {
//把job的实例放进jobDataMap
result.getJobDataMap().put(ELASTIC_JOB_DATA_MAP_KEY, Class.forName(jobClass).newInstance());
} catch (final ReflectiveOperationException ex) {
throw new JobConfigurationException("Elastic-Job: Job class '%s' can not initialize.", jobClass);
}
}
return result;
}
3.2.1.1具体创建一个调度作业的流程
//io.elasticjob.lite.internal.schedule.LiteJob
/**
* Lite调度作业.
*
* @author zhangliang
*/
public final class LiteJob implements Job {
@Setter
private ElasticJob elasticJob;
@Setter
private JobFacade jobFacade;
@Override
public void execute(final JobExecutionContext context) throws JobExecutionException {
//通过 JobExecutorFactory 获得到作业执行器并进行执行
JobExecutorFactory.getJobExecutor(elasticJob, jobFacade).execute();
}
}
根据不同类型获取不同的作业执行器
//io.elasticjob.lite.executor.JobExecutorFactory
/**
* 作业执行器工厂.
*
* @author zhangliang
*/
@NoArgsConstructor(access = AccessLevel.PRIVATE)
public final class JobExecutorFactory {
/**
* 获取作业执行器.
* 首先会根据elasticJob的类型去找到相应的执行器
* @param elasticJob 分布式弹性作业
* @param jobFacade 作业内部服务门面服务
* @return 作业执行器
*/
@SuppressWarnings("unchecked")
public static AbstractElasticJobExecutor getJobExecutor(final ElasticJob elasticJob, final JobFacade jobFacade) {
if (null == elasticJob) {
return new ScriptJobExecutor(jobFacade);
}
if (elasticJob instanceof SimpleJob) {
return new SimpleJobExecutor((SimpleJob) elasticJob, jobFacade);
}
if (elasticJob instanceof DataflowJob) {
return new DataflowJobExecutor((DataflowJob) elasticJob, jobFacade);
}
throw new JobConfigurationException("Cannot support job type '%s'", elasticJob.getClass().getCanonicalName());
}
}
执行作业
//io.elasticjob.lite.executor.AbstractElasticJobExecutor
/**
* 执行作业.
*/
public final void execute() {
// 检查 作业执行环境
try {
jobFacade.checkJobExecutionEnvironment();
} catch (final JobExecutionEnvironmentException cause) {
jobExceptionHandler.handleException(jobName, cause);
}
// 获取 当前作业服务器的分片上下文
ShardingContexts shardingContexts = jobFacade.getShardingContexts();
// 发布作业状态追踪事件(State.TASK_STAGING)
if (shardingContexts.isAllowSendJobEvent()) {
jobFacade.postJobStatusTraceEvent(shardingContexts.getTaskId(), State.TASK_STAGING, String.format("Job '%s' execute begin.", jobName));
}
// 跳过 存在运行中的被错过作业
if (jobFacade.misfireIfRunning(shardingContexts.getShardingItemParameters().keySet())) {
// 发布作业状态追踪事件(State.TASK_FINISHED)
if (shardingContexts.isAllowSendJobEvent()) {
jobFacade.postJobStatusTraceEvent(shardingContexts.getTaskId(), State.TASK_FINISHED, String.format(
"Previous job '%s' - shardingItems '%s' is still running, misfired job will start after previous job completed.", jobName,
shardingContexts.getShardingItemParameters().keySet()));
}
return;
}
// 执行 作业执行前的方法
try {
jobFacade.beforeJobExecuted(shardingContexts);
//CHECKSTYLE:OFF
} catch (final Throwable cause) {
//CHECKSTYLE:ON
jobExceptionHandler.handleException(jobName, cause);
}
// 执行 普通触发的作业
execute(shardingContexts, JobExecutionEvent.ExecutionSource.NORMAL_TRIGGER);
// 执行 被跳过触发的作业
while (jobFacade.isExecuteMisfired(shardingContexts.getShardingItemParameters().keySet())) {
jobFacade.clearMisfire(shardingContexts.getShardingItemParameters().keySet());
execute(shardingContexts, JobExecutionEvent.ExecutionSource.MISFIRE);
}
// 执行 作业失效转移
jobFacade.failoverIfNecessary();
// 执行 作业执行后的方法
try {
jobFacade.afterJobExecuted(shardingContexts);
//CHECKSTYLE:OFF
} catch (final Throwable cause) {
//CHECKSTYLE:ON
jobExceptionHandler.handleException(jobName, cause);
}
}
//io.elasticjob.lite.executor.AbstractElasticJobExecutor
private void execute(final ShardingContexts shardingContexts, final JobExecutionEvent.ExecutionSource executionSource) {
// 无可执行的分片,发布作业状态追踪事件
if (shardingContexts.getShardingItemParameters().isEmpty()) {
if (shardingContexts.isAllowSendJobEvent()) {
jobFacade.postJobStatusTraceEvent(shardingContexts.getTaskId(), State.TASK_FINISHED, String.format("Sharding item for job '%s' is empty.", jobName));
}
return;
}
// 注册作业启动信息
jobFacade.registerJobBegin(shardingContexts);
// 发布作业状态追踪事件(State.TASK_RUNNING)
String taskId = shardingContexts.getTaskId();
if (shardingContexts.isAllowSendJobEvent()) {
jobFacade.postJobStatusTraceEvent(taskId, State.TASK_RUNNING, "");
}
try {
//执行作业
process(shardingContexts, executionSource);
} finally {
// TODO 考虑增加作业失败的状态,并且考虑如何处理作业失败的整体回路
// 注册作业完成信息
jobFacade.registerJobCompleted(shardingContexts);
// 根据是否有异常,发布作业状态追踪事件
if (itemErrorMessages.isEmpty()) {
if (shardingContexts.isAllowSendJobEvent()) {
jobFacade.postJobStatusTraceEvent(taskId, State.TASK_FINISHED, "");
}
} else {
if (shardingContexts.isAllowSendJobEvent()) {
jobFacade.postJobStatusTraceEvent(taskId, State.TASK_ERROR, itemErrorMessages.toString());
}
}
}
}
//io.elasticjob.lite.executor.AbstractElasticJobExecutor
/**
* 执行作业
* @param shardingContexts 分片上下文集合.
* @param executionSource 执行来源
*/
private void process(final ShardingContexts shardingContexts, final JobExecutionEvent.ExecutionSource executionSource) {
Collection<Integer> items = shardingContexts.getShardingItemParameters().keySet();
// 1个分片,直接执行
if (1 == items.size()) {
int item = shardingContexts.getShardingItemParameters().keySet().iterator().next();
JobExecutionEvent jobExecutionEvent = new JobExecutionEvent(shardingContexts.getTaskId(), jobName, executionSource, item);
// 执行一个作业
process(shardingContexts, item, jobExecutionEvent);
return;
}
// 多分片,并行执行
final CountDownLatch latch = new CountDownLatch(items.size());
for (final int each : items) {
final JobExecutionEvent jobExecutionEvent = new JobExecutionEvent(shardingContexts.getTaskId(), jobName, executionSource, each);
if (executorService.isShutdown()) {
return;
}
//作业执行线程池
executorService.submit(new Runnable() {
@Override
public void run() {
try {
// 执行一个作业
process(shardingContexts, each, jobExecutionEvent);
} finally {
latch.countDown();
}
}
});
}
// 等待多分片全部完成
try {
latch.await();
} catch (final InterruptedException ex) {
Thread.currentThread().interrupt();
}
}
private void process(final ShardingContexts shardingContexts, final int item, final JobExecutionEvent startEvent) {
// 发布执行事件(开始)
if (shardingContexts.isAllowSendJobEvent()) {
jobFacade.postJobExecutionEvent(startEvent);
}
log.trace("Job '{}' executing, item is: '{}'.", jobName, item);
JobExecutionEvent completeEvent;
try {
// 执行单个作业
process(new ShardingContext(shardingContexts, item));
// 发布执行事件(成功)
completeEvent = startEvent.executionSuccess();
log.trace("Job '{}' executed, item is: '{}'.", jobName, item);
if (shardingContexts.isAllowSendJobEvent()) {
jobFacade.postJobExecutionEvent(completeEvent);
}
// CHECKSTYLE:OFF
} catch (final Throwable cause) {
// CHECKSTYLE:ON
// 发布执行事件(失败)
completeEvent = startEvent.executionFailure(cause);
jobFacade.postJobExecutionEvent(completeEvent);
// 设置该分片执行异常信息
itemErrorMessages.put(item, ExceptionUtil.transform(cause));
jobExceptionHandler.handleException(jobName, cause);
}
}
5.1
//io.elasticjob.lite.internal.schedule.SchedulerFacade
/**
* 注册作业启动信息.
*
* @param enabled 作业是否启用
*/
public void registerStartUpInfo(final boolean enabled) {
//开启所有监听器.
listenerManager.startAllListeners();
//选举主节点.
leaderService.electLeader();
//持久化作业服务器上线信息.
serverService.persistOnline(enabled);
//持久化作业运行实例上线相关信息.
instanceService.persistOnline();
//设置需要重新分片的标记.
shardingService.setReshardingFlag();
//初始化作业监听服务.
monitorService.listen();
//如果没运行就运行起来
if (!reconcileService.isRunning()) {
reconcileService.startAsync();
}
}
6.1 最后执行调度作业
//io.elasticjob.lite.internal.schedule.SchedulerFacade
/**
* 调度作业.
*
* @param cron CRON表达式
*/
public void scheduleJob(final String cron) {
try {
if (!scheduler.checkExists(jobDetail.getKey())) {
scheduler.scheduleJob(jobDetail, createTrigger(cron));
}
scheduler.start();
} catch (final SchedulerException ex) {
throw new JobSystemException(ex);
}
}
重新分片的4种情况:
1.注册作业启动信息时
2.作业分片总数变化时
3.服务器变化时
服务器变化有2种情况
1.服务器被开启或禁用
2.作业节点新增或移除
4.待补充
只有在主节点的时候才分片
执行失败后会把失败信息放进itemErrorMessages的map里面,然后下面就会判断执行是否有异常,如果有异常就重试
失效转移是通过监听器,监听zk的节点变化来执行的
通过FailoverListenerManager这个失效转移监听管理器来监听zk的节点
最后附上执行任务流程图