elasticJob源碼剖析

 

job的執行過程

首先從demo開始看

 

//com.dangdang.ddframe.job.example.JavaMain
public static void main(final String[] args) throws IOException {
    // CHECKSTYLE:ON
        //zk創建 
        CoordinatorRegistryCenter regCenter = setUpRegistryCenter();
        //作業事件配置
        JobEventConfiguration jobEventConfig = 
        new JobEventRdbConfiguration(setUpEventTraceDataSource());
        setUpSimpleJob(regCenter, jobEventConfig);//-->方法在下面
    }

 

//com.dangdang.ddframe.job.example.JavaMain
//overwrite(true)  是否覆蓋zk的配置,如果爲false,則改動後重啓也不會覆蓋zk已有的配置
private static void setUpSimpleJob(final CoordinatorRegistryCenter regCenter, final JobEventConfiguration jobEventConfig) {
    //作業的核心配置
    JobCoreConfiguration coreConfig = JobCoreConfiguration.newBuilder("javaSimpleJob", "0/1 * * * * ?", 3).shardingItemParameters("0=Beijing,1=Shanghai,2=Guangzhou").build();
    SimpleJobConfiguration simpleJobConfig = new SimpleJobConfiguration(coreConfig, JavaSimpleJob.class.getCanonicalName());
    //創建一個job,並且初始化
    new JobScheduler(regCenter, LiteJobConfiguration.newBuilder(simpleJobConfig).overwrite(true).build(), jobEventConfig).init();
}

構造一個JobScheduler

//io.elasticjob.lite.api.JobScheduler
private JobScheduler(final CoordinatorRegistryCenter regCenter, final LiteJobConfiguration liteJobConfig, final JobEventBus jobEventBus, final ElasticJobListener... elasticJobListeners) {
    //將job任務統一交由註冊器JobRegistry統一管理
    JobRegistry.getInstance().addJobInstance(liteJobConfig.getJobName(), new JobInstance());
    this.liteJobConfig = liteJobConfig;
    this.regCenter = regCenter;
    //彈性化分佈式作業監聽器集合
    List<ElasticJobListener> elasticJobListenerList = Arrays.asList(elasticJobListeners);
    setGuaranteeServiceForElasticJobListeners(regCenter, elasticJobListenerList);
    schedulerFacade = new SchedulerFacade(regCenter, liteJobConfig.getJobName(), elasticJobListenerList);
    jobFacade = new LiteJobFacade(regCenter, liteJobConfig.getJobName(), Arrays.asList(elasticJobListeners), jobEventBus);
}

在init方法初始化作業

 

//io.elasticjob.lite.api.JobScheduler
/**
 * 初始化作業.
 */
public void init() {
    //1.更新作業配置.會判斷,是否存在節點,或者是否LiteJobConfiguration裏面的overwrite是否爲true,如果滿足一個條件就更新配置
    //然後直接從註冊中心而非本地緩存獲取作業節點最新數據.
    LiteJobConfiguration liteJobConfigFromRegCenter = schedulerFacade.updateJobConfiguration(liteJobConfig);
    //2.設置當前分片總數.和作業名稱
    JobRegistry.getInstance().setCurrentShardingTotalCount(liteJobConfigFromRegCenter.getJobName(), liteJobConfigFromRegCenter.getTypeConfig().getCoreConfig().getShardingTotalCount());
    //3.創建一個作業調度控制器.
    JobScheduleController jobScheduleController = new JobScheduleController(
            //3.1根據配置文件創建一個調度器
            createScheduler(),
            //3.2根據執行任務的class創建一個調度器詳情實例
            createJobDetail(liteJobConfigFromRegCenter.getTypeConfig().getJobClass()),
            //3.3獲取job名稱
            liteJobConfigFromRegCenter.getJobName());
    //4.job實例設置job的名稱,作業調度控制器,和用於協調分佈式服務的註冊中心.
    JobRegistry.getInstance().registerJob(liteJobConfigFromRegCenter.getJobName(), jobScheduleController, regCenter);
    //5.註冊並且啓動信息,啓動各種監聽器
    schedulerFacade.registerStartUpInfo(!liteJobConfigFromRegCenter.isDisabled());
    //6.根據cron表達式調度作業
    jobScheduleController.scheduleJob(liteJobConfigFromRegCenter.getTypeConfig().getCoreConfig().getCron());
}

1.1

/**
 * 更新作業配置.
 * SchedulerFacade.java
 * @param liteJobConfig 作業配置
 * @return 更新後的作業配置
 */
public LiteJobConfiguration updateJobConfiguration(final LiteJobConfiguration liteJobConfig) {
    //更新或者添加作業配置
    configService.persist(liteJobConfig);
    //從zk去獲取更新後的作業配置
    return configService.load(false);
}

1.2.1

/**
 * 讀取作業配置.
 * 
 * @param fromCache 是否從緩存中讀取
 * @return 作業配置
 */
public LiteJobConfiguration load(final boolean fromCache) {
    String result;
    if (fromCache) {
        result = jobNodeStorage.getJobNodeData(ConfigurationNode.ROOT);
        if (null == result) {
            result = jobNodeStorage.getJobNodeDataDirectly(ConfigurationNode.ROOT);
        }
    } else {
        //直接從註冊中心而非本地緩存獲取作業節點數據.
        result = jobNodeStorage.getJobNodeDataDirectly(ConfigurationNode.ROOT);
    }
    return LiteJobConfigurationGsonFactory.fromJson(result);
}

2.1

/**
 * 設置當前分片總數.
 *
 * @param jobName 作業名稱
 * @param currentShardingTotalCount 當前分片總數
 */
public void setCurrentShardingTotalCount(final String jobName, final int currentShardingTotalCount) {
    currentShardingTotalCountMap.put(jobName, currentShardingTotalCount);
}

3.1.1

//io.elasticjob.lite.api.JobScheduler
/**
 * 創建調度程序
 * @return
 */
private Scheduler createScheduler() {
    Scheduler result;
    try {
        StdSchedulerFactory factory = new StdSchedulerFactory();
        //獲取基本Quartz的屬性
        factory.initialize(getBaseQuartzProperties());
        //從工廠獲取一個調度程序
        result = factory.getScheduler();
        //往調度程序添加一個作業觸發監聽器
        result.getListenerManager().addTriggerListener(schedulerFacade.newJobTriggerListener());
    } catch (final SchedulerException ex) {
        throw new JobSystemException(ex);
    }
    return result;
}

/**
 * 獲取基本Quartz的屬性
 * @return
 */
private Properties getBaseQuartzProperties() {
    Properties result = new Properties();
    result.put("org.quartz.threadPool.class", org.quartz.simpl.SimpleThreadPool.class.getName());
    result.put("org.quartz.threadPool.threadCount", "1");
    result.put("org.quartz.scheduler.instanceName", liteJobConfig.getJobName());
    result.put("org.quartz.jobStore.misfireThreshold", "1");
    result.put("org.quartz.plugin.shutdownhook.class", JobShutdownHookPlugin.class.getName());
    result.put("org.quartz.plugin.shutdownhook.cleanShutdown", Boolean.TRUE.toString());
    return result;
}

3.2.1

//io.elasticjob.lite.api.JobScheduler
/**
 * 創建一個job詳情
 * @param jobClass
 * @return
 */
private JobDetail createJobDetail(final String jobClass) {
    //3.2.1.1通過建造者模式,創建一個調度作業
    JobDetail result = JobBuilder.newJob(LiteJob.class).withIdentity(liteJobConfig.getJobName()).build();
    result.getJobDataMap().put(JOB_FACADE_DATA_MAP_KEY, jobFacade);
    Optional<ElasticJob> elasticJobInstance = createElasticJobInstance();
    //flase
    if (elasticJobInstance.isPresent()) {
        result.getJobDataMap().put(ELASTIC_JOB_DATA_MAP_KEY, elasticJobInstance.get());
    } else if (!jobClass.equals(ScriptJob.class.getCanonicalName())) {
        try {
            //把job的實例放進jobDataMap
            result.getJobDataMap().put(ELASTIC_JOB_DATA_MAP_KEY, Class.forName(jobClass).newInstance());
        } catch (final ReflectiveOperationException ex) {
            throw new JobConfigurationException("Elastic-Job: Job class '%s' can not initialize.", jobClass);
        }
    }
    return result;
}

3.2.1.1具體創建一個調度作業的流程

//io.elasticjob.lite.internal.schedule.LiteJob
/**
 * Lite調度作業.
 *
 * @author zhangliang
 */
public final class LiteJob implements Job {
    
    @Setter
    private ElasticJob elasticJob;
    
    @Setter
    private JobFacade jobFacade;
    
    @Override
    public void execute(final JobExecutionContext context) throws JobExecutionException {
        //通過 JobExecutorFactory 獲得到作業執行器並進行執行
        JobExecutorFactory.getJobExecutor(elasticJob, jobFacade).execute();
    }
}

根據不同類型獲取不同的作業執行器

//io.elasticjob.lite.executor.JobExecutorFactory
/**
 * 作業執行器工廠.
 *
 * @author zhangliang
 */
@NoArgsConstructor(access = AccessLevel.PRIVATE)
public final class JobExecutorFactory {
    
    /**
     * 獲取作業執行器.
     * 首先會根據elasticJob的類型去找到相應的執行器
     * @param elasticJob 分佈式彈性作業
     * @param jobFacade 作業內部服務門面服務
     * @return 作業執行器
     */
    @SuppressWarnings("unchecked")
    public static AbstractElasticJobExecutor getJobExecutor(final ElasticJob elasticJob, final JobFacade jobFacade) {
        if (null == elasticJob) {
            return new ScriptJobExecutor(jobFacade);
        }
        if (elasticJob instanceof SimpleJob) {
            return new SimpleJobExecutor((SimpleJob) elasticJob, jobFacade);
        }
        if (elasticJob instanceof DataflowJob) {
            return new DataflowJobExecutor((DataflowJob) elasticJob, jobFacade);
        }
        throw new JobConfigurationException("Cannot support job type '%s'", elasticJob.getClass().getCanonicalName());
    }
}

執行作業

//io.elasticjob.lite.executor.AbstractElasticJobExecutor
/**
 * 執行作業.
 */
public final void execute() {
    // 檢查 作業執行環境
    try {
        jobFacade.checkJobExecutionEnvironment();
    } catch (final JobExecutionEnvironmentException cause) {
        jobExceptionHandler.handleException(jobName, cause);
    }
    // 獲取 當前作業服務器的分片上下文
    ShardingContexts shardingContexts = jobFacade.getShardingContexts();
    // 發佈作業狀態追蹤事件(State.TASK_STAGING)
    if (shardingContexts.isAllowSendJobEvent()) {
        jobFacade.postJobStatusTraceEvent(shardingContexts.getTaskId(), State.TASK_STAGING, String.format("Job '%s' execute begin.", jobName));
    }
    // 跳過 存在運行中的被錯過作業
    if (jobFacade.misfireIfRunning(shardingContexts.getShardingItemParameters().keySet())) {
        // 發佈作業狀態追蹤事件(State.TASK_FINISHED)
        if (shardingContexts.isAllowSendJobEvent()) {
            jobFacade.postJobStatusTraceEvent(shardingContexts.getTaskId(), State.TASK_FINISHED, String.format(
                    "Previous job '%s' - shardingItems '%s' is still running, misfired job will start after previous job completed.", jobName, 
                    shardingContexts.getShardingItemParameters().keySet()));
        }
        return;
    }
    // 執行 作業執行前的方法
    try {
        jobFacade.beforeJobExecuted(shardingContexts);
        //CHECKSTYLE:OFF
    } catch (final Throwable cause) {
        //CHECKSTYLE:ON
        jobExceptionHandler.handleException(jobName, cause);
    }
    // 執行 普通觸發的作業
    execute(shardingContexts, JobExecutionEvent.ExecutionSource.NORMAL_TRIGGER);
    // 執行 被跳過觸發的作業
    while (jobFacade.isExecuteMisfired(shardingContexts.getShardingItemParameters().keySet())) {
        jobFacade.clearMisfire(shardingContexts.getShardingItemParameters().keySet());
        execute(shardingContexts, JobExecutionEvent.ExecutionSource.MISFIRE);
    }
    // 執行 作業失效轉移
    jobFacade.failoverIfNecessary();
    // 執行 作業執行後的方法
    try {
        jobFacade.afterJobExecuted(shardingContexts);
        //CHECKSTYLE:OFF
    } catch (final Throwable cause) {
        //CHECKSTYLE:ON
        jobExceptionHandler.handleException(jobName, cause);
    }
}

//io.elasticjob.lite.executor.AbstractElasticJobExecutor
private void execute(final ShardingContexts shardingContexts, final JobExecutionEvent.ExecutionSource executionSource) {
    // 無可執行的分片，發佈作業狀態追蹤事件
    if (shardingContexts.getShardingItemParameters().isEmpty()) {
        if (shardingContexts.isAllowSendJobEvent()) {
            jobFacade.postJobStatusTraceEvent(shardingContexts.getTaskId(), State.TASK_FINISHED, String.format("Sharding item for job '%s' is empty.", jobName));
        }
        return;
    }
    // 註冊作業啓動信息
    jobFacade.registerJobBegin(shardingContexts);
    // 發佈作業狀態追蹤事件(State.TASK_RUNNING)
    String taskId = shardingContexts.getTaskId();
    if (shardingContexts.isAllowSendJobEvent()) {
        jobFacade.postJobStatusTraceEvent(taskId, State.TASK_RUNNING, "");
    }
    try {
        //執行作業
        process(shardingContexts, executionSource);
    } finally {
        // TODO 考慮增加作業失敗的狀態，並且考慮如何處理作業失敗的整體迴路
        // 註冊作業完成信息
        jobFacade.registerJobCompleted(shardingContexts);
        // 根據是否有異常，發佈作業狀態追蹤事件
        if (itemErrorMessages.isEmpty()) {
            if (shardingContexts.isAllowSendJobEvent()) {
                jobFacade.postJobStatusTraceEvent(taskId, State.TASK_FINISHED, "");
            }
        } else {
            if (shardingContexts.isAllowSendJobEvent()) {
                jobFacade.postJobStatusTraceEvent(taskId, State.TASK_ERROR, itemErrorMessages.toString());
            }
        }
    }
}

//io.elasticjob.lite.executor.AbstractElasticJobExecutor
/**
 *  執行作業
 * @param shardingContexts 分片上下文集合.
 * @param executionSource 執行來源
 */
private void process(final ShardingContexts shardingContexts, final JobExecutionEvent.ExecutionSource executionSource) {
    Collection<Integer> items = shardingContexts.getShardingItemParameters().keySet();
    // 1個分片，直接執行
    if (1 == items.size()) {
        int item = shardingContexts.getShardingItemParameters().keySet().iterator().next();
        JobExecutionEvent jobExecutionEvent = new JobExecutionEvent(shardingContexts.getTaskId(), jobName, executionSource, item);
        // 執行一個作業
        process(shardingContexts, item, jobExecutionEvent);
        return;
    }
    // 多分片，並行執行
    final CountDownLatch latch = new CountDownLatch(items.size());
    for (final int each : items) {
        final JobExecutionEvent jobExecutionEvent = new JobExecutionEvent(shardingContexts.getTaskId(), jobName, executionSource, each);
        if (executorService.isShutdown()) {
            return;
        }
        //作業執行線程池
        executorService.submit(new Runnable() {

            @Override
            public void run() {
                try {
                    // 執行一個作業
                    process(shardingContexts, each, jobExecutionEvent);
                } finally {
                    latch.countDown();
                }
            }
        });
    }
    // 等待多分片全部完成
    try {
        latch.await();
    } catch (final InterruptedException ex) {
        Thread.currentThread().interrupt();
    }
}

private void process(final ShardingContexts shardingContexts, final int item, final JobExecutionEvent startEvent) {
    // 發佈執行事件(開始)
    if (shardingContexts.isAllowSendJobEvent()) {
        jobFacade.postJobExecutionEvent(startEvent);
    }
    log.trace("Job '{}' executing, item is: '{}'.", jobName, item);
    JobExecutionEvent completeEvent;
    try {
        // 執行單個作業
        process(new ShardingContext(shardingContexts, item));
        // 發佈執行事件(成功)
        completeEvent = startEvent.executionSuccess();
        log.trace("Job '{}' executed, item is: '{}'.", jobName, item);
        if (shardingContexts.isAllowSendJobEvent()) {
            jobFacade.postJobExecutionEvent(completeEvent);
        }
        // CHECKSTYLE:OFF
    } catch (final Throwable cause) {
        // CHECKSTYLE:ON
        // 發佈執行事件(失敗)
        completeEvent = startEvent.executionFailure(cause);
        jobFacade.postJobExecutionEvent(completeEvent);
        // 設置該分片執行異常信息
        itemErrorMessages.put(item, ExceptionUtil.transform(cause));
        jobExceptionHandler.handleException(jobName, cause);
    }
}

5.1

//io.elasticjob.lite.internal.schedule.SchedulerFacade
/**
 * 註冊作業啓動信息.
 * 
 * @param enabled 作業是否啓用
 */
public void registerStartUpInfo(final boolean enabled) {
    //開啓所有監聽器.
    listenerManager.startAllListeners();
    //選舉主節點.
    leaderService.electLeader();
    //持久化作業服務器上線信息.
    serverService.persistOnline(enabled);
    //持久化作業運行實例上線相關信息.
    instanceService.persistOnline();
    //設置需要重新分片的標記.
    shardingService.setReshardingFlag();
    //初始化作業監聽服務.
    monitorService.listen();
    //如果沒運行就運行起來
    if (!reconcileService.isRunning()) {
        reconcileService.startAsync();
    }
}

6.1 最後執行調度作業

//io.elasticjob.lite.internal.schedule.SchedulerFacade
/**
 * 調度作業.
 * 
 * @param cron CRON表達式
 */
public void scheduleJob(final String cron) {
    try {
        if (!scheduler.checkExists(jobDetail.getKey())) {
            scheduler.scheduleJob(jobDetail, createTrigger(cron));
        }
        scheduler.start();
    } catch (final SchedulerException ex) {
        throw new JobSystemException(ex);
    }
}

重新分片的4種情況:

1.註冊作業啓動信息時

2.作業分片總數變化時

3.服務器變化時

    服務器變化有2種情況

        1.服務器被開啓或禁用

        2.作業節點新增或移除

4.待補充

只有在主節點的時候才分片

執行失敗後會把失敗信息放進itemErrorMessages的map裏面,然後下面就會判斷執行是否有異常,如果有異常就重試

失效轉移是通過監聽器,監聽zk的節點變化來執行的

通過FailoverListenerManager這個失效轉移監聽管理器來監聽zk的節點

 

最後附上執行任務流程圖