【第三方訂單導入運營平臺】SpringBoot整合Elastic-Job分佈式定時任務框架

前言

從第三方獲取訂單，然後進行處理存入我們庫中。

---

◇整體方案

◇數據庫表設計

CREATE TABLE `jd_order` (
  `id` int NOT NULL AUTO_INCREMENT,
  `status` int NOT NULL DEFAULT '0' COMMENT '0:未抓取，1：表示已經抓取',
  `amount` decimal(10,2) NOT NULL COMMENT '訂單金額',
  `create_user` varchar(50) NOT NULL COMMENT '創建人',
  `create_time` datetime NOT NULL,
  `update_user` varchar(50) NOT NULL COMMENT '更新人',
  `update_time` datetime NOT NULL COMMENT '更新時間',
  PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;

CREATE TABLE `tmall_order` (
  `id` int NOT NULL AUTO_INCREMENT COMMENT '訂單id',
  `order_status` int NOT NULL DEFAULT '0' COMMENT '訂單狀態(0:未抓取，1:已抓取)',
  `money` decimal(10,2) NOT NULL COMMENT '訂單金額',
  `create_user` varchar(50) NOT NULL COMMENT '創建人',
  `create_time` datetime NOT NULL COMMENT '創建時間',
  `update_user` varchar(50) NOT NULL COMMENT '更新人',
  `update_time` datetime NOT NULL COMMENT '更新時間',
  PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;

CREATE TABLE `all_order` (
  `id` int NOT NULL AUTO_INCREMENT COMMENT '訂單id',
  `third_order_id` int NOT NULL COMMENT '第三方訂單id',
  `type` int NOT NULL COMMENT '類型：0京東訂單，1天貓訂單',
  `total_amount` decimal(10,2) NOT NULL COMMENT '金額',
  `create_user` varchar(50) NOT NULL COMMENT '創建用戶',
  `create_time` datetime NOT NULL COMMENT '創建時間',
  `update_user` varchar(50) NOT NULL COMMENT '創建用戶',
  `update_time` datetime NOT NULL COMMENT '更新時間',
  PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;

◇模擬第三方訂單表

• 使用SimpleJob模擬第三方訂單產生
• 模擬天貓與京東的訂單
  【每五秒鐘，產生5個訂單】
• 修改數據庫連接地址

server.port=8074
elasticjob.zookeeper.namespace=springboot-elasticjob
elasticjob.zookeeper.server-list=localhost:2181


spring.datasource.username=root
spring.datasource.password=root
spring.datasource.url=jdbc:mysql://localhost:3306/dataflow?serverTimezone=Asia/Shanghai&useSSL=false

mybatis.mapper-locations=/mybatis/*.xml

<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE generatorConfiguration
        PUBLIC "-//mybatis.org//DTD MyBatis Generator Configuration 1.0//EN"
        "http://mybatis.org/dtd/mybatis-generator-config_1_0.dtd">

<generatorConfiguration>

    <context id="MysqlTables" targetRuntime="MyBatis3">
        <jdbcConnection driverClass="com.mysql.cj.jdbc.Driver"
                        connectionURL="jdbc:mysql://localhost:3306/dataflow?serverTimezone=Asia/Shanghai"
                        userId="root"
                        password="root">
            <property name="nullCatalogMeansCurrent" value="true"/>
        </jdbcConnection>

        <javaTypeResolver >
            <property name="forceBigDecimals" value="false" />
        </javaTypeResolver>

        <javaModelGenerator targetPackage="com.zcw.springbootelasticjob.model" targetProject="src\main\java">
            <property name="enableSubPackages" value="true" />
            <property name="trimStrings" value="true" />
        </javaModelGenerator>

        <sqlMapGenerator targetPackage="mybatis"  targetProject="src\main\resources">
            <property name="enableSubPackages" value="true" />
        </sqlMapGenerator>

        <javaClientGenerator type="XMLMAPPER" targetPackage="com.zcw.springbootelasticjob.dao"  targetProject="src\main\java">
            <property name="enableSubPackages" value="true" />
        </javaClientGenerator>

        <table schema="order" tableName="t_order" domainObjectName="Order" >

        </table>

    </context>
</generatorConfiguration>

• 通過逆向工程，生成我們對應的實體類與mapper文件
• 
  
  

編寫第三方訂單產生的訂單任務

• job編寫

 package com.zcw.springbootelasticjob.job;

import com.dangdang.ddframe.job.api.ShardingContext;
import com.dangdang.ddframe.job.api.simple.SimpleJob;
import com.zcw.autoconfig.ElasticSimpleJob;
import com.zcw.springbootelasticjob.service.OrderService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

/**
 * @ClassName : ThirdOrderProduceJob
 * @Description : 模擬第三方產生訂單的job
 * @Author : Zhaocunwei
 * @Date: 2020-06-08 13:05
 */
@Component
@ElasticSimpleJob(
        jobName="thirdOrderProduceJob",
        cron = "0/5 * * * * ？",
        shardingTotalCount = 1,
        overwrite = true
)
public class ThirdOrderProduceJob implements SimpleJob {
    @Autowired
    private OrderService orderService;
    @Override
    public void execute(ShardingContext shardingContext) {
        orderService.produceThirdOrder();
    }
}
  

• 業務類代碼
  

 public void produceThirdOrder() {
        for(int i =0;i<5;i++){
            Random random = new Random();
            int randomInt = random.nextInt(2);
            //0 時生成京東的訂單
            if(randomInt ==0){
                JdOrder jdOrder = new JdOrder();
                jdOrder.setAmount(BigDecimal.TEN);
                jdOrder.setCreateTime(new Date());
                jdOrder.setUpdateTime(new Date());
                jdOrder.setCreateUser("jdUser");
                jdOrder.setUpdateUser("jdUser");
                jdOrder.setStatus(0);//表示爲抓取
                jdOrderMapper.insert(jdOrder);
            }else{
                TmallOrder tmallOrder = new TmallOrder();
                tmallOrder.setCreateTime(new Date());
                tmallOrder.setCreateUser("tmallUser");
                tmallOrder.setMoney(new BigDecimal(100));
                tmallOrder.setOrderStatus(0);
                tmallOrder.setUpdateUser("tmallUser");
                tmallOrder.setUpdateTime(new Date());
                tmallOrderMapper.insert(tmallOrder);
            }
        }
    }

• 編寫Test測試類

• 啓動
  
  
  

◇模擬訂單抓取過程–導入系統

• 使用Dataflow作業完成業務需求
  1.設計分片總數2，分片項0抓取京東訂單，分片項1抓取天貓訂單
  2.Dataflow作業每15秒執行一次，每個分片抓取5個訂單

package com.zcw.autoconfig;

import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;

@Target(ElementType.TYPE)
@Retention(RetentionPolicy.RUNTIME)
public @interface ElasticDataflowJob {
    String jobName() default "";
    String cron() default "";
    int shardingTotalCount() default 1;
    boolean overwrite() default false;
    boolean streamingProcess() default false;
}

   @Transactional(rollbackFor = Exception.class)
    public void processJdOrder(AllOrder allOrder) {
        allOrderMapper.insertSelective(allOrder);
        JdOrder jdOrder = new JdOrder();
        jdOrder.setStatus(1);//已經抓取
        jdOrder.setUpdateUser("system");
        jdOrder.setUpdateTime(new Date());
        jdOrderMapper.updateByPrimaryKeySelective(jdOrder);
    }
    @Transactional(rollbackFor = Exception.class)
    public void processTmallOrder(AllOrder allOrder) {
        allOrderMapper.insertSelective(allOrder);
        TmallOrder tmallOrder= new TmallOrder();
        tmallOrder.setOrderStatus(1);
        tmallOrder.setUpdateTime(new Date());
        tmallOrder.setUpdateUser("system");
        tmallOrderMapper.updateByPrimaryKeySelective(tmallOrder);
    }

核心job

package com.zcw.springbootelasticjob.job;

import com.dangdang.ddframe.job.api.ShardingContext;
import com.dangdang.ddframe.job.api.dataflow.DataflowJob;
import com.zcw.autoconfig.ElasticDataflowJob;
import com.zcw.springbootelasticjob.dao.JdOrderMapper;
import com.zcw.springbootelasticjob.dao.TmallOrderMapper;
import com.zcw.springbootelasticjob.model.AllOrder;
import com.zcw.springbootelasticjob.model.JdOrder;
import com.zcw.springbootelasticjob.model.TmallOrder;
import com.zcw.springbootelasticjob.service.OrderService;
import org.springframework.beans.factory.annotation.Autowired;

import java.util.Date;
import java.util.List;
import java.util.stream.Collectors;

/**
 * @ClassName : FetchThirdOrderJob
 * @Description : 獲取第三方訂單信息
 * @Author : Zhaocunwei
 * @Date: 2020-06-08 13:35
 */
@ElasticDataflowJob(
        jobName = "fetchThirdOrderJob",
        cron = "0/15 * * * * ?",
        shardingTotalCount = 2,
        overwrite = true,
        streamingProcess = true
)
@Component
public class FetchThirdOrderJob implements DataflowJob<Object> {
    @Autowired
    private OrderService orderService;
    @Autowired
    private JdOrderMapper jdOrderMapper;
    @Autowired
    private TmallOrderMapper tmallOrderMapper;
    @Override
    public List<Object> fetchData(ShardingContext shardingContext) {
        if(shardingContext.getShardingItem()==0){
            List<JdOrder> jdOrders = jdOrderMapper.getNotFetchedOrder(5);
            if(jdOrders !=null && jdOrders.size()>0){
                List<Object> jdOrderList = jdOrders.stream().map(jdOrder -> (Object) jdOrder).collect(Collectors.toList());
                return jdOrderList;
            }

        }else{
            List<TmallOrder> tmallOrders = tmallOrderMapper.getNotFetchedOrder(5);
            if(tmallOrders !=null && tmallOrders.size()>0){
                List<Object> tmallOrderList = tmallOrders.stream().map(tmallOrder -> (Object)tmallOrder)
                        .collect(Collectors.toList());
                return tmallOrderList;
            }
        }

        return null;
    }

    @Override
    public void processData(ShardingContext shardingContext, List<Object> data) {
        //根據分片項，區分，是哪裏訂單
        //京東訂單
        if(shardingContext.getShardingItem()==0){
            if(data !=null && data.size()>0){
                List<JdOrder> jdOrders = data.stream().map(d -> (JdOrder) d).collect(Collectors.toList());
                for (JdOrder jdOrder: jdOrders){
                    AllOrder allOrder = new AllOrder();
                    allOrder.setThirdOrderId(jdOrder.getId());
                    allOrder.setType(0);//表示京東ID
                    allOrder.setTotalAmount(jdOrder.getAmount());
                    allOrder.setCreateTime(new Date());
                    allOrder.setCreateUser("system");
                    allOrder.setUpdateTime(new Date());
                    allOrder.setUpdateUser("system");
                    orderService.processJdOrder(allOrder);
                }
            }
        }else{
            //天貓
            if(data!=null&& data.size()>0){
                List<TmallOrder> tmallOrders = data.stream().map(t->(TmallOrder)t).collect(Collectors.toList());
                for(TmallOrder tmallOrder:tmallOrders){
                    AllOrder allOrder = new AllOrder();
                    allOrder.setThirdOrderId(tmallOrder.getId());
                    allOrder.setType(1);//表示天貓訂單
                    allOrder.setTotalAmount(tmallOrder.getMoney());
                    allOrder.setCreateUser("system");
                    allOrder.setCreateTime(new Date());
                    allOrder.setUpdateUser("system");
                    allOrder.setUpdateTime(new Date());
                    orderService.processTmallOrder(allOrder);
                }
            }
        }

    }
}

• 自動配置類

package com.zcw.autoconfig;

/**
 * @ClassName : DataflowJobAutoConfig
 * @Description :
 * @Author : Zhaocunwei
 * @Date: 2020-06-09 12:44
 */
import com.dangdang.ddframe.job.api.ElasticJob;
import com.dangdang.ddframe.job.api.dataflow.DataflowJob;
import com.dangdang.ddframe.job.config.JobCoreConfiguration;
import com.dangdang.ddframe.job.config.dataflow.DataflowJobConfiguration;
import com.dangdang.ddframe.job.lite.api.JobScheduler;
import com.dangdang.ddframe.job.lite.config.LiteJobConfiguration;
import com.dangdang.ddframe.job.lite.spring.api.SpringJobScheduler;
import com.dangdang.ddframe.job.reg.base.CoordinatorRegistryCenter;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.AutoConfigureAfter;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.Configuration;

import java.util.Map;

/**
 * @ClassName : DataflowJobAutoConfig
 * @Description :
 * @Author : Zhaocunwei
 * @Date: 2020-06-05 15:32
 */
@Configuration
@ConditionalOnBean(CoordinatorRegistryCenter.class)
@AutoConfigureAfter(ZookeeperAutoConfig.class)
public class DataflowJobAutoConfig {
    @Autowired
    private CoordinatorRegistryCenter coordinatorRegistryCenter;
    @Autowired
    private ApplicationContext applicationContext;

    public void initDataflowJob(){
        Map<String, Object> beans = applicationContext
                .getBeansWithAnnotation(ElasticDataflowJob.class);
        for(Map.Entry<String,Object> entry:beans.entrySet()){
            Object instance = entry.getValue();
            Class<?>[] interfaces = instance.getClass().getInterfaces();
            for(Class<?> superInterface:interfaces){
                if(superInterface == DataflowJob.class){
                    ElasticDataflowJob annotation = instance.getClass().getAnnotation(ElasticDataflowJob.class);
                    String jobName = annotation.jobName();
                    String cron =annotation.cron();
                    int shardingTotalCount = annotation.shardingTotalCount();
                    boolean overwrite = annotation.overwrite();
                    boolean streamingProcess =annotation.streamingProcess();

                    //job核心配置
                    JobCoreConfiguration buildJobCoreConfiguration = JobCoreConfiguration
                            .newBuilder(jobName, cron, shardingTotalCount)
                            .build();
                    //job類型配置
                    DataflowJobConfiguration dataflowJobConfiguration = new DataflowJobConfiguration(
                            buildJobCoreConfiguration,
                            instance.getClass().getCanonicalName(),
                            streamingProcess);
                    //job根的配置（LiteJobConfiguration）
                    LiteJobConfiguration buildLiteJobConfiguration = LiteJobConfiguration
                            .newBuilder(dataflowJobConfiguration)
                            .overwrite(overwrite)
                            .build();
//                    new JobScheduler(coordinatorRegistryCenter,buildLiteJobConfiguration)
//                            .init();
                    new SpringJobScheduler((ElasticJob) instance,coordinatorRegistryCenter,buildLiteJobConfiguration).init();

                }
            }
        }
    }
}

測試：