實時數據集成--數據一致性校驗

一、背景

      目前實時抽數系統不太穩定，數據庫、kafka、網絡等問題，導致實時抽取的數據存在誤差。

二、方案

      方案一：監控kafka採集的數量與spark streaming從kafka消費的數量（不能保證源庫和目標庫的數量一致）；

      方案二：監控源庫當天截止到某時的數量與目標庫當日分區截止某時的數量；

      方案三：監控源庫截止到某時的數量與目標庫增量合併後截止到某時的數量(數據量會比較大)；

三、設計

採用方案二

1、創建兩張表：數據庫配置、表配置

• 數據庫配置：主要包括源庫ip、端口、庫名、用戶名、密碼(加密)，通常配從庫的；
• 表配置：主要包括源表名、目標表名、數據庫配置id、主鍵、增量時間、是否開啓監控；

2、開發加解密工具類--使用AES算法；

3、開發定時任務

• jdbc連接配置庫查詢需要監控的表；
• 線程池開啓多線程處理業務邏輯；
• 查詢源庫數量；
• 查詢目標庫hive數量；
• 比較兩者，若不同調告警接口；

四、實現

AES加解密工具類：

package com.aaaaa.realtimeappmonitor.jobstatetrack.util;

import sun.misc.BASE64Decoder;
import sun.misc.BASE64Encoder;

import javax.crypto.Cipher;
import javax.crypto.KeyGenerator;
import java.security.Key;
import java.security.SecureRandom;

/**
 * Description AES加密解密工具類
 *
 * @author Bob
 * @date 2020/4/26
 **/
public class AESUtil {

    public static String CIPHER_ALGORITHM = "AES"; // optional value AES/DES/DESede

    public static Key getKey(String strKey) {
        try {
            if (strKey == null) {
                strKey = "";
            }
            KeyGenerator _generator = KeyGenerator.getInstance("AES");
            SecureRandom secureRandom = SecureRandom.getInstance("SHA1PRNG");
            secureRandom.setSeed(strKey.getBytes());
            _generator.init(128, secureRandom);
            return _generator.generateKey();
        } catch (Exception e) {
            throw new RuntimeException(" 初始化密鑰出現異常 ");
        }
    }


    public static String encrypt(String data, String key) throws Exception {
        SecureRandom sr = new SecureRandom();
        Key secureKey = getKey(key);
        Cipher cipher = Cipher.getInstance(CIPHER_ALGORITHM);
        cipher.init(Cipher.ENCRYPT_MODE, secureKey, sr);
        byte[] bt = cipher.doFinal(data.getBytes());
        String strS = new BASE64Encoder().encode(bt);
        return strS;
    }


    public static String decrypt(String message, String key) {
        try {
            SecureRandom sr = new SecureRandom();
            Cipher cipher = Cipher.getInstance(CIPHER_ALGORITHM);
            Key secureKey = getKey(key);
            cipher.init(Cipher.DECRYPT_MODE, secureKey, sr);
            byte[] res = new BASE64Decoder().decodeBuffer(message);
            res = cipher.doFinal(res);
            return new String(res);
        } catch (Exception e) {
            e.printStackTrace();
        }
        return null;
    }

}

定時任務：

package com.aaa.realtimeappmonitor.jobstatetrack.task;

import com.google.common.collect.Lists;
import com.aaa.realtimeappmonitor.jobstatetrack.model.TableDbConfigInfo;
import com.aaa.realtimeappmonitor.jobstatetrack.util.AESUtil;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import org.springframework.web.client.RestClientException;
import org.springframework.web.client.RestTemplate;

import java.sql.*;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.Date;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;

/**
 * Description 源庫與落地庫數據量監控(方案一)
 * 方案一：統計源庫當天截止某時數量,比較增量當日分區截止某時數量
 * 方案二：統計源庫截止某時數量，比較合併後全庫截止某時數量
 *
 * @author Bob
 * @date 2020/4/23
 **/
@Slf4j
@Component
public class StatisticMonitorTask {

    @Value("${data.validation.monitor.driver}")
    private String driver;
    @Value("${data.validation.monitor.url}")
    private String url;
    @Value("${data.validation.monitor.username}")
    private String username;
    @Value("${data.validation.monitor.password}")
    private String password;
    @Value("${data.validation.monitor.schema}")
    private String schema;
    @Value("${data.validation.monitor.target.driver}")
    private String targetDriver;
    @Value("${data.validation.monitor.target.url}")
    private String targetUrl;
    @Value("${data.validation.monitor.target.username}")
    private String targetUsername;
    @Value("${data.validation.monitor.target.password}")
    private String targetPassword;
    @Value("${data.validation.monitor.aes.key}")
    private String AES_KEY;
    @Value("${data.validation.monitor.work.no}")
    private String workNo;
    private static final int CORE_POOL_SIZE = 5;
    private static final int MAXIMUM_POOL_SIZE = 5;
    private static final long KEEP_ALIVE_TIME = 0L;

    @Scheduled(cron = "${data.validation.monitor.cron}")
    public void statisticMonitorTask() {
        log.info("Start timing task:monitor statistic...");
        Date date = new Date();
        SimpleDateFormat sdf1 = new SimpleDateFormat("yyyyMMdd");
        SimpleDateFormat sdf2 = new SimpleDateFormat("yyyyMMddHH");
        SimpleDateFormat sdf3 = new SimpleDateFormat("HH");
        String day = sdf1.format(date);
        String dayHour = sdf2.format(date);
        String hour = sdf3.format(date);
        //0點比較前一日數量
        if ("00".equals(hour)) {
            Calendar calendar = Calendar.getInstance();
            calendar.setTime(date);
            calendar.add(Calendar.DAY_OF_YEAR, -1);
            day = sdf1.format(calendar.getTime());
        }
        List<TableDbConfigInfo> monitorTables = getMonitorTables();
        // 開啓多線程
        ThreadPoolExecutor threadPoolExecutor = new ThreadPoolExecutor(CORE_POOL_SIZE, MAXIMUM_POOL_SIZE, KEEP_ALIVE_TIME, TimeUnit.MILLISECONDS, new LinkedBlockingDeque<Runnable>());
        for (TableDbConfigInfo info : monitorTables) {
            try {
                threadPoolExecutor.execute(new CountCheck(info, day, dayHour));
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
        threadPoolExecutor.shutdown();
        log.info("End timing task:monitor statistic...");
    }

    /**
     * @description 查詢需要監控的表
     * @author Bob
     * @date 2020/4/28
     */
    public List<TableDbConfigInfo> getMonitorTables() {
        // 查詢需要統計的數據
        Connection conn = null;
        PreparedStatement preparedStatement = null;
        ResultSet rs = null;
        List<TableDbConfigInfo> list = Lists.newArrayList();
        String sql = "select a.source_schema_name as sourceSchemaName,a.source_table_name as sourceTableName,a.integration_db_alias as integrationDbAlias,a.row_key as rowKey,a.integration_sys_name as integrationSysName,\n" +
                "a.target_table_name as targetTableName,a.src_db_id as srcDbId,a.presto_view_name as prestoViewName,a.alias_create_time as aliasCreateTime,a.alias_update_time as aliasUpdateTime,\n" +
                "b.src_db_name as srcDbName,b.src_db_type as srcDbType,b.src_db_ip_master as srcDbIpMaster,b.src_db_port_master as srcDbPortMaster,\n" +
                "b.src_db_ip_slave as srcDbIpSlave,b.src_db_port_slave as srcDbPortSlave,b.integration_db_alias as integrationDbAlias,b.integration_sys_name as integrationSysName,\n" +
                "b.master_db_username as masterDbUsername,b.master_db_passwd as masterDbPasswd,b.slave_db_username as slaveDbUsername,b.slave_db_passwd as slaveDbPasswd\n" +
                "from " + schema + ".integration_job_info a\n" +
                "join " + schema + ".integration_src_db_info b\n" +
                "on a.src_db_id=b.id\n" +
                "where a.is_monitor='1';";
        try {
            conn = getConn(driver, url, username, password);
            preparedStatement = conn.prepareStatement(sql);
            rs = preparedStatement.executeQuery();
            while (rs.next()) {
                TableDbConfigInfo info = new TableDbConfigInfo();
                info.setSrcDbName(rs.getString("srcDbName"));
                info.setSrcDbType(rs.getString("srcDbType"));
                info.setSrcDbIpSlave(rs.getString("srcDbIpSlave"));
                info.setSrcDbPortSlave(rs.getString("srcDbPortSlave"));
                info.setSlaveDbUsername(rs.getString("slaveDbUsername"));
                info.setSlaveDbPasswd(rs.getString("slaveDbPasswd"));
                info.setSourceTableName(rs.getString("sourceTableName"));
                info.setTargetTableName(rs.getString("targetTableName"));
                info.setSrcDbId(rs.getLong("srcDbId"));
                info.setRowKey(rs.getString("rowKey"));
                info.setAliasCreateTime(rs.getString("aliasCreateTime"));
                info.setAliasUpdateTime(rs.getString("aliasUpdateTime"));
                list.add(info);
            }
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            closeConn(rs, preparedStatement, conn);
        }
        return list;
    }

    class CountCheck implements Runnable {
        private TableDbConfigInfo info;
        private String day;
        private String dayHour;

        public CountCheck(TableDbConfigInfo info, String day, String dayHour) {
            this.info = info;
            this.day = day;
            this.dayHour = dayHour;
        }

        @Override
        public void run() {
            // 查詢源庫表數據量s1
            Long sCount = countSource(info, day, dayHour);
            // 查詢落地庫數據量s2
            Long tCount = countTarget(info, day, dayHour);
            StringBuffer sb = new StringBuffer();
            // 比較s1、s2
            if (sCount.compareTo(tCount) != 0) {
                log.warn("源庫和目標庫數量不一致,源表:" + info.getSourceTableName() + ",數量:" + sCount + "目標表:" + info.getTargetTableName() + ",數量:" + tCount);
                sb.append("源庫表名:").append(info.getSourceTableName()).append(",數量:").append(sCount).append(",目標庫表名:").append(info.getTargetTableName()).append(",數量:").append(tCount).append(",相差:").append(Math.abs(sCount - tCount)).append("\n");
                // 調用告警接口
                send("實時數據集成監控告警:實時數據不一致", sb.toString());
            }
        }
    }

    /**
     * @description 統計源庫數量
     * @author Bob
     * @date 2020/4/28
     */
    public Long countSource(TableDbConfigInfo info, String day, String dayHour) {
        log.info("-------------countSource table:" + info.getSourceTableName() + " start---------------");
        Long sCount = 0L;
        Connection conn = null;
        PreparedStatement preparedStatement = null;
        ResultSet rs = null;
        String sourceDriver = null;
        String sourceUrl = null;
        String srcDbIpSlave = info.getSrcDbIpSlave();
        String srcDbPortSlave = info.getSrcDbPortSlave();
        String srcDbName = info.getSrcDbName();
        String sourceUsername = info.getSlaveDbUsername();
        String sourcePassword = AESUtil.decrypt(info.getSlaveDbPasswd(), AES_KEY);
        String srcDbType = info.getSrcDbType();
        String sourceTableName = info.getSourceTableName();
        String aliasCreateTime = info.getAliasCreateTime();
        String aliasUpdateTime = info.getAliasUpdateTime();
        StringBuffer sb = new StringBuffer();
        sb.append("select count(1) as cn from ").append(sourceTableName);
        sb.append(" where 1=1 ");
        if ("postgresql".equalsIgnoreCase(srcDbType)) {
            sourceDriver = "org.postgresql.Driver";
            sourceUrl = "jdbc:postgresql://" + srcDbIpSlave + ":" + srcDbPortSlave + "/" + srcDbName;
            if (aliasCreateTime != null) {
                sb.append(" and (to_char(").append(aliasCreateTime).append(",'yyyymmdd')").append("='").append(day).append("' and to_char(").append(aliasCreateTime).append(",'yyyymmddhh24')").append("<'").append(dayHour).append("')");
            }
            if (aliasUpdateTime != null) {
                sb.append(" or (to_char(").append(aliasUpdateTime).append(",'yyyymmdd')").append("='").append(day).append("' and to_char(").append(aliasUpdateTime).append(",'yyyymmddhh24')").append("<'").append(dayHour).append("')");
            }
        } else if ("mysql".equalsIgnoreCase(srcDbType)) {
            sourceDriver = "com.mysql.jdbc.Driver";
            sourceUrl = "jdbc:mysql://" + srcDbIpSlave + ":" + srcDbPortSlave + "/" + srcDbName;
            if (aliasCreateTime != null) {
                sb.append(" and (to_char(").append(aliasCreateTime).append(",'yyyyMMdd')").append("='").append(day).append("' and to_char(").append(aliasCreateTime).append(",'yyyyMMddHH')").append("<'").append(dayHour).append("')");
            }
            if (aliasUpdateTime != null) {
                sb.append(" or (to_char(").append(aliasUpdateTime).append(",'yyyyMMdd')").append("='").append(day).append("' and to_char(").append(aliasUpdateTime).append(",'yyyyMMddHH')").append("<'").append(dayHour).append("')");
            }
        } else if ("oracle".equalsIgnoreCase(srcDbType)) {
            sourceDriver = "oracle.jdbc.driver.OracleDriver";
            sourceUrl = "jdbc:oracle:thin:@" + srcDbIpSlave + ":" + srcDbPortSlave + "/" + srcDbName;
            if (aliasCreateTime != null) {
                sb.append(" and (to_char(").append(aliasCreateTime).append(",'yyyyMMdd')").append("='").append(day).append("' and to_char(").append(aliasCreateTime).append(",'yyyyMMddHH')").append("<'").append(dayHour).append("')");
            }
            if (aliasUpdateTime != null) {
                sb.append(" or (to_char(").append(aliasUpdateTime).append(",'yyyyMMdd')").append("='").append(day).append("' and to_char(").append(aliasUpdateTime).append(",'yyyyMMddHH')").append("<'").append(dayHour).append("')");
            }
        }
        log.info(sb.toString());
        try {
            conn = getConn(sourceDriver, sourceUrl, sourceUsername, sourcePassword);
            preparedStatement = conn.prepareStatement(sb.toString());
            rs = preparedStatement.executeQuery();
            while (rs.next()) {
                sCount = rs.getLong("cn");
            }
        } catch (Exception e) {
            send("實時數據集成監控告警:源數據庫連接異常", e.getMessage());
            e.printStackTrace();
        } finally {
            closeConn(rs, preparedStatement, conn);
        }
        log.info("-------------countSource table:" + info.getSourceTableName() + ",sCount=" + sCount + " end---------------");
        return sCount;
    }

    /**
     * @description 統計目標庫數量
     * @author Bob
     * @date 2020/4/28
     */
    public Long countTarget(TableDbConfigInfo info, String day, String dayHour) {
        log.info("-------------countTarget table:" + info.getTargetTableName() + " start---------------");
        Long tCount = 0L;
        Connection conn = null;
        PreparedStatement preparedStatement = null;
        ResultSet rs = null;
        StringBuffer sb = new StringBuffer();
        sb.append("select count(distinct ").append(info.getRowKey()).append(") as cn from ").append(info.getTargetTableName());
        sb.append(" where dt='").append(day).append("'").append(" and (1=1");
        if (info.getAliasCreateTime() != null) {
            sb.append(" and from_unixtime(unix_timestamp(").append(info.getAliasCreateTime()).append(",'yyyyMMddHH'))").append("<'").append(dayHour).append("'");
        }
        if (info.getAliasUpdateTime() != null) {
            sb.append(" or from_unixtime(unix_timestamp(").append(info.getAliasUpdateTime()).append(",'yyyyMMddHH'))").append("<'").append(dayHour).append("'");
        }
        sb.append(")");
        log.info(sb.toString());
        try {
            conn = getConn(targetDriver, targetUrl, targetUsername, targetPassword);
            preparedStatement = conn.prepareStatement(sb.toString());
            rs = preparedStatement.executeQuery();
            while (rs.next()) {
                tCount = rs.getLong("cn");
            }
        } catch (Exception e) {
            send("實時數據集成監控告警:目標數據庫連接異常", e.getMessage());
            e.printStackTrace();
        } finally {
            closeConn(rs, preparedStatement, conn);
        }
        log.info("-------------countTarget table:" + info.getTargetTableName() + ",tCount=" + tCount + " end---------------");
        return tCount;
    }

    /**
     * @description 發送告警
     * @author Bob
     * @date 2020/5/7
     */
    public void send(String title, String message) {
        RestTemplate restTemplate = new RestTemplate();
        String send = "http://aaa.net/message/qy/send/?work_no=" + workNo + "&title=" + title + "&message=" + message;
        try {
            restTemplate.getForObject(send, String.class);
        } catch (RestClientException e) {
            e.printStackTrace();
        }
    }

    /**
     * @description 獲取數據庫連接
     * @author Bob
     * @date 2020/4/28
     */
    public Connection getConn(String driver, String url, String username, String password) {
        Connection conn = null;
        try {
            Class.forName(driver);
            conn = DriverManager.getConnection(url, username, password);
        } catch (Exception e) {
            e.printStackTrace();
        }
        return conn;
    }

    /**
     * @description 關閉數據庫連接
     * @author Bob
     * @date 2020/4/28
     */
    public void closeConn(ResultSet rs, PreparedStatement preparedStatement, Connection conn) {
        if (rs != null) {
            try {
                rs.close();
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
        if (preparedStatement != null) {
            try {
                preparedStatement.close();
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
        if (conn != null) {
            try {
                conn.close();
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    }
}

五、存在問題

1、由於通過hive查詢目標庫太慢，監控表一旦多了，排隊的就會很多，後續考慮用presto；

2、目前是一個小時監控一次，實時性較差；

3、庫、表需手動刷數據配置；

4、開啓監控任務太多的話，可能導致監控系統蹦掉，這個待驗證；

5、一個嚴重問題：源庫數據創建後，隨時可以update，但是目標庫會保存所有的歷史記錄，這樣就會導致目標庫數據會一直大於等於源庫(未合併之前)，待優化。。