kafka消費者API之自定義存儲offset 到mysql中

pom文件

<dependencies>
        <dependency>
            <groupId>org.apache.kafka</groupId>
            <artifactId>kafka-clients</artifactId>
            <version>0.11.0.0</version>
        </dependency>
        
        <dependency>
            <groupId>mysql</groupId>
            <artifactId>mysql-connector-java</artifactId>
            <version>5.1.47</version>
        </dependency>
    </dependencies>

定義producer

package com.cw.kafka.consumer.mysql;

import org.apache.kafka.clients.producer.*;
import org.apache.kafka.common.serialization.StringSerializer;

import java.math.BigInteger;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Properties;
import java.util.Random;

/**
 * @author 陳小哥cw
 * @date 2020/6/19 19:22
 */
public class KafkaProducerTest {
    static Properties properties = null;
    static KafkaProducer<String, String> producer = null;

    static {
        properties = new Properties();
        // kafka集羣，broker-list
        properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "cm1:9092,cm2:9092,cm3:9092");
        properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
        properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());

        // 重試次數
        properties.put(ProducerConfig.ACKS_CONFIG, "all");
        // 批次大小
        properties.put(ProducerConfig.BATCH_SIZE_CONFIG, 16384);
        // 等待時間
        properties.put(ProducerConfig.LINGER_MS_CONFIG, 100);
        // RecordAccumulator緩衝區大小
        properties.put(ProducerConfig.BUFFER_MEMORY_CONFIG, 33554432);

        producer = new KafkaProducer<String, String>(properties);
    }

    public static void main(String[] args) throws NoSuchAlgorithmException {
        for (int i = 0; i < 100; i++) {
            System.out.println("第" + (i + 1) + "條消息開始發送");
            sendData();
            try {
                Thread.sleep(1000);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }
        producer.close();
    }

    public static String generateHash(String input) throws NoSuchAlgorithmException {
        MessageDigest digest = MessageDigest.getInstance("MD5");
        int random = new Random().nextInt(1000);
        digest.update((input + random).getBytes());
        byte[] bytes = digest.digest();
        BigInteger bi = new BigInteger(1, bytes);
        String string = bi.toString(16);
        return string.substring(0, 3) + input + random;
    }

    public static void sendData() throws NoSuchAlgorithmException {
        String topic = "mysql_store_offset";

        producer.send(new ProducerRecord<String, String>(
                        topic,
                        generateHash(topic),
                        new Random().nextInt(1000) + "\t金鎖家庭財產綜合險(家順險)\t1\t金鎖家庭財產綜合險(家順險)\t213\t自住型家財險\t10\t家財保險\t44\t人保財險\t23:50.0"),
                new Callback() {
                    @Override
                    public void onCompletion(RecordMetadata metadata, Exception exception) {
                        if (exception != null) {
                            System.out.println("|----------------------------\n|topic\tpartition\toffset\n|" + metadata.topic() + "\t" + metadata.partition() + "\t" + metadata.offset() + "\n|----------------------------");
                        } else {
                            exception.printStackTrace();
                        }
                    }
                });
    }
}

創建消費者

package com.cw.kafka.consumer.mysql;

import org.apache.kafka.clients.consumer.*;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.serialization.StringDeserializer;

import java.text.SimpleDateFormat;
import java.util.*;

/**
 * @author 陳小哥cw
 * @date 2020/6/19 20:12
 */
public class KafkaConsumerTest {
    private static Properties properties = null;
    private static String group = "mysql_offset";
    private static String topic = "mysql_store_offset";
    private static KafkaConsumer<String, String> consumer;

    static {
        properties = new Properties();
        // kafka集羣，broker-list
        properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "cm1:9092,cm2:9092,cm3:9092");

        properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
        properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
        // 消費者組，只要group.id相同，就屬於同一個消費者組
        properties.put(ConsumerConfig.GROUP_ID_CONFIG, group);
        // 關閉自動提交offset
        properties.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");

        // 1.創建一個消費者
        consumer = new KafkaConsumer<>(properties);
    }

    public static void main(String[] args) {

        consumer.subscribe(Arrays.asList(topic), new ConsumerRebalanceListener() {

            // rebalance之前將記錄進行保存
            @Override
            public void onPartitionsRevoked(Collection<TopicPartition> partitions) {
                for (TopicPartition partition : partitions) {
                    // 獲取分區
                    int sub_topic_partition_id = partition.partition();
                    // 對應分區的偏移量
                    long sub_topic_partition_offset = consumer.position(partition);
                    String date = new SimpleDateFormat("yyyy年MM月dd日 HH:mm:ss").format(
                            new Date(
                                    new Long(
                                            System.currentTimeMillis()
                                    )
                            )
                    );

                    DBUtils.update("replace into offset values(?,?,?,?,?)",
                            new Offset(
                                    group,
                                    topic,
                                    sub_topic_partition_id,
                                    sub_topic_partition_offset,
                                    date
                            )
                    );
                }
            }

            // rebalance之後讀取之前的消費記錄，繼續消費
            @Override
            public void onPartitionsAssigned(Collection<TopicPartition> partitions) {
                for (TopicPartition partition : partitions) {
                    int sub_topic_partition_id = partition.partition();
                    long offset = DBUtils.queryOffset(
                            "select sub_topic_partition_offset from offset where consumer_group=? and sub_topic=? and sub_topic_partition_id=?",
                            group,
                            topic,
                            sub_topic_partition_id
                    );
                    System.out.println("partition = " + partition + "offset = " + offset);
                    // 定位到最近提交的offset位置繼續消費
                    consumer.seek(partition, offset);

                }
            }
        });

        while (true) {
            ConsumerRecords<String, String> records = consumer.poll(100);
            List<Offset> offsets = new ArrayList<>();
            for (ConsumerRecord<String, String> record : records) {
                String date = new SimpleDateFormat("yyyy年MM月dd日 HH:mm:ss").format(
                        new Date(
                                new Long(
                                        System.currentTimeMillis()
                                )
                        )
                );
                offsets.add(new Offset(group, topic, record.partition(), record.offset(), date));

                System.out.println("|---------------------------------------------------------------\n" +
                        "|group\ttopic\tpartition\toffset\ttimestamp\n" +
                        "|" + group + "\t" + topic + "\t" + record.partition() + "\t" + record.offset() + "\t" + record.timestamp() + "\n" +
                        "|---------------------------------------------------------------"
                );
            }
            for (Offset offset : offsets) {
                DBUtils.update("replace into offset values(?,?,?,?,?)", offset);
            }
            offsets.clear();
        }
    }
}

其他類

數據庫工具類

package com.cw.kafka.consumer.mysql;

import java.io.IOException;
import java.sql.*;
import java.util.Properties;

/**
 * JDBC操作工具類, 提供註冊驅動, 連接, 發送器, 動態綁定參數, 關閉資源等方法
 * jdbc連接參數的提取, 使用Properties進行優化(軟編碼)
 *
 * @author 陳小哥cw
 * @date 2020/6/19 19:41
 */
public class DBUtils {

    private static String driver;
    private static String url;
    private static String user;
    private static String password;

    static {
        // 藉助靜態代碼塊保證配置文件只讀取一次就行
        // 創建Properties對象
        Properties prop = new Properties();
        try {
            // 加載配置文件, 調用load()方法
            // 類加載器加載資源時, 去固定的類路徑下查找資源, 因此, 資源文件必須放到src目錄纔行
            prop.load(DBUtils.class.getClassLoader().getResourceAsStream("db.properties"));
            // 從配置文件中獲取數據爲成員變量賦值
            driver = prop.getProperty("db.driver").trim();
            url = prop.getProperty("db.url").trim();
            user = prop.getProperty("db.user").trim();
            password = prop.getProperty("db.password").trim();
            // 加載驅動
            Class.forName(driver);
        } catch (IOException e) {
            e.printStackTrace();
        } catch (ClassNotFoundException e) {
            e.printStackTrace();
        }
    }

    /**
     * 動態綁定參數
     *
     * @param pstmt
     * @param params
     */
    public static void bindParam(PreparedStatement pstmt, Object... params) {
        try {
            for (int i = 0; i < params.length; i++) {
                pstmt.setObject(i + 1, params[i]);
            }
        } catch (SQLException e) {
            e.printStackTrace();
        }
    }


    /**
     * 預處理髮送器
     *
     * @param conn
     * @param sql
     * @return
     */
    public static PreparedStatement getPstmt(Connection conn, String sql) {
        PreparedStatement pstmt = null;
        try {
            pstmt = conn.prepareStatement(sql);
        } catch (SQLException e) {
            e.printStackTrace();
        }
        return pstmt;
    }

    /**
     * 獲取發送器的方法
     *
     * @param conn
     * @return
     */
    public static Statement getStmt(Connection conn) {
        Statement stmt = null;
        try {
            stmt = conn.createStatement();
        } catch (SQLException e) {
            e.printStackTrace();
        }
        return stmt;
    }

    /**
     * 獲取數據庫連接的方法
     *
     * @return
     */
    public static Connection getConn() {
        Connection conn = null;
        try {
            conn = DriverManager.getConnection(url, user, password);
        } catch (SQLException e) {
            e.printStackTrace();
        }
        return conn;
    }

    /**
     * 獲取特定消費者組，主題，分區下的偏移量
     *
     * @return offset
     */
    public static long queryOffset(String sql, Object... params) {
        Connection conn = getConn();
        long offset = 0;
        PreparedStatement preparedStatement = getPstmt(conn, sql);
        bindParam(preparedStatement, params);

        ResultSet resultSet = null;
        try {
            resultSet = preparedStatement.executeQuery();
            while (resultSet.next()) {
                offset = resultSet.getLong("sub_topic_partition_offset");
            }
        } catch (SQLException e) {
            e.printStackTrace();
        } finally {
            close(resultSet, preparedStatement, conn);
        }

        return offset;
    }

    /**
     * 根據特定消費者組，主題，分區，更新偏移量
     *
     * @param offset
     */
    public static void update(String sql, Offset offset) {
        Connection conn = getConn();
        PreparedStatement preparedStatement = getPstmt(conn, sql);

        bindParam(preparedStatement,
                offset.getConsumer_group(),
                offset.getSub_topic(),
                offset.getSub_topic_partition_id(),
                offset.getSub_topic_partition_offset(),
                offset.getTimestamp()
        );

        try {
            preparedStatement.executeUpdate();
        } catch (SQLException e) {
            e.printStackTrace();
        } finally {
            close(null, preparedStatement, conn);
        }

    }


    /**
     * 統一關閉資源
     *
     * @param rs
     * @param stmt
     * @param conn
     */
    public static void close(ResultSet rs, Statement stmt, Connection conn) {
        try {
            if (rs != null) {
                rs.close();
            }
        } catch (SQLException e) {
            e.printStackTrace();
        }
        try {
            if (stmt != null) {
                stmt.close();
            }
        } catch (SQLException e) {
            e.printStackTrace();
        }
        try {
            if (conn != null) {
                conn.close();
            }
        } catch (SQLException e) {
            e.printStackTrace();
        }
    }

    public static void main(String[] args) {

    }
}

offset實體類

package com.cw.kafka.consumer.mysql;

/**
 * @author 陳小哥cw
 * @date 2020/6/19 20:00
 */
public class Offset {
    private String consumer_group;
    private String sub_topic;
    private Integer sub_topic_partition_id;
    private Long sub_topic_partition_offset;
    private String timestamp;

    public Offset() {
    }

    public Offset(String consumer_group, String sub_topic, Integer sub_topic_partition_id, Long sub_topic_partition_offset, String timestamp) {
        this.consumer_group = consumer_group;
        this.sub_topic = sub_topic;
        this.sub_topic_partition_id = sub_topic_partition_id;
        this.sub_topic_partition_offset = sub_topic_partition_offset;
        this.timestamp = timestamp;
    }

    public String getConsumer_group() {
        return consumer_group;
    }

    public void setConsumer_group(String consumer_group) {
        this.consumer_group = consumer_group;
    }

    public String getSub_topic() {
        return sub_topic;
    }

    public void setSub_topic(String sub_topic) {
        this.sub_topic = sub_topic;
    }

    public Integer getSub_topic_partition_id() {
        return sub_topic_partition_id;
    }

    public void setSub_topic_partition_id(Integer sub_topic_partition_id) {
        this.sub_topic_partition_id = sub_topic_partition_id;
    }

    public Long getSub_topic_partition_offset() {
        return sub_topic_partition_offset;
    }

    public void setSub_topic_partition_offset(Long sub_topic_partition_offset) {
        this.sub_topic_partition_offset = sub_topic_partition_offset;
    }

    public String getTimestamp() {
        return timestamp;
    }

    public void setTimestamp(String timestamp) {
        this.timestamp = timestamp;
    }

    @Override
    public String toString() {
        return "Offset{" +
                "consumer_group='" + consumer_group + '\'' +
                ", sub_topic='" + sub_topic + '\'' +
                ", sub_topic_partition_id=" + sub_topic_partition_id +
                ", sub_topic_partition_offset=" + sub_topic_partition_offset +
                ", timestamp='" + timestamp + '\'' +
                '}';
    }
}

配置文件

# mysql連接相關信息
db.driver=com.mysql.jdbc.Driver
db.user=root
db.password=123456
db.url=jdbc:mysql://192.168.139.101:3306/mydb?useUnicode=true&characterEncoding=UTF8&useSSL=false

數據庫建表語句

CREATE TABLE `offset` (
  `consumer_group` varchar(255) NOT NULL DEFAULT '',
  `sub_topic` varchar(255) NOT NULL DEFAULT '',
  `sub_topic_partition_id` int(11) NOT NULL DEFAULT '0',
  `sub_topic_partition_offset` bigint(20) NOT NULL,
  `timestamp` varchar(255) CHARACTER SET utf8 NOT NULL,
  PRIMARY KEY (`consumer_group`,`sub_topic`,`sub_topic_partition_id`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;

mysql數據截圖

使用replace的話，必須有相應的主鍵作爲限制，不然起不到我們想要的目的

replace：根據三個主鍵，先去查詢是否存在三個主鍵對應值得存在，不存在的話直接insert，存在的話就覆蓋

根據需求，如果需要加入consumer_id的話，那就同樣可以設置爲4號主鍵，動態的數據不能設置爲主鍵，動手嘗試一下就知道其中的奧妙了

==============回收的分區=============
==============重新得到的分區==========
partition = first-2
partition = first-1
partition = first-0

此時在不關閉已開啓的程序的情況下，再啓動一次程序

第一次運行的程序結果

==============回收的分區=============
partition = first-2
partition = first-1
partition = first-0
==============重新得到的分區==========
partition = first-2

第二次運行的程序結果

==============回收的分區=============
==============重新得到的分區==========
partition = first-1
partition = first-0

這是因爲兩次運行的程序的消費者組id都是test，爲同一個消費者組，當第二次運行程序時，對原來的分區進行回收，進行了分區的rebalance重新分配(range分配)。

kafka消費者API之自定義存儲offset 到mysql中

sqoop安裝和命令介紹

卸載虛擬網卡命令

hbase刪除標記和注意事項

Git的實戰操練---分支branch

PostgreSQL語法

https://yachay.unat.edu.pe/blog/index.php?comment_area=format_blog&comment_component=blog&comment_co

linux以太網驅動總結