基於HDP使用Flume採集MySQL中數據傳到Kafka

注意：HDP中Kafka broker的端口是6667，不是9092

---

1.將flume-ng-sql-source-1.5.2.jar，mysql-connector-java.jar放入Flume安裝目錄lib下

下載地址：Flume採集MySQL數據所需jar包

[root@hqc-test-hdp3 kafka]# cd /usr/hdp/2.5.3.0-37/flume/lib/

2.編寫Flume 配置文件mysql_kafka.conf

配置可參考：https://github.com/keedio/flume-ng-sql-source

[root@hqc-test-hdp3 ~]# vim mysql_kafka.conf 
agent.sources = r1
agent.channels = c1
agent.sinks = k1

####define source begin

##define sqlSource
agent.sources.r1.channels = c1
agent.sources.r1.type = org.keedio.flume.source.SQLSource
agent.sources.r1.hibernate.connection.driver_class = com.mysql.jdbc.Driver
agent.sources.r1.hibernate.connection.url = jdbc:mysql://10.35.xx.xx:3306/test
agent.sources.r1.hibernate.connection.user = root
agent.sources.r1.hibernate.connection.password = xxxxx
agent.sources.r1.hibernate.connection.autocommit = true
agent.sources.r1.hibernate.dialect = org.hibernate.dialect.MySQL5Dialect
agent.sources.r1.run.query.delay=5000


agent.sources.r1.table = dajizu
# agent.sources.r1.columns.to.select = *
# agent.sources.r1.start.from = 0

# source狀態寫入路徑（必須存在且可寫入）
agent.sources.r1.status.file.path = /data/flume-log/
agent.sources.r1.status.file.name = sqlsource.status

####define sink begin

##define sink-k1-kafka
agent.sinks.k1.channel = c1
agent.sinks.k1.type = org.apache.flume.sink.kafka.KafkaSink
agent.sinks.k1.topic = test
agent.sinks.k1.brokerList = hqc-test-hdp1:6667,hqc-test-hdp2:6667,hqc-test-hdp3:6667
agent.sinks.k1.requiredAcks = 1
agent.sinks.k1.batchSize = 20
# agent.sinks.k1.serializer.class = Kafka.serializer.StringEncoder
agent.sinks.k1.producer.type = async
# agent.sinks.k1.batchSize = 100


####define channel begin

##define channel-c1-memory
agent.channels.c1.type = memory
agent.channels.c1.capacity = 1000000
agent.channels.c1.transactionCapacity = 100

3.啓動Flume

[root@hqc-test-hdp3 ~]# flume-ng agent --conf conf --conf-file ./mysql_kafka.conf --name agent -Dflume.root.logger=INFO,console 

4.開啓MySQL定時器

SET GLOBAL event_scheduler = 1;  -- 啓動定時器

詳見：MySQL 創建存儲過程 定時器 實現5秒插入一條測試數據

5.使用消費者消費topic裏的消息

[root@hqc-test-hdp1 kafka]# bin/kafka-console-consumer.sh --zookeeper hqc-test-hdp1:2181,hqc-test-hdp2:2181,hqc-test-hdp3:2181 --from-beginning --topic test  
{metadata.broker.list=hqc-test-hdp2:6667,hqc-test-hdp1:6667,hqc-test-hdp3:6667, request.timeout.ms=30000, client.id=console-consumer-65192, security.protocol=PLAINTEXT}
"125","2019-10-21 15:29:18.0","42.4","45.3","-0.03","-0.03","45.8","45.2","8592.7","8592.3","0.0","8592.7","8592.7","8594.0","15.2","49.3","12.7","394.2","3.438","18102.9","16.3","48.9","12.5","1.046","75.6","75.7","76.1","22.0","75.5","12.1","56.4","6.5"
"126","2019-10-21 15:29:23.0","42.4","45.3","-0.03","-0.03","45.8","45.2","8592.7","8592.3","0.0","8592.7","8592.7","8594.0","15.2","49.3","12.7","394.2","3.438","18102.9","16.3","48.9","12.5","1.046","75.6","75.7","76.1","22.0","75.5","12.1","56.4","6.5"
"127","2019-10-21 15:29:28.0","42.4","45.3","-0.03","-0.03","45.8","45.2","8592.7","8592.3","0.0","8592.7","8592.7","8594.0","15.2","49.3","12.7","394.2","3.438","18102.9","16.3","48.9","12.5","1.046","75.6","75.7","76.1","22.0","75.5","12.1","56.4","6.5"