Flume增量抽取Oracle數據到Kafka

• 在Flume的lib下引入如下兩個包

flume-ng-sql-source-1.5.2.jar

ojdbc5.jar

• 配置flink文件

# declare source channel sink
agentTest0.channels = channelTest0
agentTest0.sources = sourceTest0
agentTest0.sinks = sinkTest0
###########sql source#################
# declare source type
agentTest0.sources.sourceTest0.type = org.keedio.flume.source.SQLSource

# Hibernate Database connection properties
agentTest0.sources.sourceTest0.hibernate.connection.url = jdbc:oracle:thin:@12.1.93.4:1521/zhgsdb
agentTest0.sources.sourceTest0.hibernate.connection.user = ******
agentTest0.sources.sourceTest0.hibernate.connection.password = ******
agentTest0.sources.sourceTest0.hibernate.connection.autocommit = true
agentTest0.sources.sourceTest0.hibernate.dialect = org.hibernate.dialect.Oracle10gDialect
agentTest0.sources.sourceTest0.hibernate.connection.driver_class = oracle.jdbc.driver.OracleDriver
# 查詢間隔 ms
agentTest0.sources.sourceTest0.run.query.delay= 60000
# 聲明flume保存狀態的文件夾位置
agentTest0.sources.sourceTest0.status.file.path = /opt/third/apache-flume-1.8.0-bin
# 聲明保存flume狀態的文件名稱
agentTest0.sources.sourceTest0.status.file.name = agentTest0.sqlSource.status
# 從指定時間地方開始查詢，只能爲整形
agentTest0.sources.sourceTest0.start.from = 20191115133000
# 自定義查詢語句，增量只能針對主鍵列，而且必須把主鍵查詢出來
agentTest0.sources.sourceTest0.custom.query = select to_char(IN_TIME,'yyyymmddhh24miss'),N_EX_DATE,N_EX_TIME,N_EX_LANE_ID,C_EX_VEHICLE_CLASS,C_CATEGORY,C_CARD_LICENSE,C_EX_LICENSE,VC_TAC from ZHGS.Exit_Jour where IN_TIME>TO_TIMESTAMP(to_char($@$),'yyyymmddhh24miss') order by IN_TIME asc
agentTest0.sources.sourceTest0.batch.size = 60000
agentTest0.sources.sourceTest0.max.rows = 10000
agentTest0.sources.sourceTest0.hibernate.connection.provider_class = org.hibernate.connection.C3P0ConnectionProvider
agentTest0.sources.sourceTest0.hibernate.c3p0.min_size=1
agentTest0.sources.sourceTest0.hibernate.c3p0.max_size=10
##############################
agentTest0.channels.channelTest0.type = memory
agentTest0.channels.channelTest0.capacity = 60000
agentTest0.channels.channelTest0.transactionCapacity = 60000
agentTest0.channels.channelTest0.byteCapacityBufferPercentage = 20
agentTest0.channels.channelTest0.byteCapacity = 16000000
 
agentTest0.sinks.sinkTest0.type = org.apache.flume.sink.kafka.KafkaSink
agentTest0.sinks.sinkTest0.topic = TestTopic
agentTest0.sinks.sinkTest0.brokerList = 12.1.150.111:9092,12.1.150.112:9092,12.1.150.113:9092
agentTest0.sinks.sinkTest0.requiredAcks = 1
agentTest0.sinks.sinkTest0.batchSize = 20

agentTest0.sinks.sinkTest0.channel = channelTest0
agentTest0.sources.sourceTest0.channels=channelTest0

• 執行啓動flume
• 注意事項

# 從指定時間地方開始查詢，只能爲整形
agentDate.sources.sourceTest.start.from = 20191115125000
# 自定義查詢語句，增量只能針對主鍵列，而且必須把主鍵查詢出來
agentDate.sources.sourceTest.custom.query =  select to_char(IN_TIME,'YYYYMMDDHH24MISS'),N_EX_DATE,N_EX_TIME,N_EX_LANE_ID,C_EX_VEHICLE_CLASS,C_CATEGORY,C_CARD_LICENSE,C_EX_LICENSE,VC_TAC from ZHGS.Exit_Jour  where IN_TIME>TO_DATE(to_char($@$),'YYYY-MM-DD HH24:MI:SS') order by IN_TIME asc