本文來自華爲雲中間件最佳實踐
源地址連接
場景介紹
在DMS提供的原生Kafka SDK中，消費者可以自定義拉取消息的時長，如果需要長時間的拉取消息，只需要把poll(long)方法的參數設置合適的值即可。但是這樣的長連接可能會對客戶端和服務端造成一定的壓力，特別是分區數較多且每個消費者開啓多個線程的情況下。
如圖1所示，Kafka隊列含有多個分區，消費組中有多個消費者同時進行消費，每個線程均爲長連接。當隊列中消息較少或者沒有時，連接不斷開，所有消費者不間斷地拉取消息，這樣造成了一定的資源浪費。
圖1 Kafka消費者多線程消費模式
優化方案
在開了多個線程同時訪問的情況下，如果隊列裏已經沒有消息了，其實不需要所有的線程都在poll，只需要有一個線程poll各分區的消息就足夠了，當在polling的線程發現隊列中有消息，可以喚醒其他線程一起消費消息，以達到快速響應的目的。如圖2所示。
這種方案適用於對消費消息的實時性要求不高的應用場景。如果要求準實時消費消息，則建議保持所有消費者處於活躍狀態。
圖2 優化後的多線程消費方案
消費者（Consumer）和消息分區（Partition）並不強制數量相等，Kafka的poll(long)方法幫助實現獲取消息、分區平衡、消費者與Kafka broker節點間的心跳檢測等功能。 因此在對消費消息的實時性要求不高場景下，當消息數量不多的時候，可以選擇讓一部分消費者處於wait狀態。
代碼示例
-
注意：
以下僅貼出與消費者線程喚醒與睡眠相關代碼，如需運行整個demo，請先下載完整的代碼示例包，同時參考DMS開發指南進行部署和運行。 -
消費消息代碼示例如下：
package com.huawei.dms.kafka;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Iterator;
import java.util.Properties;
import org.apache.kafka.clients.consumer.ConsumerRebalanceListener;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.common.TopicPartition;
import org.apache.log4j.Logger;
public class DmsKafkaConsumeDemo
{
private static Logger logger = Logger.getLogger(DmsKafkaProduceDemo.class);
public static void WorkerFunc(int workerId, KafkaConsumer<String, String> kafkaConsumer) throws IOException
{
Properties consumerConfig = Config.getConsumerConfig();
RecordReceiver receiver = new RecordReceiver(workerId, kafkaConsumer, consumerConfig.getProperty("topic"));
while (true)
{
ConsumerRecords<String, String> records = receiver.receiveMessage();
Iterator<ConsumerRecord<String, String>> iter = records.iterator();
while (iter.hasNext())
{
ConsumerRecord<String, String> cr = iter.next();
System.out.println("Thread" + workerId + " recievedrecords" + cr.value());
logger.info("Thread" + workerId + " recievedrecords" + cr.value());
}
}
}
public static KafkaConsumer<String, String> getConsumer() throws IOException
{
Properties consumerConfig = Config.getConsumerConfig();
consumerConfig.put("ssl.truststore.location", Config.getTrustStorePath());
System.setProperty("java.security.auth.login.config", Config.getSaslConfig());
KafkaConsumer<String, String> kafkaConsumer = new KafkaConsumer<>(consumerConfig);
kafkaConsumer.subscribe(Arrays.asList(consumerConfig.getProperty("topic")),
new ConsumerRebalanceListener()
{
@Override
public void onPartitionsRevoked(Collection<TopicPartition> arg0)
{
}
@Override
public void onPartitionsAssigned(Collection<TopicPartition> tps)
{
}
});
return kafkaConsumer;
}
public static void main(String[] args) throws IOException
{
//創建當前消費組的consumer
final KafkaConsumer<String, String> consumer1 = getConsumer();
Thread thread1 = new Thread(new Runnable()
{
public void run()
{
try
{
WorkerFunc(1, consumer1);
}
catch (IOException e)
{
// TODO Auto-generated catch block
e.printStackTrace();
}
}
});
final KafkaConsumer<String, String> consumer2 = getConsumer();
Thread thread2 = new Thread(new Runnable()
{
public void run()
{
try
{
WorkerFunc(2, consumer2);
}
catch (IOException e)
{
// TODO Auto-generated catch block
e.printStackTrace();
}
}
});
final KafkaConsumer<String, String> consumer3 = getConsumer();
Thread thread3 = new Thread(new Runnable()
{
public void run()
{
try
{
WorkerFunc(3, consumer3);
}
catch (IOException e)
{
// TODO Auto-generated catch block
e.printStackTrace();
}
}
});
//啓動線程
thread1.start();
thread2.start();
thread3.start();
try
{
Thread.sleep(5000);
}
catch (InterruptedException e)
{
e.printStackTrace();
}
//線程加入
try
{
thread1.join();
thread2.join();
thread3.join();
}
catch (InterruptedException e)
{
e.printStackTrace();
}
}
}
- 消費者線程管理示例代碼
示例僅提供簡單的設計思路，開發者可結合實際場景優化線程休眠和喚醒機制。
package com.huawei.dms.kafka;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.log4j.Logger;
public class RecordReceiver
{
private static Logger logger = Logger.getLogger(DmsKafkaProduceDemo.class);
//polling的間隔時間
public static final int WAIT_SECONDS = 10 * 1000;
protected static final Map<String, Object> sLockObjMap = new HashMap<String, Object>();
protected static Map<String, Boolean> sPollingMap = new ConcurrentHashMap<String, Boolean>();
protected Object lockObj;
protected String topicName;
protected KafkaConsumer<String, String> kafkaConsumer;
protected int workerId;
public RecordReceiver(int id, KafkaConsumer<String, String> kafkaConsumer, String queue)
{
this.kafkaConsumer = kafkaConsumer;
this.topicName = queue;
this.workerId = id;
synchronized (sLockObjMap)
{
lockObj = sLockObjMap.get(topicName);
if (lockObj == null)
{
lockObj = new Object();
sLockObjMap.put(topicName, lockObj);
}
}
}
public boolean setPolling()
{
synchronized (lockObj)
{
Boolean ret = sPollingMap.get(topicName);
if (ret == null || !ret)
{
sPollingMap.put(topicName, true);
return true;
}
return false;
}
}
//喚醒全部線程
public void clearPolling()
{
synchronized (lockObj)
{
sPollingMap.put(topicName, false);
lockObj.notifyAll();
System.out.println("Everyone WakeUp and Work!");
logger.info("Everyone WakeUp and Work!");
}
}
public ConsumerRecords<String, String> receiveMessage()
{
boolean polling = false;
while (true)
{
//檢查線程的poll狀態，必要時休眠
synchronized (lockObj)
{
Boolean p = sPollingMap.get(topicName);
if (p != null && p)
{
try
{
System.out.println("Thread" + workerId + " Have a nice sleep!");
logger.info("Thread" + workerId +" Have a nice sleep!");
polling = false;
lockObj.wait();
}
catch (InterruptedException e)
{
System.out.println("MessageReceiver Interrupted! topicName is " + topicName);
logger.error("MessageReceiver Interrupted! topicName is "+topicName);
return null;
}
}
}
//開始消費，必要時喚醒其他線程消費
try
{
ConsumerRecords<String, String> Records = null;
if (!polling)
{
Records = kafkaConsumer.poll(100);
if (Records.count() == 0)
{
polling = true;
continue;
}
}
else
{
if (setPolling())
{
System.out.println("Thread" + workerId + " Polling!");
logger.info("Thread " + workerId + " Polling!");
}
else
{
continue;
}
do
{
System.out.println("Thread" + workerId + " KEEP Poll records!");
logger.info("Thread" + workerId + " KEEP Poll records!");
try
{
Records = kafkaConsumer.poll(WAIT_SECONDS);
}
catch (Exception e)
{
System.out.println("Exception Happened when polling records: " + e);
logger.error("Exception Happened when polling records: " + e);
}
} while (Records.count()==0);
clearPolling();
}
//消息確認
kafkaConsumer.commitSync();
return Records;
}
catch (Exception e)
{
System.out.println("Exception Happened when poll records: " + e);
logger.error("Exception Happened when poll records: " + e);
}
}
}
}
- 說明：
topicName配置爲“隊列名稱”或者“Kafka Topic”。
示例代碼運行結果
[2018-01-25 22:40:51,841] INFO Thread 2 Polling! (com.huawei.dms.kafka.DmsKafkaProduceDemo:119)
[2018-01-25 22:40:51,841] INFO Thread2 KEEP Poll records! (com.huawei.dms.kafka.DmsKafkaProduceDemo:128)
[2018-01-25 22:40:52,122] INFO Everyone WakeUp and Work! (com.huawei.dms.kafka.DmsKafkaProduceDemo:69)
[2018-01-25 22:40:52,169] INFO Thread2 recievedrecordshello, dms kafka. (com.huawei.dms.kafka.DmsKafkaProduceDemo:32)
[2018-01-25 22:40:52,169] INFO Thread2 recievedrecordshello, dms kafka. (com.huawei.dms.kafka.DmsKafkaProduceDemo:32)
[2018-01-25 22:40:52,216] INFO Thread2 recievedrecordshello, dms kafka. (com.huawei.dms.kafka.DmsKafkaProduceDemo:32)
[2018-01-25 22:40:52,325] INFO Thread 2 Polling! (com.huawei.dms.kafka.DmsKafkaProduceDemo:119)
[2018-01-25 22:40:52,325] INFO Thread2 KEEP Poll records! (com.huawei.dms.kafka.DmsKafkaProduceDemo:128)
[2018-01-25 22:40:54,947] INFO Thread1 Have a nice sleep! (com.huawei.dms.kafka.DmsKafkaProduceDemo:87)
[2018-01-25 22:40:54,979] INFO Thread3 Have a nice sleep! (com.huawei.dms.kafka.DmsKafkaProduceDemo:87)
[2018-01-25 22:41:32,347] INFO Thread2 KEEP Poll records! (com.huawei.dms.kafka.DmsKafkaProduceDemo:128)
[2018-01-25 22:41:42,353] INFO Thread2 KEEP Poll records! (com.huawei.dms.kafka.DmsKafkaProduceDemo:128)
[2018-01-25 22:41:47,816] INFO Everyone WakeUp and Work! (com.huawei.dms.kafka.DmsKafkaProduceDemo:69)
[2018-01-25 22:41:47,847] INFO Thread2 recievedrecordshello, dms kafka. (com.huawei.dms.kafka.DmsKafkaProduceDemo:32)
[2018-01-25 22:41:47,925] INFO Thread 3 Polling! (com.huawei.dms.kafka.DmsKafkaProduceDemo:119)
[2018-01-25 22:41:47,925] INFO Thread1 Have a nice sleep! (com.huawei.dms.kafka.DmsKafkaProduceDemo:87)
[2018-01-25 22:41:47,925] INFO Thread3 KEEP Poll records! (com.huawei.dms.kafka.DmsKafkaProduceDemo:128)
[2018-01-25 22:41:47,957] INFO Thread2 Have a nice sleep! (com.huawei.dms.kafka.DmsKafkaProduceDemo:87)
[2018-01-25 22:41:48,472] INFO Everyone WakeUp and Work! (com.huawei.dms.kafka.DmsKafkaProduceDemo:69)
[2018-01-25 22:41:48,503] INFO Thread3 recievedrecordshello, dms kafka. (com.huawei.dms.kafka.DmsKafkaProduceDemo:32)
[2018-01-25 22:41:48,518] INFO Thread1 recievedrecordshello, dms kafka. (com.huawei.dms.kafka.DmsKafkaProduceDemo:32)
[2018-01-25 22:41:48,550] INFO Thread2 recievedrecordshello, dms kafka. (com.huawei.dms.kafka.DmsKafkaProduceDemo:32)
[2018-01-25 22:41:48,597] INFO Thread1 recievedrecordshello, dms kafka. (com.huawei.dms.kafka.DmsKafkaProduceDemo:32)
[2018-01-25 22:41:48,659] INFO Thread 2 Polling! (com.huawei.dms.kafka.DmsKafkaProduceDemo:119)
[2018-01-25 22:41:48,659] INFO Thread2 KEEP Poll records! (com.huawei.dms.kafka.DmsKafkaProduceDemo:128)
[2018-01-25 22:41:48,675] INFO Thread3 recievedrecordshello, dms kafka. (com.huawei.dms.kafka.DmsKafkaProduceDemo:32)
[2018-01-25 22:41:48,675] INFO Everyone WakeUp and Work! (com.huawei.dms.kafka.DmsKafkaProduceDemo:69)
[2018-01-25 22:41:48,706] INFO Thread 1 Polling! (com.huawei.dms.kafka.DmsKafkaProduceDemo:119)
[2018-01-25 22:41:48,706] INFO Thread1 KEEP Poll records! (com.huawei.dms.kafka.DmsKafkaProduceDemo:128)