1、Zookeeper集羣構建
我們有3個zk實例，分別爲zk-0,zk-1,zk-2;如果你僅僅是測試使用，可以使用1個zk實例.（本示例基於分佈式部署）
1) zk-0
調整配置文件：
clientPort=2181
dataDir=/opt/zookeeper-3.4.6/data
server.0=10.10.73.53:2888:3888
server.1=10.10.73.54:2888:3888
server.2=10.10.73.58:2888:3888
##只需要修改上述配置，其他配置保留默認值
啓動zookeeper
./zkServer.sh start
2) zk-1
調整配置文件(其他配置和zk-0一隻)：
clientPort=2182
##只需要修改上述配置，其他配置保留默認值
啓動zookeeper
./zkServer.sh start
3) zk-2
調整配置文件(其他配置和zk-0一隻)：
clientPort=2183
##只需要修改上述配置，其他配置保留默認值
啓動zookeeper
./zkServer.sh start
2. Kafka集羣構建
因爲Broker配置文件涉及到zookeeper的相關約定，因此我們先展示broker配置文件.我們使用2個kafka broker來構建這個集羣環境，分別爲kafka-0,kafka-1.
1) kafka-0(10.10.73.58)
在config目錄下修改配置文件爲：
broker.id=0
port=9092
num.network.threads=2
num.io.threads=2
socket.send.buffer.bytes=1048576
socket.receive.buffer.bytes=1048576
socket.request.max.bytes=104857600
log.dir=/opt/env/kafka_2.11-0.9.0.1/logs
num.partitions=2
log.flush.interval.messages=10000
log.flush.interval.ms=1000
log.retention.hours=168
#log.retention.bytes=1073741824
log.segment.bytes=536870912
##replication機制,讓每個topic的partitions在kafka-cluster中備份2個
##用來提高cluster的容錯能力..
default.replication.factor=1
log.cleanup.interval.mins=10
zookeeper.connect=10.10.73.53:2181,10.10.73.54:2181,10.10.73.58:2181
zookeeper.connection.timeout.ms=1000000
啓動kafka broker：
> JMS_PORT=9997 bin/kafka-server-start.sh config/server.properties &
因爲zookeeper環境已經正常運行了，我們無需通過kafka來掛載啓動zookeeper.如果你的一臺機器上部署了多個kafka broker，你需要聲明JMS_PORT.
2) kafka-1(10.10.73.53)
broker.id=1
port=9092
##其他配置和kafka-0保持一致
然後和kafka-0一樣執行打包命令，然後啓動此broker.
> JMS_PORT=9998 bin/kafka-server-start.sh config/server.properties &
仍然可以通過如下指令查看topic的"partition"/"replicas"的分佈和存活情況.
Kafka集羣中broker.id=1參數的value不能重複
3、Java端的Producer
kafka-producer.properties配置文件
##metadata.broker.list=127.0.0.1:9092,127.0.0.1:9093
##,127.0.0.1:9093
metadata.broker.list=10.10.73.58:9092,10.10.73.53:9092
producer.type=sync
compression.codec=0
serializer.class=kafka.serializer.StringEncoder
#batch.num.messages=100
Java客戶端程序：
public class KafkaProducerClient {
private Producer<String, String> inner;
private String brokerList;//for metadata discovery,spring setter
private String location = "kafka/kafka-producer.properties";//spring setter
private String defaultTopic;//spring setter
public void setBrokerList(String brokerList) {
this.brokerList = brokerList;
}
public void setLocation(String location) {
this.location = location;
}
public void setDefaultTopic(String defaultTopic) {
this.defaultTopic = defaultTopic;
}
public KafkaProducerClient(){}
public void init() throws Exception {
Properties properties = new Properties();
properties.load(Thread.currentThread().getContextClassLoader().getResourceAsStream(location));
if(brokerList != null) {
properties.put("metadata.broker.list", brokerList);
}
ProducerConfig config = new ProducerConfig(properties);
inner = new Producer<String, String>(config);
}
public void send(String message){
send(defaultTopic,message);
}
public void send(Collection<String> messages){
send(defaultTopic,messages);
}
public void send(String topicName, String message) {
if (topicName == null || message == null) {
return;
}
KeyedMessage<String, String> km = new KeyedMessage<String, String>(topicName,message);
inner.send(km);
}
public void send(String topicName, Collection<String> messages) {
if (topicName == null || messages == null) {
return;
}
if (messages.isEmpty()) {
return;
}
List<KeyedMessage<String, String>> kms = new ArrayList<KeyedMessage<String, String>>();
int i= 0;
for (String entry : messages) {
KeyedMessage<String, String> km = new KeyedMessage<String, String>(topicName,entry);
kms.add(km);
i++;
if(i % 20 == 0){
inner.send(kms);
kms.clear();
}
}
if(!kms.isEmpty()){
inner.send(kms);
}
}
public void close() {
inner.close();
}
/**
* @param args
*/
public static void main(String[] args) {
KafkaProducerClient producer = null;
try {
producer = new KafkaProducerClient();
producer.init();
//producer.setBrokerList("");
int i = 0;
while (true) {
producer.send("test-topic", "this is a sample" + i);
i++;
Thread.sleep(2000);
}
} catch (Exception e) {
e.printStackTrace();
} finally {
if (producer != null) {
producer.close();
}
}
}
}
4、Java端的Consumer
kafka-consumer.properties配置文件
##zookeeper.connect=127.0.0.1:2181,127.0.0.1:2182,127.0.0.1:2183
##,127.0.0.1:2182,127.0.0.1:2183
# timeout in ms for connecting to zookeeper
zookeeper.connect=10.10.73.53:2181,10.10.73.54:2181,10.10.73.58:2181
zookeeper.connectiontimeout.ms=100000
#consumer group id
group.id=test-group
#consumer timeout
#consumer.timeout.ms=5000
Java客戶端程序：
public class KafkaConsumerClient {
private String groupid; //can be setting by spring
private String zkConnect;//can be setting by spring
private String location = "kafka/kafka-consumer.properties";//配置文件位置
private String topic;
private int partitionsNum;
private MessageExecutor executor; //message listener
private ExecutorService threadPool;
private ConsumerConnector connector;
private Charset charset = Charset.forName("utf8");
public void setGroupid(String groupid) {
this.groupid = groupid;
}
public void setZkConnect(String zkConnect) {
this.zkConnect = zkConnect;
}
public void setLocation(String location) {
this.location = location;
}
public void setTopic(String topic) {
this.topic = topic;
}
public void setPartitionsNum(int partitionsNum) {
this.partitionsNum = partitionsNum;
}
public void setExecutor(MessageExecutor executor) {
this.executor = executor;
}
public KafkaConsumerClient() {}
//init consumer,and start connection and listener
public void init() throws Exception {
if(executor == null){
throw new RuntimeException("KafkaConsumer,exectuor cant be null!");
}
Properties properties = new Properties();
properties.load(Thread.currentThread().getContextClassLoader().getResourceAsStream(location));
if(groupid != null){
properties.put("groupid", groupid);
}
if(zkConnect != null){
properties.put("zookeeper.connect", zkConnect);
}
ConsumerConfig config = new ConsumerConfig(properties);
connector = Consumer.createJavaConsumerConnector(config);
Map<String, Integer> topics = new HashMap<String, Integer>();
topics.put(topic, partitionsNum);
Map<String, List<KafkaStream<byte[], byte[]>>> streams = connector.createMessageStreams(topics);
List<KafkaStream<byte[], byte[]>> partitions = streams.get(topic);
threadPool = Executors.newFixedThreadPool(partitionsNum * 2);
//start
for (KafkaStream<byte[], byte[]> partition : partitions) {
threadPool.execute(new MessageRunner(partition));
}
}
public void close() {
try {
threadPool.shutdownNow();
} catch (Exception e) {
//
} finally {
connector.shutdown();
}
}
class MessageRunner implements Runnable {
private KafkaStream<byte[], byte[]> partition;
MessageRunner(KafkaStream<byte[], byte[]> partition) {
this.partition = partition;
}
public void run() {
ConsumerIterator<byte[], byte[]> it = partition.iterator();
while (it.hasNext()) {
// connector.commitOffsets();手動提交offset,當autocommit.enable=false時使用
MessageAndMetadata<byte[], byte[]> item = it.next();
try{
executor.execute(new String(item.message(),charset));// UTF-8,注意異常
}catch(Exception e){
//
}
}
}
public String getContent(Message message){
ByteBuffer buffer = message.payload();
if (buffer.remaining() == 0) {
return null;
}
CharBuffer charBuffer = charset.decode(buffer);
return charBuffer.toString();
}
}
interface MessageExecutor {
public void execute(String message);
}
/**
* @param args
*/
public static void main(String[] args) {
KafkaConsumerClient consumer = null;
try {
MessageExecutor executor = new MessageExecutor() {
public void execute(String message) {
System.out.println(message);
}
};
consumer = new KafkaConsumerClient();
consumer.setTopic("test-topic");
consumer.setPartitionsNum(2);
consumer.setExecutor(executor);
consumer.init();
Thread.sleep(10000);
} catch (Exception e) {
e.printStackTrace();
} finally {
if(consumer != null){
consumer.close();
} } } }
學習參考：
http://www.infoq.com/cn/articles/kafka-analysis-part-1
http://flychao88.iteye.com/category/350737
http://shift-alt-ctrl.iteye.com/blog/1930791
http://www.tuicool.com/articles/mErEZn
