Kafka Stream定位是輕量級的流計算類庫，所有功能放在Lib中實現，實現的程序不依賴單獨執行環境。
一、測試環境搭建
1、下載kafka_2.11-1.1.1.tgz包解壓後修改config目錄下server.properties文件，主要是修改zk地址爲本集羣的例如：zookeeper.connect=xxx.xxx.xxx.xxx:2181
2、啓動kafka，命令如下：bin/zookeeper-server-start.sh config/zookeeper.properties
3、創建topic,命令如下:bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic test
4、查看topic是否創建成功，命令如下bin/kafka-topics.sh --list --zookeeper localhost:2181
5、啓動命令行生產者生產消息，命令如下：bin/kafka-console-producer.sh --broker-list localhost:9092 --topic test
This is a message
This is another message
6、啓動命令行消費者消費消息，命令如下：bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic test --from-beginning
This is a message
This is another message
二、正式開始demo過程，直接貼例子：
public class WordCountProcessorDemo {
private static class MyProcessorSupplier implements ProcessorSupplier<String, String> {
@Override
public Processor<String, String> get() {
return new Processor<String, String>() {
private ProcessorContext context;
private KeyValueStore<String, Integer> kvStore;
@Override
@SuppressWarnings("unchecked")
public void init(final ProcessorContext context) {
this.context = context;
//這裏1000在PunctuationType.STREAM_TIME沒有作用，只有在PunctuationType.WALL_CLOCK_TIME纔有作用
this.context.schedule(1000, PunctuationType.STREAM_TIME, new Punctuator() {
@Override
public void punctuate(long timestamp) {
try (KeyValueIterator<String, Integer> iter = kvStore.all()) {
while (iter.hasNext()) {
KeyValue<String, Integer> entry = iter.next();
//將統計結果sink到下一個topic中
context.forward(entry.key,entry.key+":"+ entry.value.toString());
}
}
}
});
this.kvStore = (KeyValueStore<String, Integer>) context.getStateStore("Counts");
}
@Override
public void process(String dummy, String line) {
String[] words = line.toLowerCase(Locale.getDefault()).split(" ");
for (String word : words) {
Integer oldValue = this.kvStore.get(word);
if (oldValue == null) {
this.kvStore.put(word, 1);
} else {
this.kvStore.put(word, oldValue + 1);
}
}
context.commit();
}
@Override
@Deprecated
public void punctuate(long timestamp) {}
@Override
public void close() {}
};
}
}
public static void main(String[] args) throws Exception {
Properties props = new Properties();
props.put(StreamsConfig.APPLICATION_ID_CONFIG, "streams-wordcount-processor");
props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
props.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, 0);
props.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass());
props.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass());
// setting offset reset to earliest so that we can re-run the demo code with the same pre-loaded data
props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
Topology builder = new Topology();
builder.addSource("Source", "test");
builder.addProcessor("Process", new MyProcessorSupplier(), "Source");
builder.addStateStore(Stores.keyValueStoreBuilder(
//這裏存儲有內存、持久化形式
Stores.inMemoryKeyValueStore("Counts"),
Serdes.String(),
Serdes.Integer()),
"Process");
builder.addProcessor("MyProcessorA", MyProcessorA::new, "Process");
builder.addSink("Sink", "streams-wordcount-processor-output", "Process");
builder.addSink("SINK1", "topicA", "MyProcessorA");
final KafkaStreams streams = new KafkaStreams(builder, props);
//這個門栓厲害了，幫助我們釋放資源
final CountDownLatch latch = new CountDownLatch(1);
// attach shutdown handler to catch control-c
Runtime.getRuntime().addShutdownHook(new Thread("streams-wordcount-shutdown-hook") {
@Override
public void run() {
streams.close();
//釋放完資源後釋放門栓，然程序正常退出
latch.countDown();
}
});
try {
streams.start();
//程序啓動時阻塞這這裏，等接收到control -c 信號後執行jvm鉤子，釋放門栓然後執行退出
latch.await();
} catch (Throwable e) {
System.exit(1);
}
System.exit(0);
}
}
public class MyProcessorA implements Processor<String, String> {
private ProcessorContext context;
@Override
public void init(ProcessorContext processorContext) {
this.context = processorContext;
this.context.schedule(1000);
}
/**
* @param key 消息的key，一般爲null
* @param value 消息的value
*/
@Override
public void process(String key, String value) {
String line = key + " ---- MyProcessor A ---- " + value;
System.out.println(line);
// 將處理完成的數據轉發到downstream processor，比如當前是processor1處理器，通過forward流向到processor2處理器
context.forward(key, line);
}
@Override
public void punctuate(long timestamp) {
}
@Override
public void close() {
}
}
一下是我測試的結果截圖
