kafka與Spring的集成配置生產者:前提kafka安裝完成,及創建好主題
pom文件配置:
<!-- https://mvnrepository.com/artifact/org.apache.kafka/kafka -->
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka_2.12</artifactId>
<version>2.1.0</version>
<exclusions>
<exclusion>
<artifactId>jmxri</artifactId>
<groupId>com.sun.jmx</groupId>
</exclusion>
<exclusion>
<artifactId>jms</artifactId>
<groupId>javax.jms</groupId>
</exclusion>
<exclusion>
<artifactId>jmxtools</artifactId>
<groupId>com.sun.jdmk</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
<version>2.1.0</version>
</dependency>
<dependency>
<groupId>org.springframework.kafka</groupId>
<artifactId>spring-kafka</artifactId>
<version>2.1.11.RELEASE</version><!--2.1.11.RELEASE 2.2以上版本不支持 指定具體監聽類-->
</dependency>
config.properties:
#kafka
#kafka訪問地址
kafka.serverHost=192.168.3.117:9092
#kafka主題名稱
kafka.topic=goods
spring-kafkaProducer.xml:
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:context="http://www.springframework.org/schema/context"
xsi:schemaLocation="http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/context
http://www.springframework.org/schema/context/spring-context.xsd">
<!--基本配置 -->
<bean id="producerProperties" class="java.util.HashMap">
<constructor-arg>
<map>
<!-- kafka服務地址,可能是集羣-->
<entry key="bootstrap.servers" value="${kafka.serverHost}" />
<entry key="group.id" value="0"/>
<!-- 有可能導致broker接收到重複的消息,默認值爲3-->
<entry key="retries" value="10" />
<!-- 每次批量發送消息的數量-->
<entry key="batch.size" value="1638" />
<!-- 默認0ms,在異步IO線程被觸發後(任何一個topic,partition滿都可以觸發)-->
<entry key="linger.ms" value="1" />
<!--producer可以用來緩存數據的內存大小。如果數據產生速度大於向broker發送的速度,producer會阻塞或者拋出異常 -->
<entry key="buffer.memory" value="33554432 " />
<!-- producer需要server接收到數據之後發出的確認接收的信號,此項配置就是指procuder需要多少個這樣的確認信號-->
<entry key="acks" value="all" />
<entry key="key.serializer" value="org.apache.kafka.common.serialization.StringSerializer" />
<entry key="value.serializer" value="org.apache.kafka.common.serialization.StringSerializer" />
</map>
</constructor-arg>
</bean>
<!-- 創建kafkatemplate需要使用的producerfactory bean -->
<bean id="producerFactory"
class="org.springframework.kafka.core.DefaultKafkaProducerFactory">
<constructor-arg>
<ref bean="producerProperties" />
</constructor-arg>
</bean>
<!-- 創建kafkatemplate bean,使用的時候,只需要注入這個bean,即可使用template的send消息方法 -->
<bean id="KafkaTemplate" class="org.springframework.kafka.core.KafkaTemplate">
<constructor-arg ref="producerFactory" />
<constructor-arg name="autoFlush" value="true" />
<!--設置對應topic-->
<property name="defaultTopic" value="${kafka.topic}" />
</bean>
<bean id="producerListener" class="com.hanshow.wise.base.goods.servlet.KafkaProducerListener" />
</beans>
spring.xml:
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:context="http://www.springframework.org/schema/context"
xsi:schemaLocation="http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans-4.0.xsd
http://www.springframework.org/schema/context
http://www.springframework.org/schema/context/spring-context-4.0.xsd">
<!-- 引入config.properties屬性文件 -->
<context:property-placeholder location="classpath:config.properties"/>
<!-- 自動掃描(自動注入),掃描這個包以及它的子包的所有使用@Service註解標註的類 -->
<context:component-scan base-package="com.hanshow.wise.base.*.service" />
<context:component-scan base-package="com.hanshow.wise.base.goods.servlet"/>
<import resource="classpath:spring-kafkaProducer.xml" />
</beans>
KafkaProducerListener監聽:
package com.hanshow.wise.base.goods.servlet;
import org.apache.kafka.clients.producer.RecordMetadata;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.kafka.support.ProducerListener;
/**
* kafkaProducer監聽器,在producer配置文件中開啓
* @author
*
*/
@SuppressWarnings("rawtypes")
public class KafkaProducerListener implements ProducerListener{
protected final Logger LOG = LoggerFactory.getLogger("kafkaProducer");
/**
* 發送消息成功後調用
*/
public void onSuccess(String topic, Integer partition, Object key,
Object value, RecordMetadata recordMetadata) {
LOG.info("==========kafka發送數據成功(日誌開始)==========");
LOG.info("----------topic:"+topic);
LOG.info("----------partition:"+partition);
LOG.info("----------key:"+key);
LOG.info("----------value:"+value);
LOG.info("----------RecordMetadata:"+recordMetadata);
LOG.info("~~~~~~~~~~kafka發送數據成功(日誌結束)~~~~~~~~~~");
}
/**
* 發送消息錯誤後調用
*/
public void onError(String topic, Integer partition, Object key,
Object value, Exception exception) {
LOG.info("==========kafka發送數據錯誤(日誌開始)==========");
LOG.info("----------topic:"+topic);
LOG.info("----------partition:"+partition);
LOG.info("----------key:"+key);
LOG.info("----------value:"+value);
LOG.info("----------Exception:"+exception);
LOG.info("~~~~~~~~~~kafka發送數據錯誤(日誌結束)~~~~~~~~~~");
exception.printStackTrace();
}
/**
* 方法返回值代表是否啓動kafkaProducer監聽器
*/
public boolean isInterestedInSuccess() {
LOG.info("///kafkaProducer監聽器啓動///");
return true;
}
}
KafkaProducerService接口:
package com.hanshow.wise.base.goods.service;
import java.util.Map;
import com.hanshow.wise.base.goods.model.dto.StoreGoodsDTO;
import com.hanshow.wise.common.jo.BaseDTO;
/**
* kafka接口
*
* @author
* @date 2018年6月2日
* @since 1.0.0
*/
public interface KafkaProducerService {
BaseDTO<Map<String, Object>> getProducer(StoreGoodsDTO storeGoodsDTO);
}
KafkaProducerServiceImpl實現:
package com.hanshow.wise.base.goods.service.impl;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.stereotype.Service;
import com.hanshow.wise.base.goods.model.dto.StoreGoodsDTO;
import com.hanshow.wise.base.goods.service.KafkaProducerService;
import com.hanshow.wise.base.goods.util.ConfigUtils;
import com.hanshow.wise.common.jo.BaseDTO;
/**
* 提供商品分類信息查詢支持Service實現
*
* @author
* @date 2018年6月2日
* @since 1.0.0
*/
@Service
public class KafkaProducerServiceImpl implements KafkaProducerService {
Logger logger = LoggerFactory.getLogger(KafkaProducerServiceImpl.class);
@Autowired
private KafkaTemplate<String, String> kafkaTemplate;
@Override
public void getProducer() {
kafkaTemplate.send(ConfigUtils.getType("kafka.topic"), "內容");
}
}
測試調用結果:
以上是kafka與Spring的集成內容。
附加簡單測試生產者與消費者,無需spring配置。前提kafka安裝完成,及創建好主題。
ProducerDemo.java,生產者:
package com.hanshow.wise.base.goods;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.Producer;
import org.apache.kafka.clients.producer.ProducerRecord;
import java.util.Properties;
public class ProducerDemo {
public static String serverHost="192.168.3.117:9092";
public static void main(String[] args) {
Properties props = new Properties();
props.put("bootstrap.servers", serverHost);
//The "all" setting we have specified will result in blocking on the full commit of the record, the slowest but most durable setting.
//“所有”設置將導致記錄的完整提交阻塞,最慢的,但最持久的設置。
props.put("acks", "all");
//如果請求失敗,生產者也會自動重試,即使設置成0 the producer can automatically retry.
props.put("retries", 0);
//The producer maintains buffers of unsent records for each partition.
props.put("batch.size", 16384);
//默認立即發送,這裏這是延時毫秒數
props.put("linger.ms", 1);
//生產者緩衝大小,當緩衝區耗盡後,額外的發送調用將被阻塞。時間超過max.block.ms將拋出TimeoutException
props.put("buffer.memory", 33554432);
//The key.serializer and value.serializer instruct how to turn the key and value objects the user provides with their ProducerRecord into bytes.
props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
//創建kafka的生產者類
Producer<String, String> producer = new KafkaProducer<String, String>(props);
for (int i = 0; i < 10; i++) {
// 這裏平均寫入4個分區
// producer.send(new ProducerRecord<String, String>("test", i % 4, Integer.toString(i), Integer.toString(i)));
// 因爲沒有建立集羣所以只能是0個分區
producer.send(new ProducerRecord<String, String>("my-first-topic", 0, Integer.toString(i), Integer.toString(i)));
}
producer.close();
}
}
ConsumerDemo.java消費者
package com.hanshow.wise.base.goods;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import java.util.Arrays;
import java.util.Properties;
public class ConsumerDemo {
public static void main(String[] args) {
Properties props = new Properties();
props.put("bootstrap.servers", ProducerDemo.serverHost);
System.out.println("this is the group part test 1");
//消費者的組id
props.put("group.id", "GroupA");//這裏是GroupA或者GroupB
props.put("enable.auto.commit", "true");
props.put("auto.commit.interval.ms", "1000");
//從poll(拉)的回話處理時長
props.put("session.timeout.ms", "30000");
//poll的數量限制
//props.put("max.poll.records", "100");
props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
KafkaConsumer<String, String> consumer = new KafkaConsumer<String, String>(props);
//訂閱主題列表topic
consumer.subscribe(Arrays.asList("my-first-topic"));
while (true) {
ConsumerRecords<String, String> records = consumer.poll(100);
for (ConsumerRecord<String, String> record : records)
// 正常這裏應該使用線程池處理,不應該在這裏處理
System.out.printf("offset = %d, key = %s, value = %s", record.offset(), record.key(), record.value() + "\n");
}
}
}