spring-kafka連接Kafka 2.1.1消費者示例

一、maven

<spring.version>4.3.13.RELEASE</spring.version>
<dependency>
    <groupId>org.springframework.kafka</groupId>
    <artifactId>spring-kafka</artifactId>
    <version>1.3.8.RELEASE</version>
</dependency>
<dependency>
    <groupId>org.apache.kafka</groupId>
    <artifactId>kafka-clients</artifactId>
    <version>2.0.1</version>
</dependency>

二、kafka consumer 配置

import org.apache.kafka.clients.producer.ProducerConfig;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.kafka.annotation.EnableKafka;
import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory;
import org.springframework.kafka.config.KafkaListenerContainerFactory;
import org.springframework.kafka.core.ConsumerFactory;
import org.springframework.kafka.core.DefaultKafkaConsumerFactory;
import org.springframework.kafka.listener.ConcurrentMessageListenerContainer;
 
import java.util.HashMap;
import java.util.Map;
 
/**
 * spring 註解配置 - kafka consumer 配置
 */
@Configuration
@EnableKafka
public class SpringKafkaConsumerConfig {
    @Value("${kafka.bootstrap.servers}")
    private String bootstrapServer;
    @Value("${kafka.consumer.group}")
    private String consumerGroup;
    @Value("${kafka.consumer.concurrency:3}")
    private int consumerConcurrency;
    @Value("${kafka.consumer.fetch.max.bytes:1048576}")
    private int fetchMaxBytes;
    @Value("${kafka.consumer.max.poll.records:10000}")
    private int maxPollRecords;
    @Value("${kafka.client.id}")
    private String clientId;
    @Value("${kafka.sasl.jaas.config}")
    private String saslJaasConfig;
 
 
    @Bean
    public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<Integer, String>> kafkaListenerContainerFactory() {
        ConcurrentKafkaListenerContainerFactory<Integer, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
        factory.setConsumerFactory(consumerFactory());
        factory.setConcurrency(consumerConcurrency);
        factory.getContainerProperties().setPollTimeout(30000);
        return factory;
    }
 
    @Bean
    public ConsumerFactory<Integer, String> consumerFactory() {
        return new DefaultKafkaConsumerFactory<>(consumerConfigs());
    }
 
    @Bean
    public Map<String, Object> consumerConfigs() {
        Map<String, Object> props = new HashMap<>();
        props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServer);
        props.put("fetch.max.bytes",fetchMaxBytes);
        props.put("max.poll.records",maxPollRecords);
        props.put("enable.auto.commit", true);
        props.put("group.id", consumerGroup);
        props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        props.put(ProducerConfig.CLIENT_ID_CONFIG, clientId);
        props.put("security.protocol", "SASL_PLAINTEXT");
        props.put("sasl.mechanism", "SCRAM-SHA-256");
        props.put("sasl.jaas.config", saslJaasConfig);
        return props;
    }
}

三、接收kafka消息

import lombok.extern.slf4j.Slf4j;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.kafka.annotation.KafkaListener;
import org.springframework.stereotype.Component;
 
/**
 * 接收kafka消息
 */
@Slf4j
@Component
public class KafkaListener {
 
    @Autowired
    private MessageHandler messageHandler;
 
    /**
     *
     *
     * @param record
     */
    @KafkaListener(topics = "${kafka.topic}")
    public void onMessageDataSync(ConsumerRecord<String, String> record) {
        String value = record.value();
        log.info("接收到record: {}", record);
        try {
            messageHandler.handleData(value);
            log.info("process success, record: {}", record);
        } catch (Exception ex) {
            log.error("process error record: {}", record, ex);
        }
    }
}