kafka優雅集成（多線程提升性能）

序
最近項目改造，原先是使用的rabbitMq。此次改造，我們重新調研整體流程，爲結合原已有服務，所以使用的Kafka。
我的集成思路簡單清晰，不過還有提升的地方，直接上菜。
一、消費者配置

import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.kafka.annotation.EnableKafka;
import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory;
import org.springframework.kafka.config.KafkaListenerContainerFactory;
import org.springframework.kafka.core.ConsumerFactory;
import org.springframework.kafka.core.DefaultKafkaConsumerFactory;
import org.springframework.kafka.listener.ConcurrentMessageListenerContainer;
import org.springframework.kafka.listener.ContainerProperties;

import java.util.HashMap;
import java.util.Map;

/**
 * kafka消費者配置
 * @author zhengwen
 */
@EnableKafka
@Configuration
public class KafkaConsumerConfig {

    /**
     * 服務地址
     */
    @Value("${spring.kafka.bootstrap-servers}")
    private String servers;
    /**
     * 分組id
     */
    @Value("${spring.kafka.consumer.group-id}")
    private String groupId;
    /**
     * 自動提交
     */
    @Value("${spring.kafka.consumer.enable-auto-commit}")
    private boolean enableAutoCommit;
    /**
     * 自動提交的間隔時間，默認值是 5000，單位是毫秒
     */
    @Value("${spring.kafka.consumer.auto-commit-interval}")
    private String autoCommitInterval;
    /**
     * 批量消費一次最大拉取的數據量
     */
    @Value("${spring.kafka.consumer.max-poll-records}")
    private int maxPollRecordsConfig;

    /**
     * 消費者配置
     * @return 配置map
     */
    public Map<String, Object> consumerConfigs() {
        Map<String, Object> propsMap = new HashMap<>(10);
        propsMap.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, servers);
        propsMap.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, enableAutoCommit);
        propsMap.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, autoCommitInterval);
        propsMap.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
        propsMap.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
        // 組名
        propsMap.put(ConsumerConfig.GROUP_ID_CONFIG, groupId);
        propsMap.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, maxPollRecordsConfig);
        return propsMap;
    }

    @Bean
    public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {
        ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
        factory.setConsumerFactory(consumerFactory());
        factory.setConcurrency(10);
        factory.getContainerProperties().setPollTimeout(1500);
        factory.setBatchListener(true);
        factory.getContainerProperties().setAckMode(ContainerProperties.AckMode.MANUAL_IMMEDIATE);
        return factory;
    }

    public ConsumerFactory<String, String> consumerFactory() {
        return new DefaultKafkaConsumerFactory<>(consumerConfigs());
    }

}

二、生產者配置

import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.common.serialization.StringSerializer;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.kafka.annotation.EnableKafka;
import org.springframework.kafka.core.DefaultKafkaProducerFactory;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.kafka.core.ProducerFactory;

import java.util.HashMap;
import java.util.Map;

/**
 * kafka生產者配置
 * @author zhengwen
 */
@EnableKafka
@Configuration
public class KafkaProducerConfig {

    /**
     * 服務地址
     */
    @Value("${spring.kafka.bootstrap-servers}")
    private String servers;
    /**
     * 重試失敗的發送次數
     */
    @Value("${spring.kafka.producer.retries}")
    private int retries;
    /**
     * 批量數據大小
     */
    @Value("${spring.kafka.producer.batch-size}")
    private int batchSize;
    /**
     * 緩衝內存大小
     */
    @Value("${spring.kafka.producer.buffer-memory}")
    private int bufferMemory;

    /**
     * 生產者配置
     * @return 配置map
     */
    public Map<String, Object> producerConfigs() {
        Map<String, Object> props = new HashMap<>();
        props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, servers);
        props.put(ProducerConfig.RETRIES_CONFIG, retries);
        props.put(ProducerConfig.BATCH_SIZE_CONFIG, batchSize);
        props.put(ProducerConfig.BUFFER_MEMORY_CONFIG, bufferMemory);
        props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
        props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
        return props;
    }

    @Bean
    public KafkaTemplate<String, String> kafkaTemplate() {
        return new KafkaTemplate(producerFactory());
    }

    public ProducerFactory<String, String> producerFactory() {
        return new DefaultKafkaProducerFactory<>(producerConfigs());
    }
}

三、多線程配置

import lombok.extern.slf4j.Slf4j;
import org.springframework.aop.interceptor.AsyncUncaughtExceptionHandler;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Configuration;
import org.springframework.scheduling.annotation.AsyncConfigurer;
import org.springframework.scheduling.annotation.EnableAsync;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;

import java.lang.reflect.Method;
import java.util.concurrent.Executor;

/**
 * @author zhengwen
 */
@Configuration
@EnableAsync
@Slf4j
public class ThreadConfig implements AsyncConfigurer {

    /**
     * 核心線程樹
     */
    @Value("${thread.config.corePoolSize:5}")
    private Integer corePoolSize;
    /**
     * 最大線程池數量
     */
    @Value("${thread.config.maxPoolSize:10}")
    private Integer maxPoolSize;
    /**
     * 隊列長度
     */
    @Value("${thread.config.queueCapacity:10}")
    private Integer queueCapacity;

    @Override
    public Executor getAsyncExecutor() {
        ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
        log.info("---------[多線程配置初始化]，最大核心線程數:{}，最大線程池數量:{},線程處理隊列長度:{}", corePoolSize, maxPoolSize, queueCapacity);
        //核心線程數
        executor.setCorePoolSize(corePoolSize);
        //最大線程池數量
        executor.setMaxPoolSize(maxPoolSize);
        //線程處理隊列長度
        executor.setQueueCapacity(queueCapacity);
        executor.initialize();
        return executor;
    }

    @Override
    public AsyncUncaughtExceptionHandler getAsyncUncaughtExceptionHandler() {
        log.info("-----------多線程異常handler------------");
        return new SpringAsyncExceptionHandler();
    }

    class SpringAsyncExceptionHandler implements AsyncUncaughtExceptionHandler {
        @Override
        public void handleUncaughtException(Throwable ex, Method method, Object... params) {
            log.error("Asyn返回異常：" + ex.getCause().getMessage() + method.getName());
        }
    }
}

四、監聽方式消費

import lombok.extern.slf4j.Slf4j;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.kafka.annotation.KafkaListener;
import org.springframework.kafka.support.Acknowledgment;
import org.springframework.stereotype.Component;

import java.util.List;

/**
 * 消費者listener
 *
 * @author zhengwen
 **/
@Slf4j
@Component
public class AnalyzeConsumer {

    @Autowired
    private AnalyzeService analyzeService;


    /**
     * 車輛抓拍
     * @param records 消費信息
     * @param ack Ack機制
     */
    @KafkaListener(topics = "${fillersmart.analyze.car.topic.consumer}", containerFactory = "kafkaListenerContainerFactory")
    public void carListen(List<ConsumerRecord> records, Acknowledgment ack) {
        log.info("=====車輛carListen消費者接收信息====");
        try {
            for (ConsumerRecord record : records) {
                log.info("---開啓線程解析車輛數據:{}",record.toString());
                analyzeService.carAnalyze(record);
            }
        } catch (Exception e) {
            e.printStackTrace();
            log.error("----車輛消費者解析數據異常:{}",e.getMessage(),e);
        } finally {
            //手動提交偏移量
            ack.acknowledge();
        }
    }

}

四、生產者
我這裏是消費信息做了處理以後再放回去到另一個topic。如果不需要生產的，可以不要了。

import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.stereotype.Component;
import org.springframework.util.concurrent.FailureCallback;
import org.springframework.util.concurrent.ListenableFuture;
import org.springframework.util.concurrent.SuccessCallback;

/**
 * 解析生產者
 * @author zhengwen
 **/
@Slf4j
@Component
public class AnalyzeProducer {

    @Autowired
    private KafkaTemplate kafkaTemplate;

    /**
     * 發送數據到kafka
     * @param topic topic名稱
     * @param msg 發送信息字符串
     * @param sendCallBack 發送回調
     */
    public void send(String topic, String msg, SendCallBack sendCallBack) {

        ListenableFuture listenableFuture = kafkaTemplate.send(topic,msg);
        //發送成功後回調
        SuccessCallback<String> successCallback = new SuccessCallback() {
            @Override
            public void onSuccess(Object result) {
                sendCallBack.sendSuccessCallBack(topic,msg);
            }
        };
        //發送失敗回調
        FailureCallback failureCallback = new FailureCallback() {
            @Override
            public void onFailure(Throwable ex) {
                sendCallBack.sendFailCallBack(topic,msg,ex);
            }
        };

        listenableFuture.addCallback(successCallback,failureCallback);
    }
}

五、回調方法函數
定義了回調接口，發送的時候傳入。

/**
 * @author zhengwen
 **/
public interface SendCallBack {
    /**
     * 生產成功回調
     * @param topic topic
     * @param msg 信息字符串
     */
    void sendSuccessCallBack(String topic,String msg);

    /**
     * 生產失敗回調
     * @param topic topic
     * @param msg 信息字符串
     * @param ex 異常
     */
    void sendFailCallBack(String topic,String msg,Throwable ex);
}

六、使用方法
就這上面幾步步就ok了，AnalyzeService就是普通的service，注意它的實現類的方法要體現多線程，需要再實現方法上加 @Async標籤，否則無效。多線程配置之前其實我也分享過，我是個怕麻煩的人，都是喜歡簡潔的集成方法。實現方法裏就可以放飛自我了，這裏就看看實現方法怎麼調用的。

//解析完成的數據發送到kafka
        String msg = JSON.toJSONString(collectDataDto);
        analyzeProducer.send(carProducerTopic,msg, new SendCallBack() {
            @Override
            public void sendSuccessCallBack(String topic, String msg) {
                log.info("----車輛抓拍kafka記錄解析完成放入topic：{}，發送成功:{}",topic, msg);
            }

            @Override
            public void sendFailCallBack(String topic, String msg, Throwable ex) {
                log.error("----車輛抓拍kafka記錄解析完成放入topic：{}，發送失敗{}",topic,msg,ex);
            }
        });

實現方法裏處理完成後生產到kafka就是這樣優雅的調用，回調函數裏我這裏只打印了topic，沒有做其他處理，有需要的就再這裏放飛自我。
七、關鍵配置文件與引包
我是springBoot2.3.0 + springCloud：Hoxton.SR4

<dependency>
    <groupId>org.springframework.kafka</groupId>
    <artifactId>spring-kafka</artifactId>
</dependency>
//配置文件
spring:
  kafka:
    bootstrap-servers: 127.0.0.1:9092
    consumer:
      auto-commit-interval: 1000
      enable-auto-commit: false
      group-id: test-consumer-group
      max-poll-records: 10
    producer:
      batch-size: 4096
      buffer-memory: 40960
      retries: 1

其他配置文件都就隨意了。
使用就是這麼優雅，我這個場景同事說更適合使用kafkaStream，下次分享。