Kafka異常處理(消費者不消費數據)

問題

生產問題，OffsetMonitor 監控發現運行了一個月的kafka突然間消費有lag.而且消費端不消費數據

分析

在客戶端寫try..catch…捕獲異常： 

2017-08-27 09:47:48,103 ERROR [com.ecar.eoc.message.platform.kafka.Kafka211Context] - [kafka_Exception———->>org.apache.kafka.clients.consumer.CommitFailedException: Commit cannot be completed since the group has already rebalanced and assigned the partitions to another member. This means that the time between subsequent calls to poll() was longer than the configured max.poll.interval.ms

, which typically implies that the poll loop is spending too much time message processing. You can address this either by increasing the session timeout or by reducing the maximum size of batches returned in poll() with max.poll.records.]

解決

暫時沒有找到引起該報錯的根本原因。不過提醒如果客戶端失敗，是需要重連服務器的。修改消費客戶端的代碼，異常重連。

監控KafkaConsumer對象

KafkaConsumer<String, String> c = null;

• 1

異常捕獲後KafkaConsumer置爲null

監控KafkaConsumer對象爲null後重新初始化代碼。 

整個代碼見附件(公司對java代碼加密了，只能用txt文件。下載請轉爲.java)：

package com.ecar.eoc.message.platform.kafka;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;

import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.log4j.Logger;

import com.ecar.commons.cf.common.constant.ClientConstant;
import com.ecar.commons.cf.common.context.CFContext;
import com.ecar.commons.cf.common.context.CfMonitor;
import com.ecar.commons.cf.common.exception.CfException;
import com.ecar.commons.cf.common.vo.Head;
import com.ecar.commons.cf.common.vo.MsgDatagram;
import com.ecar.commons.cf.mq.async.ThreadPoolManager;
import com.ecar.commons.cmf.init.context.SpringConext;
import com.ecar.commons.cmf.util.PropertiesUtil;
import com.ecar.commons.cmf.util.XmlConfigUtil;
import com.ecar.commons.mail.bean.MailSenderInfo;
import com.ecar.commons.mail.utils.SimpleMailSender;
import com.ecar.eoc.message.platform.business.IMqService;
import com.ecar.eoc.message.platform.exception.MessageException;
import com.ecar.eoc.message.platform.utils.CmdIDUtil;

public class Kafka211Context implements CFContext, CfMonitor
{
    private static Logger logger = Logger.getLogger(Kafka211Context.class);

    private static Kafka211Context conext = null;

    private int state = ClientConstant.INT_SCUUCED;

    private static Long time = System.currentTimeMillis(); // 最近的kafka消費時間

    // 監控間隔時間
    private int monitorIntervalTime;

    private Long lastWarnTime;

    KafkaConsumer<String, String> c = null;

    public static Kafka211Context getConext()
    {
        if (conext != null)
        {

            return conext;
        }
        synchronized (Kafka211Context.class)
        {
            if (conext != null)
            {

                return conext;
            }
            conext = new Kafka211Context();

        }

        return conext;
    }

    public static void setConext(Kafka211Context conext)
    {
        Kafka211Context.conext = conext;
    }

    @SuppressWarnings("static-access")
    @Override
    public void init() throws CfException
    {
        this.monitorIntervalTime = XmlConfigUtil.getValueInt("config.mq-config.monitorIntervalTime");
        if (monitorIntervalTime == 0)
        {
            this.monitorIntervalTime = 10000;
        }
        this.conext = this;

    }

    @Override
    public void destroy() throws CfException
    {
        logger.info("Kafka context destroy...");

    }

    @Override
    public void start() throws CfException
    {
        synchronized (this)
        {
            if (null != conext)
            {
                try {
                    new Thread(this).start();
                } catch (Exception e1) {
                    e1.printStackTrace();
                }
                // 啓動一個監控線程，監控kafka取數據程序
                new Thread(new Runnable() {
                    @Override
                    public void run() {
                        lastWarnTime = time;
                        while(true){
                            try {
                                if (System.currentTimeMillis() - time > 1 * 60 * 1000) {
                                    logger.error("kafak error...........................");
                                    int durations = PropertiesUtil.getValueInt(
                                            "kafka_error_email_notice", 10);
                                    if (lastWarnTime == null || System.currentTimeMillis() - lastWarnTime > durations * 60 * 1000){
                                        lastWarnTime = System.currentTimeMillis();
                                        // send message
                                        String receiverMail = PropertiesUtil.getValueString(
                                                "UBI_WEEK_REPORT_RECEIVE_MAIL",
                                                "[email protected],[email protected]");
                                        SimpleMailSender sender = new SimpleMailSender();
                                        MailSenderInfo mailInfo = new MailSenderInfo();
                                        mailInfo.setToAddress(receiverMail);
                                        mailInfo.setContent("ip爲"
                                                + PropertiesUtil.getValueString(
                                                        "kafka_error_server_ip", "未配置")
                                                + "的服務器有" + durations
                                                + "分鐘未從kafka消費到數據，請及時處理!!!");
                                        mailInfo.setSubject("trip服務器報警");
                                        boolean result = sender.sendHtmlMailSSL(mailInfo);
                                        if (result) {
                                            logger.info("sendEmailForWeekReport send mail success!!!");
                                        } else {
                                            logger.info("sendEmailForWeekReport send mail fail!!!");
                                        }
                                    }
                                }
                            } catch (Exception e2) {
                                logger.error("kafka zhou monitor error,cause by ", e2);
                            }

                            try {
                                Thread.sleep(15000l);
                            } catch (InterruptedException e) {
                                e.printStackTrace();
                            }
                        }
                    }
                }).start();

            }

        }
    }

    private void customering()
    {
        try
        {
            // 資源庫
            Properties properties = new Properties();
            properties.put("bootstrap.servers", PropertiesUtil.getValueString("kafka.bootstrap.servers"));
            // 設置不自動提交，自己手動更新offset
            properties.put("enable.auto.commit", "false");
            properties.put("auto.offset.reset", "earliest");
            properties.put("session.timeout.ms", "30000");
            properties.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
            properties.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
            properties.put("group.id", PropertiesUtil.getValueString("kafka.group.id", ""));
            properties.put("auto.commit.interval.ms", "1000");
            // ExecutorService executor = Executors.newFixedThreadPool(200);

            if (Boolean.valueOf(PropertiesUtil.getValueString("kafka.production.environment", "false")))
            {
                // 執行消費

                c = new KafkaConsumer<String, String>(properties);
                c.subscribe(Arrays.asList(PropertiesUtil.getValueString("kafka.topic", "")));
                List<ConsumerRecord<String, String>> buffer = new ArrayList<ConsumerRecord<String, String>>();
                IMqService ser = null;
                // 批量提交數量
                final int minBatchSize = 1;
                ThreadPoolManager poolManager = ThreadPoolManager.getInstance();
                while (true)
                {
                    ConsumerRecords<String, String> records = c.poll(100);

                    for (ConsumerRecord<String, String> record : records){
                        time = System.currentTimeMillis();
                        try
                        {

                            String[] value = record.value().split("@@@");
                            MsgDatagram datagram = com.alibaba.fastjson.JSONObject.parseObject(value[1], MsgDatagram.class);

                            String cmdId = datagram.getCmdId();

                            ser = getMqServiceByCmdID(cmdId);
                            if (ser == null)
                            {
                                throw new MessageException("40002", "CMDID對應的IMqService爲空");
                            }
                            Head head = new Head();
                            head.setAppID(datagram.getAppType());
                            head.setCmdId(datagram.getCmdId());
                            head.setSerialNo("" + datagram.getSerialNo());
                            head.setTerminalId(datagram.getTerminalId());
                            head.setToken(datagram.getToken());
                            Map<String, String> map = new HashMap<String, String>();
                            String topicName = value[0];
                            poolManager.execute(new ConsumerThread(ser, datagram, topicName, head, map));
                        }
                        catch (MessageException e)
                        {
                            logger.error("消費者的名字爲:" + ",消費的消息爲：" + record.value() + e);
                        }
                        catch (Exception e)
                        {
                            logger.error("消費者的名字爲:" + ",消費的消息爲：" + record.value() + e);
                        }

                        buffer.add(record);
                    }
                    if (buffer.size() >= minBatchSize)
                    {
                        // 這裏就是處理成功瞭然後自己手動提交
                        c.commitSync();
                        // LOG.info("提交完畢");
                        buffer.clear();
                    }
                }
            }
        }
        catch (Exception e)
        {
            c=null;
            logger.error("kafka_Exception---------->>" + e);
        }
    }

    @Override
    public void monitor() throws CfException
    {
        if (null == c)
        {
            this.customering();
        }
    }

    @Override
    public void run()
    {

        conext.customering();
        while (true){
            try{
                this.monitor();
                Thread.sleep(monitorIntervalTime);
            }
            catch (Exception e)
            {
                logger.error("Kafka monitor error,", e);

                try
                {
                    Thread.sleep(2000L);
                }
                catch (InterruptedException e1)
                {

                }
            }

        }

    }

    private IMqService getMqServiceByCmdID(String cmdId) throws MessageException
    {
        if (null == cmdId)
        {
            throw new MessageException("40001", "CMDID爲空");
        }

        return (IMqService) SpringConext.getSpring().getBean(CmdIDUtil.getSpringBeanName(cmdId));
    }

    public int getState()
    {
        return state;
    }

    public int getMonitorIntervalTime()
    {
        return monitorIntervalTime;
    }

}