Kafka從指定位置拉取消息

某次Kafka擴partition時,消息消費出現異常,需要拉取出問題時間段內的消息重新消費。
``java

private void test() {
    Properties props = new Properties();
    props.put("bootstrap.servers", "10.11.11.72:9092");
    props.put("group.id", "xxx-xxx-service-1");
    props.put("enable.auto.commit", "true");
    //props.put("auto.commit.interval.ms", "1000");
    props.put("session.timeout.ms", "30000");
    //props.put("auto.offset.reset", "earliest");
    props.put("key.deserializer", StringDeserializer.class.getName());
    props.put("value.deserializer", StringDeserializer.class.getName());
    KafkaConsumer<String, String> consumer = new KafkaConsumer<>(props);

    String topic = "xxx-trade";
    Map<TopicPartition, Long> map = Maps.newHashMap();
    List<PartitionInfo> partitions = consumer.partitionsFor(topic);
    for (PartitionInfo par : partitions) {
        map.put(new TopicPartition(topic, par.partition()), beginTime.getTime());
    }
    Map<TopicPartition, OffsetAndTimestamp> parMap = consumer.offsetsForTimes(map);
    for (Map.Entry<TopicPartition, OffsetAndTimestamp> entry : parMap.entrySet()) {
        TopicPartition key = entry.getKey();
        OffsetAndTimestamp value = entry.getValue();
        //System.out.println(key);//topic-partition,eg:testTopic-0
        //System.out.println(value);//eg:(timestamp=1584684100465, offset=32382989)
        //根據消費裏的timestamp確定offset
        if (value != null) {
            long offset = value.offset();
            consumer.assign(Collections.singletonList(key));
            consumer.seek(key, offset);
        }
    }
    //拉取消息
    boolean isBreak = false;
    while (true) {
        ConsumerRecords<String, String> poll = consumer.poll(1000);
        FileWriter fw = null;
        try {
            File file = new File("/data/logs/xxx-trade-service/" + topic + ".txt");
            fw = new FileWriter(file, true);
        } catch (IOException e) {
            e.printStackTrace();
            return;
        }
        StringBuilder stringBuilder = new StringBuilder(20000);
        for (ConsumerRecord<String, String> record : poll) {
            if (record.timestamp() <= endTime.getTime()) {
                stringBuilder.append(record.value()).append("\r\n");
            } else if (record.timestamp() > endTime.getTime()) {
                isBreak = true;
            }
        }
        try {
            fw.write(stringBuilder.toString());
            stringBuilder.setLength(0);
            fw.flush();
            fw.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
        if (isBreak) {
            break;
        }
    }
}

``

發表評論
所有評論
還沒有人評論,想成為第一個評論的人麼? 請在上方評論欄輸入並且點擊發布.
相關文章