某次Kafka擴partition時,消息消費出現異常,需要拉取出問題時間段內的消息重新消費。
``java
private void test() {
Properties props = new Properties();
props.put("bootstrap.servers", "10.11.11.72:9092");
props.put("group.id", "xxx-xxx-service-1");
props.put("enable.auto.commit", "true");
//props.put("auto.commit.interval.ms", "1000");
props.put("session.timeout.ms", "30000");
//props.put("auto.offset.reset", "earliest");
props.put("key.deserializer", StringDeserializer.class.getName());
props.put("value.deserializer", StringDeserializer.class.getName());
KafkaConsumer<String, String> consumer = new KafkaConsumer<>(props);
String topic = "xxx-trade";
Map<TopicPartition, Long> map = Maps.newHashMap();
List<PartitionInfo> partitions = consumer.partitionsFor(topic);
for (PartitionInfo par : partitions) {
map.put(new TopicPartition(topic, par.partition()), beginTime.getTime());
}
Map<TopicPartition, OffsetAndTimestamp> parMap = consumer.offsetsForTimes(map);
for (Map.Entry<TopicPartition, OffsetAndTimestamp> entry : parMap.entrySet()) {
TopicPartition key = entry.getKey();
OffsetAndTimestamp value = entry.getValue();
//System.out.println(key);//topic-partition,eg:testTopic-0
//System.out.println(value);//eg:(timestamp=1584684100465, offset=32382989)
//根據消費裏的timestamp確定offset
if (value != null) {
long offset = value.offset();
consumer.assign(Collections.singletonList(key));
consumer.seek(key, offset);
}
}
//拉取消息
boolean isBreak = false;
while (true) {
ConsumerRecords<String, String> poll = consumer.poll(1000);
FileWriter fw = null;
try {
File file = new File("/data/logs/xxx-trade-service/" + topic + ".txt");
fw = new FileWriter(file, true);
} catch (IOException e) {
e.printStackTrace();
return;
}
StringBuilder stringBuilder = new StringBuilder(20000);
for (ConsumerRecord<String, String> record : poll) {
if (record.timestamp() <= endTime.getTime()) {
stringBuilder.append(record.value()).append("\r\n");
} else if (record.timestamp() > endTime.getTime()) {
isBreak = true;
}
}
try {
fw.write(stringBuilder.toString());
stringBuilder.setLength(0);
fw.flush();
fw.close();
} catch (IOException e) {
e.printStackTrace();
}
if (isBreak) {
break;
}
}
}
``