生產者流程圖
本文分析以下流程圖的步驟5。
概述：在原有的ProducerBatch中嘗試追加消息，如果嘗試兩次都失敗，會創建新的ProducerBatch，第三次嘗試追加消息。而追加消息，實際上，是將消息有關屬性寫入到數據流中。對於原有批次已滿或者創建了新的批次的情況，會喚醒Sender線程，做後續處理。
前言
RecordAccumulator主要用來緩存消息，以便Sender線程可以批量發送，進而減少網絡傳輸的資源消耗以提升性能。
該緩存的大小可以通過buffer.memory配置，默認32MB。如果生產者發送消息的速度超過了消息發送到服務器的速度，則容易造成緩存空間不足，此時，KafkaProducer#send()要麼阻塞，要麼拋出異常，取決於max.block.ms的值，默認是60秒。
源碼前序
KafkaProducer#doSend(…)
RecordAccumulator.RecordAppendResult result = accumulator.append(tp, timestamp, serializedKey,
serializedValue, headers, interceptCallback, remainingWaitMs);
if (result.batchIsFull || result.newBatchCreated) {
log.trace("Waking up the sender since topic {} partition {} is either full or getting a new batch", record.topic(), partition);
this.sender.wakeup();
}
通過判斷批次是否已滿、是否有新批次創建來決定是否喚醒Sender線程，進而發送批次消息。
源碼分析
public RecordAppendResult append(TopicPartition tp,
long timestamp,
byte[] key,
byte[] value,
Header[] headers,
Callback callback,
long maxTimeToBlock) throws InterruptedException {
/* 對正在追加消息的線程數加一 */
appendsInProgress.incrementAndGet();
ByteBuffer buffer = null;
if (headers == null) headers = Record.EMPTY_HEADERS;
try {
/* 1. 獲取指定TopicPartition的Deque或者創建新的Deque */
Deque<ProducerBatch> dq = getOrCreateDeque(tp);
synchronized (dq) {
if (closed)
throw new KafkaException("Producer closed while send in progress");
/* 2. 第一次嘗試追加消息到Deque中的ProducerBatch裏面 */
RecordAppendResult appendResult = tryAppend(timestamp, key, value, headers, callback, dq);
if (appendResult != null)
return appendResult;
}
/* RecordBatch.CURRENT_MAGIC_VALUE */
byte maxUsableMagic = apiVersions.maxUsableProduceMagic();
int size = Math.max(this.batchSize, AbstractRecords.estimateSizeInBytesUpperBound(maxUsableMagic, compression, key, value, headers));
log.trace("Allocating a new {} byte message buffer for topic {} partition {}", size, tp.topic(), tp.partition());
/* 3. 由BufferPool調用一個給定大小的ByteBuffer */
buffer = free.allocate(size, maxTimeToBlock);
synchronized (dq) {
// Need to check if producer is closed again after grabbing the dequeue lock.
if (closed)
throw new KafkaException("Producer closed while send in progress");
/* 4. 第二次嘗試追加消息到ProducerBatch中 */
RecordAppendResult appendResult = tryAppend(timestamp, key, value, headers, callback, dq);
if (appendResult != null) {
return appendResult;
}
/* 5. 構建MemoryRecordsBuilder實例 */
MemoryRecordsBuilder recordsBuilder = recordsBuilder(buffer, maxUsableMagic);
/* 6. 構造新的ProducerBatch實例 */
ProducerBatch batch = new ProducerBatch(tp, recordsBuilder, time.milliseconds());
/* 7. 在新的ProducerBatch中嘗試追加消息 */
FutureRecordMetadata future = Utils.notNull(batch.tryAppend(timestamp, key, value, headers, callback, time.milliseconds()));
/* 8. Deque添加該ProducerBatch */
dq.addLast(batch);
/* 9. IncompleteBatches#Set<ProducerBatch> incomplete 添加該ProducerBatch */
/* 還沒有得到確認的請求稱之爲incomplete請求，即未完成的請求 */
incomplete.add(batch);
// Don't deallocate this buffer in the finally block as it's being used in the record batch
buffer = null;
return new RecordAppendResult(future, dq.size() > 1 || batch.isFull(), true);
}
} finally {
if (buffer != null)
/* Return buffers to the pool */
free.deallocate(buffer);
appendsInProgress.decrementAndGet();
}
}
- 獲取或者創建Deque。
private Deque<ProducerBatch> getOrCreateDeque(TopicPartition tp) {
/* CopyOnWriteMap */
Deque<ProducerBatch> d = this.batches.get(tp);
if (d != null)
return d;
d = new ArrayDeque<>();
Deque<ProducerBatch> previous = this.batches.putIfAbsent(tp, d);
if (previous == null)
return d;
else
return previous;
}
- 追加消息到ProducerBatch中。
private RecordAppendResult tryAppend(long timestamp, byte[] key, byte[] value, Header[] headers,
Callback callback, Deque<ProducerBatch> deque) {
ProducerBatch last = deque.peekLast();
if (last != null) {
/* 2.1 追加消息 */
FutureRecordMetadata future = last.tryAppend(timestamp, key, value, headers, callback, time.milliseconds());
if (future == null)
/* 2.2 釋放消息追加佔用的資源，也就是關閉流 */
last.closeForRecordAppends();
else
return new RecordAppendResult(future, deque.size() > 1 || last.isFull(), false);
}
return null;
}
2.1 ProducerBatch#tryAppend(…)
public FutureRecordMetadata tryAppend(long timestamp, byte[] key, byte[] value, Header[] headers, Callback callback, long now) {
/* MemoryRecordsBuilder */
/* 檢查是否有足夠的空間進行添加消息 */
if (!recordsBuilder.hasRoomFor(timestamp, key, value, headers)) {
return null;
} else {
/* 追加消息的核心方法，主要是將相關信息寫入到數據輸出流中 */
Long checksum = this.recordsBuilder.append(timestamp, key, value, headers);
this.maxRecordSize = Math.max(this.maxRecordSize, AbstractRecords.estimateSizeInBytesUpperBound(magic(),
recordsBuilder.compressionType(), key, value, headers));
this.lastAppendTime = now;
FutureRecordMetadata future = new FutureRecordMetadata(this.produceFuture, this.recordCount,
timestamp, checksum,
key == null ? -1 : key.length,
value == null ? -1 : value.length,
Time.SYSTEM);
/* 添加一個Thunk，其封裝了Callback和FutureRecordMetadata */
thunks.add(new Thunk(callback, future));
/* 消息數加一 */
this.recordCount++;
return future;
}
}
2.2 關閉資源相關的流。
public void closeForRecordAppends() {
recordsBuilder.closeForRecordAppends();
}
public void closeForRecordAppends() {
/* appendStream : DataOutputStream類型 */
if (appendStream != CLOSED_STREAM) {
try {
appendStream.close();
} catch (IOException e) {
throw new KafkaException(e);
} finally {
/* 對於write方法直接拋出IllegalStateException */
appendStream = CLOSED_STREAM;
}
}
}
- 調用一個給定大小的buffer。
public ByteBuffer allocate(int size, long maxTimeToBlockMs) throws InterruptedException {
/* 實際上是與buffer.memory屬性進行比較 */
if (size > this.totalMemory)
throw new IllegalArgumentException("。。。");
ByteBuffer buffer = null;
this.lock.lock();
try {
// check if we have a free buffer of the right size pooled
/* free : Deque<ByteBufffer> */
if (size == poolableSize && !this.free.isEmpty())
return this.free.pollFirst();
// now check if the request is immediately satisfiable with the
// memory on hand or if we need to block
int freeListSize = this.free.size() * this.poolableSize;
/* 計算可用的緩衝區大小 */
if (this.nonPooledAvailableMemory + freeListSize >= size) {
// we have enough unallocated or pooled memory to immediately
// satisfy the request, but need to allocate the buffer
freeUp(size);
this.nonPooledAvailableMemory -= size;
} else {
// we are out of memory and will have to block
int accumulated = 0;
Condition moreMemory = this.lock.newCondition();
try {
long remainingTimeToBlockNs = TimeUnit.MILLISECONDS.toNanos(maxTimeToBlockMs);
this.waiters.addLast(moreMemory);
// loop over and over until we have a buffer or have reserved
// enough memory to allocate one
while (accumulated < size) {
long startWaitNs = time.nanoseconds();
long timeNs;
boolean waitingTimeElapsed;
try {
/* 阻塞 */
waitingTimeElapsed = !moreMemory.await(remainingTimeToBlockNs, TimeUnit.NANOSECONDS);
} finally {
long endWaitNs = time.nanoseconds();
timeNs = Math.max(0L, endWaitNs - startWaitNs);
recordWaitTime(timeNs);
}
/* 超時 */
if (waitingTimeElapsed) {
throw new TimeoutException("Failed to allocate memory within the configured max blocking time " + maxTimeToBlockMs + " ms.");
}
remainingTimeToBlockNs -= timeNs;
// check if we can satisfy this request from the free list,
// otherwise allocate memory
if (accumulated == 0 && size == this.poolableSize && !this.free.isEmpty()) {
// just grab a buffer from the free list
buffer = this.free.pollFirst();
/* 已使用的內存大小 */
accumulated = size;
} else {
// we'll need to allocate memory, but we may only get
// part of what we need on this iteration
/* 分配內存大小 */
freeUp(size - accumulated);
int got = (int) Math.min(size - accumulated, this.nonPooledAvailableMemory);
this.nonPooledAvailableMemory -= got;
/* 已使用的內存大小 */
accumulated += got;
}
}
/* 已使用的內存大小重置爲0 */
accumulated = 0;
} finally {
this.nonPooledAvailableMemory += accumulated;
this.waiters.remove(moreMemory);
}
}
} finally {
// signal any additional waiters if there is more memory left
// over for them
try {
if (!(this.nonPooledAvailableMemory == 0 && this.free.isEmpty()) && !this.waiters.isEmpty())
this.waiters.peekFirst().signal();
} finally {
lock.unlock();
}
}
if (buffer == null)
return safeAllocateByteBuffer(size);
else
return buffer;
}
分配內存
private void freeUp(int size) {
while (!this.free.isEmpty() && this.nonPooledAvailableMemory < size)
this.nonPooledAvailableMemory += this.free.pollLast().capacity();
}