主要屬性
private final long totalMemory;//最大緩存空間 ， 由配置文件指定
private final int poolableSize;//每個池的緩存空間大小
private final ReentrantLock lock; //重入鎖
private final Deque<ByteBuffer> free; //空閒的ByteBuffer
private final Deque<Condition> waiters; //等待分配空間的線程
/** Total available memory is the sum of nonPooledAvailableMemory and the number of byte buffers in free * poolableSize. */
private long nonPooledAvailableMemory; //ByteBuffer之外的緩衝區，設計爲了適應突然的大數據量
//構造方法
public BufferPool(long memory, int poolableSize, Metrics metrics, Time time, String metricGrpName) {
this.poolableSize = poolableSize; //指定的 poolableSize
this.lock = new ReentrantLock();//初始化 ReentrantLock 鎖
this.free = new ArrayDeque<>(); //初始化一個 空（empty）的Array隊列，存儲內存
this.waiters = new ArrayDeque<>(); //初始化一個空（empty）的array隊列，存儲等待線程
this.totalMemory = memory;//總的內存
this.nonPooledAvailableMemory = memory;//默認的池外內存，就是總的內存
//下面是一些數據統計，不做分析
this.metrics = metrics;
this.time = time;
this.waitTime = this.metrics.sensor(WAIT_TIME_SENSOR_NAME);
MetricName rateMetricName = metrics.metricName("bufferpool-wait-ratio",
metricGrpName,
"The fraction of time an appender waits for space allocation.");
MetricName totalMetricName = metrics.metricName("bufferpool-wait-time-total",
metricGrpName,
"The total time an appender waits for space allocation.");
this.waitTime.add(new Meter(TimeUnit.NANOSECONDS, rateMetricName, totalMetricName));
}
allocate 方法
org.apache.kafka.clients.producer.internals.BufferPool#allocate
/**
* Allocate a buffer of the given size. This method blocks if there is not enough memory and the buffer pool
* is configured with blocking mode.
分配指定空間的緩存， 如果緩衝區中沒有足夠的空閒空間，那麼會阻塞線程，
直到超時或得到足夠空間
*
* @param size The buffer size to allocate in bytes ，要獲取的指定大小空間
* @param maxTimeToBlockMs The maximum time in milliseconds to block for buffer memory to be available ， 最大等待時長
* @return The buffer
* @throws InterruptedException If the thread is interrupted while blocked
* @throws IllegalArgumentException if size is larger than the total memory controlled by the pool (and hence we would block
* forever)
*/
public ByteBuffer allocate(int size, long maxTimeToBlockMs) throws InterruptedException {
//大於總緩衝區空間，拋出異常
if (size > this.totalMemory)
throw new IllegalArgumentException("Attempt to allocate " + size
+ " bytes, but there is a hard limit of "
+ this.totalMemory
+ " on memory allocations.");
ByteBuffer buffer = null;
//會有線程爭搶，所以需要鎖
this.lock.lock();
try {
// check if we have a free buffer of the right size pooled
// 如果有空間大小正合適的空閒buffer， 走到獲取並返回
if (size == poolableSize && !this.free.isEmpty())
return this.free.pollFirst();
// now check if the request is immediately satisfiable with the
// memory on hand or if we need to block
// 判斷是否有足夠的空閒的內存
int freeListSize = freeSize() * this.poolableSize;
if (this.nonPooledAvailableMemory + freeListSize >= size) {
// we have enough unallocated or pooled memory to immediately
// 有足夠的，未分配的空閒內存
// satisfy the request, but need to allocate the buffer
// 需要整理到一個buffer外空間中，從JVM Heap 中分配內存
freeUp(size); // 循環釋放 空閒的 buffer
this.nonPooledAvailableMemory -= size;
} else {
// we are out of memory and will have to block
// 沒有足夠空閒的 內存或 buffer
int accumulated = 0; //累計已經釋放的內存
//阻塞自己，等待別的線程釋放內存
Condition moreMemory = this.lock.newCondition();
try {
long remainingTimeToBlockNs = TimeUnit.MILLISECONDS.toNanos(maxTimeToBlockMs);
//把自己添加到等待隊列中
this.waiters.addLast(moreMemory);
// loop over and over until we have a buffer or have reserved
// 循環 直到有足夠空閒，或超時
// enough memory to allocate one
while (accumulated < size) { // 已釋放內存 < 要獲取的內存 （釋放的還不夠）
//計時
long startWaitNs = time.nanoseconds();
long timeNs;
boolean waitingTimeElapsed;
try {
waitingTimeElapsed = !moreMemory.await(remainingTimeToBlockNs, TimeUnit.NANOSECONDS);
} finally {
//還沒到最大時長，被喚醒了。更新下已經等待的時長
long endWaitNs = time.nanoseconds();
timeNs = Math.max(0L, endWaitNs - startWaitNs);
recordWaitTime(timeNs);
}
if (waitingTimeElapsed) {
//等待超時了 ， 不等了。拋出異常，結束
throw new TimeoutException("Failed to allocate memory within the configured max blocking time " + maxTimeToBlockMs + " ms.");
}
remainingTimeToBlockNs -= timeNs;
// check if we can satisfy this request from the free list,
// otherwise allocate memory
// 是否有釋放的剛好足夠的空間，否則的話，還得再調整空間
if (accumulated == 0 && size == this.poolableSize && !this.free.isEmpty()) {
// just grab a buffer from the free list
// 有，直接取一個byteBuffer ,返回 ， 結束
buffer = this.free.pollFirst();
accumulated = size;
} else {
// we'll need to allocate memory, but we may only get
// part of what we need on this iteration
// 沒有足夠空閒的，需要調整分配空間 ， 如果分配多了，那麼只需要得到 足夠size的空間
// 例如： 需要 50 ，釋放出來了 80 ，那麼只取 其中的 50 。
freeUp(size - accumulated);
int got = (int) Math.min(size - accumulated, this.nonPooledAvailableMemory);
this.nonPooledAvailableMemory -= got;
accumulated += got;
}
}
// Don't reclaim memory on throwable since nothing was thrown
accumulated = 0;
} finally {
// When this loop was not able to successfully terminate don't loose available memory
// 在循環的過程中，有異常了。 那麼已經釋放出來的空間，再還回去。
this.nonPooledAvailableMemory += accumulated;
//把自己從等待隊列中移除 ， 並結束
this.waiters.remove(moreMemory);
}
}
} finally {
// signal any additional waiters if there is more memory left
// over for them
// 後續處理 ， 這裏不管分配空間是成功還是失敗，都會執行
try {
//三個條件
// this.nonPooledAvailableMemory == 0 && this.free.isEmpty() : 池外內存爲0 ，並且空閒的byteBuffer 沒有了。
// 取反，就是 nonPooledAvailableMemory > 0 || this.free.isNotEmpty() : 池外有內存，或 有空閒的 ByteBuffer
// !this.waiters.isEmpty() ： 等待隊列裏有線程正在等待
if (!(this.nonPooledAvailableMemory == 0 && this.free.isEmpty()) && !this.waiters.isEmpty())
//喚醒隊列里正在等待的線程
this.waiters.peekFirst().signal();
} finally {
// Another finally... otherwise find bugs complains
// 最後的最後，一定得解鎖。否則就是BUG了
lock.unlock();
}
}
//到這裏，說明空間足夠，並且有足夠空閒的了。可以執行真正的分配空間了。
if (buffer == null)
//沒有正好的 buffer，從緩衝區外（JVM Heap）中直接分配內存
return safeAllocateByteBuffer(size);
else
// 有正好的 buffer，返回buffer
return buffer;
}
/**
* Allocate a buffer. If buffer allocation fails (e.g. because of OOM) then return the size count back to
* available memory and signal the next waiter if it exists.
*/
private ByteBuffer safeAllocateByteBuffer(int size) {
boolean error = true;
try {
//分配空間
ByteBuffer buffer = allocateByteBuffer(size);
error = false;
//返回buffer
return buffer;
} finally {
if (error) {
//分配失敗了, 加鎖，操作內存pool
this.lock.lock();
try {
//歸還空間給 池外內存
this.nonPooledAvailableMemory += size;
if (!this.waiters.isEmpty())
//有其他在等待的線程的話，喚醒其他線程
this.waiters.peekFirst().signal();
} finally {
// 加鎖不忘解鎖
this.lock.unlock();
}
}
}
}
// Protected for testing.
protected ByteBuffer allocateByteBuffer(int size) {
// 從JVM Heap 中分配空間，並得到持有空間的ByteBuffer對象
return ByteBuffer.allocate(size);
}
/**
* Attempt to ensure we have at least the requested number of bytes of memory for allocation by deallocating pooled
* buffers (if needed)
*/
private void freeUp(int size) {
while (!this.free.isEmpty() && this.nonPooledAvailableMemory < size)
//循環把 free 裏的 byteBuffer 全撈出來，給 nonPooledAvailableMemory
this.nonPooledAvailableMemory += this.free.pollLast().capacity();
}
deallocate
org.apache.kafka.clients.producer.internals.BufferPool#deallocate(ByteBuffer, int)
/**
* Return buffers to the pool. If they are of the poolable size add them to the free list, otherwise just mark the
* memory as free.
* 歸還 buffer 到 pool 裏，即 buffer放回到 free 隊列中。
* 其他的直接標記爲 空閒內存就可以了
* @param buffer The buffer to return
* @param size The size of the buffer to mark as deallocated, note that this may be smaller than buffer.capacity
* since the buffer may re-allocate itself during in-place compression
*/
public void deallocate(ByteBuffer buffer, int size) {
//照例先加鎖
lock.lock();
try {
if (size == this.poolableSize && size == buffer.capacity()) {
//如果是完整的buffer，放回到隊列裏
buffer.clear();
this.free.add(buffer);
} else {
//不是完整的buffer，標記爲空閒內存就可以了。
this.nonPooledAvailableMemory += size;
}
//如果有內存的線程，喚醒線程
Condition moreMem = this.waiters.peekFirst();
if (moreMem != null)
moreMem.signal();
} finally {
//解鎖
lock.unlock();
}
}
主要邏輯：
- 如果 size == poolableSize , 就放到 free 中
- 如果 size != poolableSize , 歸還到 nonPooledAvailableMemory 中. buffer 對象沒有引用。等待GC釋放
- 有等待線程的話，喚醒線程
free 分析
free 的生產和歸還
free 對象的使用有點繞，在初始化時，是一個空的Array隊列。 allocate() 方法是從 free 中取 buffer 或 釋放 buffer , deallocate() 是歸還 buffer 到 free 中。
- 當 free 空時，從 allocate() 中生產 buffer 對象
- deallocate() 方法將 buffer 放到 free 中
free 爲什麼是雙向隊列
- 獲取 buffer 是從一頭取
- freeUp() 方法釋放 buffer 是從另一頭
理論上 allocate() 方法是單線程訪問。怕是以防萬一吧，一邊獲取一邊釋放。
free的最大化使用
// RecordAccumulator 的 this.batchSize == BufferPool.poolableSize
int size = Math.max(this.batchSize, AbstractRecords.estimateSizeInBytesUpperBound(maxUsableMagic, compression, key, value, headers));
buffer = bufferPool.allocate(size, maxTimeToBlock);
在傳入的參數中，在 size 和 poolableSize 中 , 取最大值。
- <= poolableSize的，可以直接使用一個ByteBuffer。
-
poolableSize 的，就需要開新的內存了。
所以，對於內存來說，poolableSize的大小設置很重要。儘可能的重複利用 緩存 byteBuffer
經驗之談的話，大概取 80% 左右的比例。最大有 100 的數據，那麼poolableSize 設置爲 80 。當然還要具體情況具體分析。
總結
- 共享變量的使用：
- Lock 鎖
- 先進先出（FIFO）
- 隊列
如果文章有幫助到您，請點個贊，您的反饋會讓我感到文章是有價值的