/*
* Copyright 2012 The Netty Project
*
* The Netty Project licenses this file to you under the Apache License,
* version 2.0 (the "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at:
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package io.netty.handler.stream;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.ByteBufAllocator;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.FileRegion;
import io.netty.util.internal.ObjectUtil;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
/**
* A {@link ChunkedInput} that fetches data from a file chunk by chunk.
* <p>
* If your operating system supports
* <a href="http://en.wikipedia.org/wiki/Zero-copy">zero-copy file transfer</a>
* such as {@code sendfile()}, you might want to use {@link FileRegion} instead.
*/
public class ChunkedFile implements ChunkedInput<ByteBuf> {
private final RandomAccessFile file;
private final long startOffset;
private final long endOffset;
private final int chunkSize;
private long offset;
/**
* Creates a new instance that fetches data from the specified file.
*/
public ChunkedFile(File file) throws IOException {
this(file, ChunkedStream.DEFAULT_CHUNK_SIZE);
}
/**
* Creates a new instance that fetches data from the specified file.
*
* @param chunkSize the number of bytes to fetch on each
* {@link #readChunk(ChannelHandlerContext)} call
*/
//只讀方式打開文件
public ChunkedFile(File file, int chunkSize) throws IOException {
this(new RandomAccessFile(file, "r"), chunkSize);
}
/**
* Creates a new instance that fetches data from the specified file.
*/
public ChunkedFile(RandomAccessFile file) throws IOException {
this(file, ChunkedStream.DEFAULT_CHUNK_SIZE);
}
/**
* Creates a new instance that fetches data from the specified file.
*
* @param chunkSize the number of bytes to fetch on each
* {@link #readChunk(ChannelHandlerContext)} call
*/
public ChunkedFile(RandomAccessFile file, int chunkSize) throws IOException {
this(file, 0, file.length(), chunkSize);
}
/**
* Creates a new instance that fetches data from the specified file.
*
* @param offset the offset of the file where the transfer begins 文件讀取起始位置偏移量
* @param length the number of bytes to transfer 一共讀取多少字節
* @param chunkSize the number of bytes to fetch on each 每次循環調用readChunk讀取的塊大小
* {@link #readChunk(ChannelHandlerContext)} call
*/
public ChunkedFile(RandomAccessFile file, long offset, long length, int chunkSize) throws IOException {
ObjectUtil.checkNotNull(file, "file");
ObjectUtil.checkPositiveOrZero(offset, "offset");
ObjectUtil.checkPositiveOrZero(length, "length");
ObjectUtil.checkPositive(chunkSize, "chunkSize");
this.file = file; //文件
this.offset = startOffset = offset; //讀取起始位置
this.endOffset = offset + length; //讀取結束位置
this.chunkSize = chunkSize; //每次讀取快大小
//設置file的偏移量
file.seek(offset);
}
/**
* Returns the offset in the file where the transfer began.
*/
public long startOffset() {
return startOffset;
}
/**
* Returns the offset in the file where the transfer will end.
*/
public long endOffset() {
return endOffset;
}
/**
* Returns the offset in the file where the transfer is happening currently.
*/
public long currentOffset() {
return offset;
}
@Override
public boolean isEndOfInput() throws Exception {
//是否讀取完畢
//offset < endOffset = true 說明沒讀到最後
//file.getChannel().isOpen()=true,說明文件打開狀態
//上訴倆種情況則任務沒有讀取完畢，二者出現任何一種情況都認爲讀取完畢
return !(offset < endOffset && file.getChannel().isOpen());
}
//關閉底層資源
@Override
public void close() throws Exception {
file.close();
}
@Deprecated
@Override
public ByteBuf readChunk(ChannelHandlerContext ctx) throws Exception {
//讀取底層文件
return readChunk(ctx.alloc());
}
@Override
public ByteBuf readChunk(ByteBufAllocator allocator) throws Exception {
//如果已經到達結束偏移量，不需要再讀取數據了
long offset = this.offset;
if (offset >= endOffset) {
return null;
}
//看看剩餘的量與chunkSize塊誰更小一些
//如果剩餘的字節非常大，則每次按照chunkSize去讀取，如果剩餘的字節數量不夠一個chunkSize了，就讀實際的數量。
int chunkSize = (int) Math.min(this.chunkSize, endOffset - offset);
// Check if the buffer is backed by an byte array. If so we can optimize it a bit an safe a copy
//分配ByteBuf,注意是堆內存
ByteBuf buf = allocator.heapBuffer(chunkSize);
boolean release = true;
try {
//讀取固定長度chunkSize這個些個字節，如果字節不夠會阻塞，如果返回EOF會拋出異常。
file.readFully(buf.array(), buf.arrayOffset(), chunkSize);
//重新設置ByteBuf的寫索引
buf.writerIndex(chunkSize);
//偏移量增加chunkSize這麼些字節.
this.offset = offset + chunkSize;
release = false;
return buf;
} finally {
if (release) {
buf.release();
}
}
}
//返回長度
@Override
public long length() {
return endOffset - startOffset;
}
//返回處理了多少字節，用當前的offset減去最開始的startOffset
@Override
public long progress() {
return offset - startOffset;
}
}
/*
* Copyright 2012 The Netty Project
*
* The Netty Project licenses this file to you under the Apache License,
* version 2.0 (the "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at:
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package io.netty.handler.stream;
import io.netty.buffer.ByteBufAllocator;
import io.netty.buffer.Unpooled;
import io.netty.channel.Channel;
import io.netty.channel.ChannelDuplexHandler;
import io.netty.channel.ChannelFuture;
import io.netty.channel.ChannelFutureListener;
import io.netty.channel.ChannelHandler;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelPipeline;
import io.netty.channel.ChannelProgressivePromise;
import io.netty.channel.ChannelPromise;
import io.netty.util.ReferenceCountUtil;
import io.netty.util.internal.logging.InternalLogger;
import io.netty.util.internal.logging.InternalLoggerFactory;
import java.nio.channels.ClosedChannelException;
import java.util.ArrayDeque;
import java.util.Queue;
/**
* A {@link ChannelHandler} that adds support for writing a large data stream
* asynchronously neither spending a lot of memory nor getting
* {@link OutOfMemoryError}. Large data streaming such as file
* transfer requires complicated state management in a {@link ChannelHandler}
* implementation. {@link ChunkedWriteHandler} manages such complicated states
* so that you can send a large data stream without difficulties.
* <p>
* To use {@link ChunkedWriteHandler} in your application, you have to insert
* a new {@link ChunkedWriteHandler} instance:
* <pre>
* {@link ChannelPipeline} p = ...;
* p.addLast("streamer", <b>new {@link ChunkedWriteHandler}()</b>);
* p.addLast("handler", new MyHandler());
* </pre>
* Once inserted, you can write a {@link ChunkedInput} so that the
* {@link ChunkedWriteHandler} can pick it up and fetch the content of the
* stream chunk by chunk and write the fetched chunk downstream:
* <pre>
* {@link Channel} ch = ...;
* ch.write(new {@link ChunkedFile}(new File("video.mkv"));
* </pre>
*
* <h3>Sending a stream which generates a chunk intermittently</h3>
*
* Some {@link ChunkedInput} generates a chunk on a certain event or timing.
* Such {@link ChunkedInput} implementation often returns {@code null} on
* {@link ChunkedInput#readChunk(ChannelHandlerContext)}, resulting in the indefinitely suspended
* transfer. To resume the transfer when a new chunk is available, you have to
* call {@link #resumeTransfer()}.
*/
public class ChunkedWriteHandler extends ChannelDuplexHandler {
private static final InternalLogger logger =
InternalLoggerFactory.getInstance(ChunkedWriteHandler.class);
//一個隊列，用來存儲需要輸出的數據
private final Queue<PendingWrite> queue = new ArrayDeque<PendingWrite>();
//當前Context上下文
private volatile ChannelHandlerContext ctx;
public ChunkedWriteHandler() {
}
/**
* @deprecated use {@link #ChunkedWriteHandler()}
*/
@Deprecated
public ChunkedWriteHandler(int maxPendingWrites) {
if (maxPendingWrites <= 0) {
throw new IllegalArgumentException(
"maxPendingWrites: " + maxPendingWrites + " (expected: > 0)");
}
}
@Override
public void handlerAdded(ChannelHandlerContext ctx) throws Exception {
this.ctx = ctx;
}
/**
* Continues to fetch the chunks from the input.
*/
public void resumeTransfer() {
final ChannelHandlerContext ctx = this.ctx;
if (ctx == null) {
return;
}
if (ctx.executor().inEventLoop()) {
resumeTransfer0(ctx);
} else {
// let the transfer resume on the next event loop round
ctx.executor().execute(new Runnable() {
@Override
public void run() {
resumeTransfer0(ctx);
}
});
}
}
private void resumeTransfer0(ChannelHandlerContext ctx) {
try {
doFlush(ctx);
} catch (Exception e) {
logger.warn("Unexpected exception while sending chunks.", e);
}
}
@Override
public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) throws Exception {
//write方法沒有向下傳遞，而是封裝爲PendingWrite對象，並存入隊列。
queue.add(new PendingWrite(msg, promise));
}
@Override
public void flush(ChannelHandlerContext ctx) throws Exception {
//flush方法沒有向下傳遞，根據情況進行輸出
doFlush(ctx);
}
@Override
public void channelInactive(ChannelHandlerContext ctx) throws Exception {
//連接斷開事件中，根據情況進行輸出
doFlush(ctx);
ctx.fireChannelInactive();
}
@Override
public void channelWritabilityChanged(ChannelHandlerContext ctx) throws Exception {
if (ctx.channel().isWritable()) {
// channel is writable again try to continue flushing
//如果當前channel再次切換到可寫狀態，則繼續doFlush
doFlush(ctx);
}
ctx.fireChannelWritabilityChanged();
}
private void discard(Throwable cause) {
for (;;) {
//循環每一個等待輸出的對象
PendingWrite currentWrite = queue.poll();
//如果爲null說明處理完畢
if (currentWrite == null) {
break;
}
//拿到msg對象
Object message = currentWrite.msg;
if (message instanceof ChunkedInput) {
//塊對象處理
ChunkedInput<?> in = (ChunkedInput<?>) message;
boolean endOfInput;
long inputLength;
try {
//判斷是否已經到達結尾
endOfInput = in.isEndOfInput();
//獲取長度
inputLength = in.length();
//關閉資源
closeInput(in);
} catch (Exception e) {
//關閉資源
closeInput(in);
//設置錯誤
currentWrite.fail(e);
if (logger.isWarnEnabled()) {
logger.warn(ChunkedInput.class.getSimpleName() + " failed", e);
}
continue;
}
//如果塊資源還沒輸出乾淨
if (!endOfInput) {
if (cause == null) {
cause = new ClosedChannelException();
}
//設置爲失敗狀態
currentWrite.fail(cause);
} else {
//如果已經輸出完畢，則設置爲成功狀態
currentWrite.success(inputLength);
}
} else {
//普通對象設置promise失敗即可
if (cause == null) {
cause = new ClosedChannelException();
}
currentWrite.fail(cause);
}
}
}
private void doFlush(final ChannelHandlerContext ctx) {
final Channel channel = ctx.channel();
//首先要判斷channel是否連接着
if (!channel.isActive()) {
//處理沒有輸出掉的對象
discard(null);
return;
}
//是否需要調用底層flush
boolean requiresFlush = true;
//內存分配器
ByteBufAllocator allocator = ctx.alloc();
//如果channel還可以寫就繼續循環，如果底層隊列寫滿這裏會返回false
while (channel.isWritable()) {
//拿出隊列頭部但不移除元素
final PendingWrite currentWrite = queue.peek();
//如果爲null則結束循環
if (currentWrite == null) {
break;
}
//如果當前元素已經完成，則從隊列中移除後繼續循環下一個
if (currentWrite.promise.isDone()) {
// This might happen e.g. in the case when a write operation
// failed, but there're still unconsumed chunks left.
// Most chunked input sources would stop generating chunks
// and report end of input, but this doesn't work with any
// source wrapped in HttpChunkedInput.
// Note, that we're not trying to release the message/chunks
// as this had to be done already by someone who resolved the
// promise (using ChunkedInput.close method).
// See https://github.com/netty/netty/issues/8700.
queue.remove();
continue;
}
//消息
final Object pendingMessage = currentWrite.msg;
//消息是塊消息
if (pendingMessage instanceof ChunkedInput) {
final ChunkedInput<?> chunks = (ChunkedInput<?>) pendingMessage;
boolean endOfInput;
boolean suspend;
Object message = null;
try {
//讀取一部分數據內容
message = chunks.readChunk(allocator);
//判斷是否讀取結束
endOfInput = chunks.isEndOfInput();
//如果消息等於null
if (message == null) {
// No need to suspend when reached at the end.
//如果此時消息塊讀取結束則endOfInput=treu，那麼suspend=false，表示不需要暫停
//否則說明消息快還沒結束，但是本次獲取數據暫時沒有。
suspend = !endOfInput;
} else {
//如果suspend不爲空，則不需要暫停
suspend = false;
}
} catch (final Throwable t) {
//異常處理，從隊列中移除
queue.remove();
//釋放msg
if (message != null) {
ReferenceCountUtil.release(message);
}
//關閉資源
closeInput(chunks);
//設置失敗
currentWrite.fail(t);
//結束循環
break;
}
//如果需要暫停，那麼也結束本次循環
if (suspend) {
// ChunkedInput.nextChunk() returned null and it has
// not reached at the end of input. Let's wait until
// more chunks arrive. Nothing to write or notify.
break;
}
//message爲null，給一個空的ByteBuf對象
if (message == null) {
// If message is null write an empty ByteBuf.
// See https://github.com/netty/netty/issues/1671
message = Unpooled.EMPTY_BUFFER;
}
// Flush each chunk to conserve memory
//向底層輸出message
ChannelFuture f = ctx.writeAndFlush(message);
if (endOfInput) {//如果塊數據已經結束
queue.remove(); //移除
//如果f已經輸出完畢
if (f.isDone()) {
handleEndOfInputFuture(f, currentWrite);
} else {
// Register a listener which will close the input once the write is complete.
// This is needed because the Chunk may have some resource bound that can not
// be closed before its not written.
//
// See https://github.com/netty/netty/issues/303
//如果沒有立即完成，則註冊回調事件
f.addListener(new ChannelFutureListener() {
@Override
public void operationComplete(ChannelFuture future) {
handleEndOfInputFuture(future, currentWrite);
}
});
}
} else { //如果塊數據沒寫完
//是否需要重新開始
final boolean resume = !channel.isWritable();
if (f.isDone()) { //上一個塊底層socket寫入成功
handleFuture(f, currentWrite, resume);
} else {
//沒有立即返回結果，註冊事件
f.addListener(new ChannelFutureListener() {
@Override
public void operationComplete(ChannelFuture future) {
handleFuture(future, currentWrite, resume);
}
});
}
}
requiresFlush = false;
} else {
//不是塊消息，則從隊列中移除
queue.remove();
//向下傳遞後把requiresFlush設置爲true
ctx.write(pendingMessage, currentWrite.promise);
requiresFlush = true;
}
//如果channel關閉了，則處理關閉邏輯，釋放資源。
if (!channel.isActive()) {
discard(new ClosedChannelException());
break;
}
}
//如果上面沒有循環，這裏需要向底層傳遞flush方法。
if (requiresFlush) {
ctx.flush();
}
}
private static void handleEndOfInputFuture(ChannelFuture future, PendingWrite currentWrite) {
ChunkedInput<?> input = (ChunkedInput<?>) currentWrite.msg;
if (!future.isSuccess()) { //底層socket輸出失敗
closeInput(input); //關閉資源
currentWrite.fail(future.cause()); //設置失敗
} else {
// read state of the input in local variables before closing it
long inputProgress = input.progress();
long inputLength = input.length();
closeInput(input); //關閉資源
currentWrite.progress(inputProgress, inputLength); //設置進度
currentWrite.success(inputLength); //設置成功
}
}
private void handleFuture(ChannelFuture future, PendingWrite currentWrite, boolean resume) {
ChunkedInput<?> input = (ChunkedInput<?>) currentWrite.msg;
if (!future.isSuccess()) { //寫入失敗
closeInput(input); //關閉資源
currentWrite.fail(future.cause());//設置失敗
} else {
//更新進度
currentWrite.progress(input.progress(), input.length());
//如果resume=false，說明當時channel還是可以寫的，則頂層循環會繼續
//不需要在這裏觸發繼續寫的邏輯
//否則此時再判斷一下isWritable，如果可以寫，則觸發繼續寫的邏輯
if (resume && future.channel().isWritable()) {
resumeTransfer();
}
}
}
private static void closeInput(ChunkedInput<?> chunks) {
try {
chunks.close();
} catch (Throwable t) {
if (logger.isWarnEnabled()) {
logger.warn("Failed to close a chunked input.", t);
}
}
}
//封裝原始消息
private static final class PendingWrite {
final Object msg; //等待輸出的消息對象
final ChannelPromise promise; //任務句柄
PendingWrite(Object msg, ChannelPromise promise) {
this.msg = msg;
this.promise = promise;
}
//設置失敗
void fail(Throwable cause) {
ReferenceCountUtil.release(msg);
promise.tryFailure(cause);
}
//設置成功
void success(long total) {
if (promise.isDone()) {
// No need to notify the progress or fulfill the promise because it's done already.
return;
}
//更新進度
progress(total, total);
promise.trySuccess();
}
//更新進度
void progress(long progress, long total) {
if (promise instanceof ChannelProgressivePromise) {
((ChannelProgressivePromise) promise).tryProgress(progress, total);
}
}
}
}
