[like Sunday like rain]
1.適用場景
適用於正在被動態按行寫入的大文件的讀取和處理。
2.BufferedInputStream類使用原因
之前使用的RandomAccessFile類速度太慢,影響了整個流程的吞吐量。
使用了BufferedInputStream的skip方法,跳過已經讀過的部分。
3.code
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.text.SimpleDateFormat;
import java.util.*;
/**
* Created by maixiaohai on 16/7/4.
*/
public class FisFileTest {
private static int SLEEP_TIME = 1000; // 1 seconds
private static Scanner sc;
private static BufferedInputStream fis;
private static boolean isRun = true;
private static boolean readStop = false;
public static void main(String[] args) throws Exception{
String logPath = args[0];
long breakpoint = 0;
if (args.length == 2) {
breakpoint = Long.parseLong(args[1]);
}
if (!logPath.endsWith("/")) {
System.out.println("path should end with /");
} else {
readFile(logPath, breakpoint);
}
}
public static void readFile(String logPath, long breakpoint) {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
SimpleDateFormat sdf2 = new SimpleDateFormat("yyyyMMdd");
String filePath = logPath + "test.log";
long startIndex = 0;
boolean segmentFlag = true;
boolean initialFlag = true;
Runtime.getRuntime().addShutdownHook(new Thread() {
@Override
public void run() {
isRun = false;
System.out.println("Inside Add Shutdown Hook");
while (!readStop) {
try {
sleep(1 * 1000);
} catch (Exception e) {
e.printStackTrace();
}
}
}
});
while (true) {
fis = null;
sc = null;
try {
File file = new File(filePath);
try {
fis = new BufferedInputStream(new FileInputStream(file));
} catch (Exception e) {
e.printStackTrace();
//切換時,舊文件得命名,新文件還沒有創建,一秒鐘後重試一次
System.out.println(sdf.format(new Date(System.currentTimeMillis()))
+ " 該文件有問題,sleep 1 seconds");
Thread.sleep(1 * 1000);
continue;
}
long fileLength = file.length();
System.out.println("fileLength " + fileLength);
if ( initialFlag && startIndex == 0 ) {
System.out.println("==========first start fileLength :" + fileLength + "=========" );
System.out.println("==========first start breakpoint :" + breakpoint + "=========" );
startIndex = breakpoint;
initialFlag = false;
} else if (startIndex > fileLength){
//0點文件切換時發生,此時需要把昨天的日誌繼續讀完
if (segmentFlag) {
String date = sdf2.format(new Date().getTime()-24*60*60*1000);
String path = logPath + "test_" + date;
File file2 = new File(path);
fis = new BufferedInputStream(new FileInputStream(file2));
segmentFlag = false;
} else {
startIndex = 0;
segmentFlag = true;
}
}
<span style="color:#ff0000;">fis.skip(startIndex)</span>;
sc = new Scanner(fis, "UTF-8").useDelimiter(System.getProperty("line.separator"));
long startTime = System.currentTimeMillis();
String line = null;
int onceCount = 0;
while ( isRun && sc.hasNext() ) {
line = sc.next();
onceCount++;
//記錄讀取位置
<span style="color:#ff0000;">startIndex += line.length() + 1;</span>
// do sth
}
long endTime = System.currentTimeMillis();
System.out.println(sdf.format(new Date(System.currentTimeMillis())) +
" once read " + onceCount + " records");
System.out.println("once read spend " + (endTime - startTime) + "ms");
} catch (Exception e) {
e.printStackTrace();
} finally {
try {
System.out.println("offset : " + startIndex);
fis.close();
// 全部處理完成後,readStop置位,1s後退出
if ( !isRun ) {
readStop = true;
}
Thread.sleep(SLEEP_TIME);
} catch (Exception e) {
e.printStackTrace();
}
}
}
}
}
4.需要注意的地方
本例僅適用於按行寫入的文件