■1.問題現象
ftp批量下載文件時,偶爾出現空文件。
■2.問題原因
A:代碼環境
整體環境:springboot2.0.4
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.0.4.RELEASE</version>
<relativePath/> <!-- lookup parent from repository -->
</parent>
ftp環境:org.apache.commons.net.ftp
<dependency>
<groupId>commons-net</groupId>
<artifactId>commons-net</artifactId>
<version>3.3</version>
</dependency>
/**
* 下載指定目錄下指定後綴名文件
*
* @param hostname FTP服務器地址
* @param port FTP服務器端口號
* @param username FTP登錄帳號
* @param password FTP登錄密碼
* @param pathname FTP服務器文件目錄
* @param localpath 下載後的文件路徑
* @return
*/
public static List<String> downloadFileForSubfix(String hostname, int port, String username, String password, String pathname,
int spanId, String localpath, String[] subfixs, Integer connectId) {
List<String> fileNames = new ArrayList<>();
List<FTPFile> fTPFiles = new ArrayList<>();
// org.apache.commons.net.ftp
FTPClient ftp = null;
try {
ftp = getFTPClient(hostname, username, password, port);
// 主動模式
ftp.enterLocalActiveMode();
// 設置文件類型 二進制文件
ftp.setFileType(FTPClient.BINARY_FILE_TYPE);
// 切換目錄
boolean flag = ftp.changeWorkingDirectory(new String(pathname.getBytes(), FTP.DEFAULT_CONTROL_ENCODING));
if (!flag) {
throw new Exception("目錄[" + pathname + "]不存在");
}
FTPFile[] fs = ftp.listFiles(pathname, file -> {
String name = file.getName();
String strSubName = name.substring(name.lastIndexOf("."));
for (String subfix : subfixs) {
if (subfix.equalsIgnoreCase(strSubName)) {
return true;
}
}
return false;
});
for (FTPFile ftpFile : fs) {
if(ftpFile.isFile()){
fTPFiles.add(ftpFile);
}
}
//文件排序--時間早的文件優先處理
Collections.sort(fTPFiles, (file, newFile) -> {
if (file.getTimestamp().getTimeInMillis() < newFile.getTimestamp().getTimeInMillis()) {
return -1;
} else if (file.getTimestamp().getTimeInMillis() == newFile.getTimestamp().getTimeInMillis()) {
return 0;
} else {
return 1;
}
});
for (int i = 0; i < fTPFiles.size(); i++) {
if (i >= DiConstants.FTP_FILE_COUNT_20s) {
break;
}
FTPFile ff = fTPFiles.get(i);
String strNewName = writeFile(ff, localpath, ftp, connectId);
//deleteFile(hostname, port, username, password, pathname, strNewName);
fileNames.add(strNewName);
}
} catch (Exception e) {
logger.error("FTP下載文件異常{}", e);
} finally {
try {
if (ftp != null) {
ftp.logout();
}
} catch (IOException e1) {
e1.printStackTrace();
}
if (ftp.isConnected()) {
try {
ftp.disconnect();
} catch (IOException e) {
e.printStackTrace();
}
}
}
return fileNames;
}
public static String writeFile(FTPFile ff, String localpath, FTPClient ftp, Integer connectId) {
FileOutputStream out = null;
InputStream in = null;
String outFileName = ff.getName();
String strConnect = "CONNECT";
String strNewName = strConnect.concat(DiConstants.MARK_LEFT_BRACKET).concat(String.format("%03d", connectId))
.concat(DiConstants.MARK_RIGHT_BRACKET).concat(outFileName);
try {
File localFile = new File(localpath.concat(File.separator).concat(strNewName));
out = new FileOutputStream(localFile);
//TODO 默認爲ASCII文件編碼格式但是需要改成二進制
ftp.setFileType(FTP.BINARY_FILE_TYPE);
in = ftp.retrieveFileStream(outFileName);
byte[] byteArray = new byte[4096];
int read = 0;
while ((read = in.read(byteArray)) != -1) {
out.write(byteArray, 0, read);
out.flush();
}
// 要多次操作這個ftp的流的通道,要等他的每次命令完成
ftp.completePendingCommand(); //TODO 這個有點問題
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} finally {
//TODO 爲什麼要這麼寫????
if (null != out) {
try {
out.close();
} catch (IOException e) {
e.printStackTrace();
}
}
if (null != in) {
try {
in.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
return strNewName;
}
經測試發現[in = ftp.retrieveFileStream(outFileName);]中獲取的in中未獲取讀出內容(猜測可能是讀取文件偶爾會很慢,造成了阻塞),程序無法進入後面的whil循環,導致空文件。
■3.解決辦法
/**
* 下載指定目錄下指定後綴名文件
*
* @param hostname FTP服務器地址
* @param port FTP服務器端口號
* @param username FTP登錄帳號
* @param password FTP登錄密碼
* @param pathname FTP服務器文件目錄
* @param localpath 下載後的文件路徑
* @return
*/
public static List<String> downloadFileForSubfix(String hostname, int port, String username, String password, String pathname,
int spanId, String localpath, String[] subfixs, Integer connectId) {
List<String> fileNames = new ArrayList<>();
List<FTPFile> fTPFiles = new ArrayList<>();
// org.apache.commons.net.ftp
FTPClient ftp = null;
try {
ftp = getFTPClient(hostname, username, password, port);
// 主動模式
ftp.enterLocalActiveMode();
// 設置文件類型 二進制文件
ftp.setFileType(FTPClient.BINARY_FILE_TYPE);
ftp.setBufferSize(1*1024*1024);
// 切換目錄
boolean flag = ftp.changeWorkingDirectory(new String(pathname.getBytes(), FTP.DEFAULT_CONTROL_ENCODING));
if (!flag) {
throw new Exception("目錄[" + pathname + "]不存在");
}
FTPFile[] fs = ftp.listFiles(pathname, file -> {
String name = file.getName();
String strSubName = name.substring(name.lastIndexOf("."));
for (String subfix : subfixs) {
if (subfix.equalsIgnoreCase(strSubName)) {
return true;
}
}
return false;
});
for (FTPFile ftpFile : fs) {
if(ftpFile.isFile()){
fTPFiles.add(ftpFile);
}
}
//文件排序--時間早的文件優先處理
Collections.sort(fTPFiles, (file, newFile) -> {
if (file.getTimestamp().getTimeInMillis() < newFile.getTimestamp().getTimeInMillis()) {
return -1;
} else if (file.getTimestamp().getTimeInMillis() == newFile.getTimestamp().getTimeInMillis()) {
return 0;
} else {
return 1;
}
});
for (int i = 0; i < fTPFiles.size(); i++) {
if (i >= DiConstants.FTP_FILE_COUNT_20s) {
break;
}
FTPFile ff = fTPFiles.get(i);
String strNewName = writeFile(ff, localpath, ftp, connectId);
//deleteFile(hostname, port, username, password, pathname, strNewName);
// 空文件判斷
if(StringUtils.isEmpty(strNewName)){
break;
}
fileNames.add(strNewName);
}
} catch (Exception e) {
logger.error("FTP下載文件異常{}", e);
} finally {
try {
if (ftp != null) {
ftp.logout();
}
} catch (IOException e1) {
e1.printStackTrace();
}
if (ftp.isConnected()) {
try {
ftp.disconnect();
} catch (IOException e) {
e.printStackTrace();
}
}
}
return fileNames;
}
public static String writeFile(FTPFile ff, String localpath, FTPClient ftp, Integer connectId) {
FileOutputStream out = null;
InputStream in = null;
String outFileName = ff.getName();
String strConnect = "CONNECT";
String strNewName = "";
try {
in = ftp.retrieveFileStream(new String(outFileName.getBytes("GB2312"),"ISO-8859-1"));
if(in.available() > 0){
strNewName = strConnect.concat(DiConstants.MARK_LEFT_BRACKET).concat(String.format("%03d", connectId)).concat(DiConstants.MARK_RIGHT_BRACKET).concat(outFileName);
File localFile = new File(localpath.concat(File.separator).concat(strNewName));
out = new FileOutputStream(localFile);
byte[] byteArray = new byte[4096];
int read = 0;
while ((read = in.read(byteArray)) != -1) {
out.write(byteArray, 0, read);
out.flush();
}
out.close();
}
in.close();
// 要多次操作這個ftp的流的通道,要等他的每次命令完成
ftp.completePendingCommand();
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} finally {
if (null != out) {
try {
out.close();
} catch (IOException e) {
e.printStackTrace();
}
}
if (null != in) {
try {
in.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
return strNewName;
}
獲取[in = ftp.retrieveFileStream(outFileName);]後判斷再進行文件下載操作。如果不能讀取文件內容,就停止本次操作。