最近這兩天一直在弄大文件的分片上傳和下載，前端選擇文件，然後使用百度的webuploader切的文件（1M一份），然後調用ftp的api上傳到了服務器上，然後再調用api對文件就行追加，其中遇到了文件損壞，輸入流爲null的問題，都一一解決掉了，現在記錄下。
一、工具類如下：(jar包 commons-net 3.6)
@Data
public class IotFtpService {
private Logger log = LoggerFactory.getLogger(this.getClass());
private static String directory;
/**
* ftpClient連接池初始化標誌
*/
private boolean hasInit = false;
/**
* ftpClient連接池
*/
private ObjectPool<FTPClient> ftpClientPool;
/**
* 上傳文件
*
* @param pathname ftp服務保存地址
* @param fileName 上傳到ftp的文件名
* @param originFilename 待上傳文件的名稱（絕對地址） *
* @return
*/
public boolean uploadFile(String pathname, String fileName, String originFilename) {
FTPClient ftpClient = getFtpClient();
boolean flag = false;
InputStream inputStream = null;
try {
ftpClient.setFileType(FTP.BINARY_FILE_TYPE);
log.info("開始上傳文件");
inputStream = new FileInputStream(new File(originFilename));
CreateDirecroty(pathname, ftpClient, fileName);
ftpClient.makeDirectory(pathname);
ftpClient.changeWorkingDirectory(pathname);
//ftpClient.storeFile(fileName, inputStream);//上傳文件
ftpClient.appendFile(fileName, inputStream);//追加文件
inputStream.close();
flag = true;
log.info("上傳文件成功");
} catch (Exception e) {
log.error("上傳文件失敗");
e.printStackTrace();
} finally {
releaseFtpClient(ftpClient);
}
return flag;
}
/**
* 上傳文件
*
* @param pathname ftp服務保存地址
* @param fileName 上傳到ftp的文件名
* @param inputStream 輸入文件流
* @return
*/
public boolean uploadFile(String pathname, String fileName, InputStream inputStream) {
boolean flag = false;
FTPClient ftpClient = getFtpClient();
try {
log.info("開始上傳文件");
ftpClient.setFileType(FTP.BINARY_FILE_TYPE);
CreateDirecroty(pathname, ftpClient, fileName);
ftpClient.makeDirectory(pathname);
ftpClient.changeWorkingDirectory(pathname);
ftpClient.storeFile(fileName, inputStream);
inputStream.close();
flag = true;
log.info("上傳文件成功");
} catch (Exception e) {
log.error("上傳文件失敗");
e.printStackTrace();
} finally {
releaseFtpClient(ftpClient);
}
return flag;
}
/**
* 合併分片文件
*/
public void mergeChunk(String fileName) {
}
/**
* 顯示路徑下的分片文件
*
* @param dir
*/
public void showDirFiles(String dir) {
List<byte[]> listByte = new ArrayList<>();
FTPClient ftpClient = getFtpClient();
try {
log.info("select");
ftpClient.changeWorkingDirectory(dir);
ftpClient.setFileType(FTP.BINARY_FILE_TYPE);
FTPFile[] list = ftpClient.listFiles();
//排序目標
for (int i = 0; i < list.length; i++) {
for (int j = i + 1; j < list.length; j++) {
if (Integer.parseInt(list[i].getName()) > Integer.parseInt(list[j].getName())) {
FTPFile o =list[i];
list[i]=list[j];
list[j]=o;
}
}
System.out.println("查看排序"+list[i].getName());
}
InputStream in = null;
for (int i = 0; i < list.length; i++) {
try {
log.info(">>>>>>>>" + list[i].getName());
//讀取小文件的輸入流
in = ftpClient.retrieveFileStream(encodingPath("/data/eanupload/xx.tar.gz.tmp/" + list[i].getName()));
byte[] bytes = IOUtils.toByteArray(in);
//關閉輸入流
in.close();
//ftp傳輸結束
ftpClient.completePendingCommand();
listByte.add(bytes);
} catch (Exception e) {
e.printStackTrace();
}
/*file.delete();*/
}
//開始合併文件
for (byte[] bytes : listByte) {
in = new ByteArrayInputStream(bytes);
boolean flag = ftpClient.appendFile("/data/eanupload/test2.gz", in);
in.close();
}
} catch (Exception e) {
e.printStackTrace();
} finally {
releaseFtpClient(ftpClient);
}
}
/**
* 下載文件 *
*
* @param pathname FTP服務器文件目錄 *
* @param filename 文件名稱 *
* @param localpath 下載後的文件路徑 *
* @return
*/
public boolean downloadFile(String pathname, String filename, String localpath) {
boolean flag = false;
String remote = pathname + "/" + filename;
OutputStream os = null;
// 獲取本地文件大小
long checkLocalFile = checkLocalFile(localpath + "/" + filename);
FTPClient ftpClient = getFtpClient();
try {
ftpClient.setFileType(FTP.BINARY_FILE_TYPE);
log.info("開始下載文件");
//切換FTP目錄
ftpClient.changeWorkingDirectory(pathname);
FTPFile[] ftpFiles = ftpClient.listFiles();
// 判斷本地文件是否存在
if (checkLocalFile != 0) {
System.out.println("本地文件已存在");
// 存在，則進行斷點下載
for (FTPFile ff : ftpFiles) {
if (ff.getName().equals(filename)) {
checkLocalFile = downBreakPoint(localpath,
checkLocalFile, ff, remote);
}
flag = true;
}
} else {
// 直接下載
for (FTPFile file : ftpFiles) {
if (filename.equalsIgnoreCase(file.getName())) {
File localFile = new File(localpath + "/" + file.getName());
os = new FileOutputStream(localFile);
ftpClient.retrieveFile(file.getName(), os);
os.close();
}
}
flag = true;
log.info("下載文件成功");
}
} catch (Exception e) {
log.error("下載文件失敗");
e.printStackTrace();
} finally {
releaseFtpClient(ftpClient);
if (null != os) {
try {
os.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
return flag;
}
/**
* 斷點下載代碼段
*
* @param localPath
* @param checkLocalFile
* @param ff
* @param remote 遠程服務器路徑+文件名
* @return checkLocalFile 返回long值
* @throws FileNotFoundException
* @throws IOException
* @throws UnsupportedEncodingException
*/
public long downBreakPoint(String localPath, long checkLocalFile, FTPFile ff, String remote) throws IOException {
FTPClient ftpClient = getFtpClient();
// 獲得服務器文件大小
long RemoteSize = ff.getSize();
if (RemoteSize <= checkLocalFile) {
System.out.println("文件已下載");
} else {
// 斷點下載
File localFile = new File(localPath + "/" + ff.getName());
OutputStream out = new FileOutputStream(localFile);
InputStream in = ftpClient.retrieveFileStream(remote);//服務器路徑+文件名
ftpClient.setRestartOffset(checkLocalFile);
byte[] bytes = new byte[1024];
long step = RemoteSize / 100;
long process = checkLocalFile / step;
int c;
while ((c = in.read(bytes)) != -1) {
out.write(bytes, 0, c);
checkLocalFile += c;
long nowProcess = RemoteSize / step;
if (nowProcess > process) {
process = nowProcess;
if (process % 10 == 0)
System.out.println("下載進度：" + process);
//TODO 更新文件下載進度,值存放在process變量中
}
}
in.close();
out.close();
ftpClient.completePendingCommand();
releaseFtpClient(ftpClient);//釋放ftp客戶端
}
return checkLocalFile;
}
/**
* 斷點續傳
*
* @param remotePath 遠程路徑
* @param filename 本地上傳的文件名
* @param file 絕對路徑
* @return
* @throws IOException
*/
public boolean uploadFileBreakPoint(String filename, File file, String remotePath) {
boolean status = false;
FTPClient ftpClient = getFtpClient();
try {
// 轉移目錄到遠端服務器目錄
ftpClient.changeWorkingDirectory(remotePath);
String f = new String(filename.getBytes("GBK"), "iso-8859-1");
// 檢查ftp服務器中目錄否已存在
boolean result = CreateDirecroty(remotePath, ftpClient, filename);
if (result) {
// 文件存在，判斷文件大小進行選擇上傳
System.out.println("文件已存在");
// 1.服務器文件大於將要上傳文件，不需要上傳，或者重新上傳
FTPFile[] listFiles = ftpClient.listFiles(remotePath
+ "/" + f);
System.out.println(file.length());
if (listFiles[0].getSize() >= file.length()) {
System.out.println("不需要上傳");
return true;
} else {
//開始執行斷點續傳
status = resumeBreakPoint(filename, file, listFiles, remotePath);
}
} else {
//文件不存在直接上傳
uploadFile(remotePath, filename, new FileInputStream(file));
}
} catch (IOException e) {
e.printStackTrace();
} finally {
releaseFtpClient(ftpClient);//釋放sftp
}
return status;
}
/**
* 斷點上傳代碼段
*
* @param filename
* @param file
* @param listFiles
* @return
* @throws FileNotFoundException
* @throws IOException
* @throws UnsupportedEncodingException
*/
private boolean resumeBreakPoint(String filename, File file, FTPFile[] listFiles, String remotePath) throws IOException {
FTPClient ftpClient = getFtpClient();
boolean status;
// 2.服務器文件小於將要上傳文件，進行斷點續傳
// 顯示上傳的進度
// long step = file.length() / 10;
// long process = 0;
long localreadbytes = 0L;
RandomAccessFile raf = new RandomAccessFile(file, "rw");
OutputStream out = ftpClient.appendFileStream(remotePath + "/" + filename);
// 斷點續傳
ftpClient.setRestartOffset(listFiles[0].getSize());
// process = listFiles[0].getSize() % step;
raf.seek(listFiles[0].getSize());
localreadbytes = listFiles[0].getSize();
byte[] bytes = new byte[1024];
int c;
while ((c = raf.read(bytes)) != -1) {
out.write(bytes, 0, c);
localreadbytes += c;
// if (localreadbytes / step != process) {
// process = localreadbytes / step;
// System.out.println("上傳進度:" + process);
// }
}
out.flush();
raf.close();
out.close();
boolean result = ftpClient.completePendingCommand();
if (listFiles[0].getSize() > 0) {
status = result ? true : false;
} else {
status = result ? true : false;
}
releaseFtpClient(ftpClient);//釋放ftp客戶端
return status;
}
/**
* 刪除文件 *
*
* @param pathname FTP服務器保存目錄 *
* @param filename 要刪除的文件名稱 *
* @return
*/
public boolean deleteFile(String pathname, String filename) {
boolean flag = false;
FTPClient ftpClient = getFtpClient();
try {
log.info("開始刪除文件");
//切換FTP目錄
ftpClient.changeWorkingDirectory(pathname);
ftpClient.dele(filename);
flag = true;
log.info("刪除文件成功");
} catch (Exception e) {
log.error("刪除文件失敗");
e.printStackTrace();
} finally {
releaseFtpClient(ftpClient);
}
return flag;
}
/**
* 按行讀取FTP文件
*
* @param remoteFilePath 文件路徑（path+fileName）
* @return
* @throws IOException
*/
public List<String> readFileByLine(String remoteFilePath) throws IOException {
FTPClient ftpClient = getFtpClient();
try (InputStream in = ftpClient.retrieveFileStream(encodingPath(remoteFilePath));
BufferedReader br = new BufferedReader(new InputStreamReader(in))) {
return br.lines().map(line -> StrUtil.trimToEmpty(line))
.filter(line -> StrUtil.isNotEmpty(line)).collect(Collectors.toList());
} finally {
ftpClient.completePendingCommand();
releaseFtpClient(ftpClient);
}
}
/**
* 獲取指定路徑下FTP文件
*
* @param remotePath 路徑
* @return FTPFile數組
* @throws IOException
*/
public FTPFile[] retrieveFTPFiles(String remotePath) throws IOException {
FTPClient ftpClient = getFtpClient();
try {
return ftpClient.listFiles(encodingPath(remotePath + "/"),
file -> file != null && file.getSize() > 0);
} finally {
releaseFtpClient(ftpClient);
}
}
/**
* 獲取指定路徑下FTP文件名稱
*
* @param remotePath 路徑
* @return ftp文件名稱列表
* @throws IOException
*/
public List<String> retrieveFileNames(String remotePath) throws IOException {
FTPFile[] ftpFiles = retrieveFTPFiles(remotePath);
if (ArrayUtil.isEmpty(ftpFiles)) {
return new ArrayList<>();
}
return Arrays.stream(ftpFiles).filter(Objects::nonNull)
.map(FTPFile::getName).collect(Collectors.toList());
}
/**
* 編碼文件路徑
*/
private static String encodingPath(String path) throws UnsupportedEncodingException {
// FTP協議裏面，規定文件名編碼爲iso-8859-1，所以目錄名或文件名需要轉碼
return new String(path.replaceAll("//", "/").getBytes("GBK"), "iso-8859-1");
}
/**
* 獲取ftpClient
*
* @return
*/
private FTPClient getFtpClient() {
checkFtpClientPoolAvailable();
FTPClient ftpClient = null;
Exception ex = null;
// 獲取連接最多嘗試3次
for (int i = 0; i < 3; i++) {
try {
ftpClient = ftpClientPool.borrowObject();
ftpClient.enterLocalPassiveMode();//被動模式
ftpClient.changeWorkingDirectory("/");
break;
} catch (Exception e) {
ex = e;
}
}
if (ftpClient == null) {
throw new RuntimeException("Could not get a ftpClient from the pool", ex);
}
return ftpClient;
}
/**
* 釋放ftpClient
*/
private void releaseFtpClient(FTPClient ftpClient) {
if (ftpClient == null) {
return;
}
try {
ftpClientPool.returnObject(ftpClient);
} catch (Exception e) {
log.error("Could not return the ftpClient to the pool", e);
// destoryFtpClient
if (ftpClient.isAvailable()) {
try {
ftpClient.logout();//嘗試解決文件損壞問題
ftpClient.disconnect();
} catch (IOException io) {
}
}
}
}
/**
* 檢查ftpClientPool是否可用
*/
private void checkFtpClientPoolAvailable() {
Assert.state(hasInit, "FTP未啓用或連接失敗！");
}
/**
* 創建多層目錄文件，如果有ftp服務器已存在該文件，則不創建，如果無，則創建
*
* @param remote
* @param ftpClient
* @return
* @throws IOException true文件存在 false文件不存在
*/
public boolean CreateDirecroty(String remote, FTPClient ftpClient, String filename) throws IOException {
boolean success = false;
//檢查遠程目錄是否存在情況
checkRemote(remote);
// 如果遠程目錄不存在，則遞歸創建遠程服務器目錄
if (!directory.equalsIgnoreCase("/") && !changeWorkingDirectory(new String(directory), ftpClient)) {
int start = 0;
int end = 0;
if (directory.startsWith("/")) {
start = 1;
} else {
start = 0;
}
end = directory.indexOf("/", start);
String path = "";
String paths = "";
while (true) {
// 對目錄進行轉碼
String subDirectory = new String(remote.substring(start, end).getBytes("GBK"), "iso-8859-1");
path = path + "/" + subDirectory;
if (!checkRemoteFile(path, ftpClient)) {
if (makeDirectory(subDirectory, ftpClient)) {
changeWorkingDirectory(subDirectory, ftpClient);
} else {
System.out.println("創建目錄[" + subDirectory + "]失敗");
changeWorkingDirectory(subDirectory, ftpClient);
}
} else {
changeWorkingDirectory(subDirectory, ftpClient);
}
paths = paths + "/" + subDirectory;
start = end + 1;
end = directory.indexOf("/", start);
// 檢查所有目錄是否創建完畢
if (end <= start) {
break;
}
}
} else {
//目錄存在則校驗服務器文件是否存在
success = checkRemoteFile(filename, ftpClient);
}
return success;
}
/**
* 改變目錄路徑
*
* @param directory
* @param ftpClient
* @return
*/
public boolean changeWorkingDirectory(String directory, FTPClient ftpClient) {
boolean flag = true;
try {
flag = ftpClient.changeWorkingDirectory(directory);
if (flag) {
System.out.println("進入文件夾" + directory + " 成功！");
} else {
System.out.println("進入文件夾" + directory + " 失敗！開始創建文件夾");
}
} catch (IOException ioe) {
ioe.printStackTrace();
}
return flag;
}
/**
* /創建目錄
*
* @param dir
* @param ftpClient
* @return
*/
public boolean makeDirectory(String dir, FTPClient ftpClient) {
boolean flag = true;
try {
flag = ftpClient.makeDirectory(dir);
if (flag) {
log.info("創建文件夾" + dir + " 成功！");
} else {
log.info("創建文件夾" + dir + " 失敗！");
}
} catch (Exception e) {
e.printStackTrace();
}
return flag;
}
/**
* 判斷ftp服務器文件是否存在
*
* @param filename
* @param ftp
* @return
* @throws IOException
*/
public boolean checkRemoteFile(String filename, FTPClient ftp)
throws IOException {
boolean flag = false;
FTPFile[] ftpFileArr = ftp.listFiles(filename);
for (FTPFile ftpFile : ftpFileArr) {
if (ftpFileArr.length > 0) {
flag = true;//存在
}
}
return flag;
}
/**
* 檢查本地文件是否存在
*
* @param localPath 本地文件路徑
* @return
*/
public long checkLocalFile(String localPath) {
File file = new File(localPath);
if (file.exists()) {
return file.length();
}
return 0;
}
/**
* 檢查遠程目錄
*
* @param remote 遠程服務器目錄
*/
private static void checkRemote(String remote) {
char c = remote.charAt(remote.length() - 1);
if ("/".charAt(0) == c) {
directory = remote;
} else {
directory = remote + "/";
}
}
}
二、注意事項
在分片文件進行合併的時候，我使用的是 ftpClient.appendFile(fileName, inputStream);
1.該方法需要設置ftpClient的傳輸方式，詳情見代碼
2.分片的文件需要排序後再合併，否則會把文件損壞切記。