OSS實現多文件多線程的斷點下載（java）

OSS實現多文件多線程的斷點下載（java）

本文地址：http://blog.csdn.net/zrk1000/article/details/46286767

開放存儲服務（Open Storage Service，OSS），是阿里雲對外提供的海量、安全和高可靠的雲存儲服務，目前越來越多的開發者將應用數據存放至OSS，對使用OSS實現文件的斷點續傳功能使用的也比較多，在這兒分享下自己使用OSS的實例。

所謂斷點下載，就是要從文件已經下載的地方開始繼續下載，對於斷點續傳這樣有狀態功能的實現，關鍵點在於如何在客戶端完成狀態維護。此篇主要介紹多文件的多線程的斷點下載。

爲了限定同時下載文件的個數和每個文件同時下載的線程數，使用了線程池嵌套線程池來完成，外層線程池downloadMainPool用來限定併發下載的文件數，內層線程池pool執行單個文件的多線程下載。

downloadMainPool 代碼片段：

    public static ExecutorService downloadMainPool = null;
    static{
        downloadMainPool = Executors.newFixedThreadPool(Constant.CONCURRENT_FILE_NUMBER,new ThreadFactory() {
            public Thread newThread(Runnable r) {
                Thread s = Executors.defaultThreadFactory().newThread(r);
                s.setDaemon(true);
                return s;
            }
        });
    }

在static中實例化爲固定大小的線程池，由於默認的ThreadFactory創建的線程爲非守護狀態，爲了避免java程序不能退出的問題，保證在文件下載完後當前java程序結束在jvm中的運行，需要重寫ThreadFactory，使其創建的線程爲守護線程。

Constant類中自定義了程序中需要使用到的變量，可在類中直接定義或讀取配置文件，Constant.CONCURRENT_FILE_NUMBER定義了併發文件數。

downloadMainPool中的每個線程負責一個文件，每個線程下載文件時創建子線程池，由子線程池分塊下載文件；要做到斷點續傳需要記錄每個子線程池中的每個線程下載位置，這裏使用定時序列化子線程池線程對象的方式，定時將包含了下載位置的線程序列化到文件,在再次下載同一文件時反序列化，直接丟到子線程池下載即可。下面是downloadMainPool中的線程OSSDownloadFile代碼清單

OSSDownloadFile 代碼：

package cloudStorage.oss;

import java.io.File;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import org.apache.log4j.Logger;
import cloudStorage.basis.Constant;
import cloudStorage.basis.Global;
import cloudStorage.basis.OSSClientFactory;
import cloudStorage.oss.download.DownloadPartObj;
import cloudStorage.oss.download.DownloadPartThread;
import cloudStorage.util.ObjectSerializableUtil;

import com.aliyun.oss.OSSClient;
import com.aliyun.oss.OSSException;
import com.aliyun.oss.model.ObjectMetadata;

/**
 * @Description: oss多線程分段下載文件
 * @author: zrk
 * @time: 2015年4月1日 上午10:37:35
 */
public class OSSDownloadFile  implements Callable<Integer>{
    public static final Logger LOGGER = Logger.getLogger(OSSDownloadFile.class);
    //外層線程池
    public static ExecutorService downloadMainPool = null;
    //內層線程池
    private ExecutorService pool ;

    static{
        downloadMainPool = Executors.newFixedThreadPool(Constant.CONCURRENT_FILE_NUMBER,new ThreadFactory() {
            public Thread newThread(Runnable r) {
                Thread s = Executors.defaultThreadFactory().newThread(r);
                s.setDaemon(true);
                return s;
            }
        });
    }
    private String localFilePath;//本地文件路徑
    private String bucketName; //bucketName
    private String key;//雲端存儲路徑

    public OSSDownloadFile() {
        super();
    }

    public OSSDownloadFile(String localFilePath,String bucketName,String key) {
        //初始化子線程池
        pool = Executors.newFixedThreadPool(Constant.SINGLE_FILE_CONCURRENT_THREADS);
        this.localFilePath = localFilePath;
        this.bucketName = bucketName;
        this.key = key;

    }

    //執行當前線程
    public Integer downloadFile() {
        Integer r = Global.ERROR;
        //向downloadMainPool中submit當前線程
        Future<Integer> result = downloadMainPool.submit(this);
        try {
            r=result.get();
        } catch (InterruptedException | ExecutionException e) {
            e.printStackTrace();
        } finally{
            return r;
        }

    }
    /**
     * 
     * @param localFilePath     需要存放的文件路徑
     * @param bucketName        bucketName
     * @param key               存儲key  -在oss的存儲路徑
     * @return
     */
    @Override
    public Integer call(){
        //OSSClient 使用單例
        OSSClient client = OSSClientFactory.getInstance();    
        ObjectMetadata objectMetadata = null;
        //判斷文件在雲端是否存在
        try {
            objectMetadata = client.getObjectMetadata(bucketName, key);
        } catch (OSSException e) {
            LOGGER.info("==請檢查bucketName或key");
            return Global.ERROR;
        }
        long fileLength = objectMetadata.getContentLength();

        //自定義的每個下載分塊大小
        Integer partSize = Constant.DOWNLOAD_PART_SIZE;

        //需要下載的文件分塊數
        int partCount=calPartCount(fileLength, partSize);

        //子線程池的線程對象封裝類（用於序列化的）
        DownloadPartObj downloadPartObj = null;
        boolean isSerializationFile = false;
        //序列化的文件路徑（與下載文件同路徑使用.dw.temp後綴）
        String serializationFilePath = localFilePath+".dw.temp";
        //若存在反序列化對象
        if(new File(serializationFilePath).exists()){
            downloadPartObj = (DownloadPartObj)ObjectSerializableUtil.load(serializationFilePath);
            isSerializationFile = true;
        }
        //序列化文件不存在，分配分塊給子線程池線程對象
        if(downloadPartObj==null||!isSerializationFile){
            downloadPartObj = new DownloadPartObj();
            for (int i = 0; i < partCount; i++) {
                  final  long startPos = partSize * i;
                  final  long endPos = partSize * i +( partSize < (fileLength - startPos) ? partSize : (fileLength - startPos)) - 1;
                  //DownloadPartThread是執行每個分塊下載任務的線程
                  downloadPartObj.getDownloadPartThreads().add(new DownloadPartThread(startPos, endPos, localFilePath, bucketName, key,Constant.ACCESS_ID,Constant.ACCESS_KEY));
                }
        }

        try {
            int i = 0;
            //download方法提交分塊下載線程至子線程池下砸，while循環用於下載失敗重複下載，Constant.RETRY定義重複下載次數
            while (download(downloadPartObj,serializationFilePath).isResult()==false) {
                if(++i == Constant.RETRY)break;
                LOGGER.info(Thread.currentThread().getName()+"重試第"+i+"次");
            }
        } catch (Exception e) {
            LOGGER.info("=="+e.getMessage());
            return Global.THREAD_ERROR;
        }
        if(!downloadPartObj.isResult()){
            return Global.NETWORK_ERROR;
        }
        return Global.SUCCESS;
    }


    /**
     *  多線程下載單個文件
     * @param partThreadObj
     * @param serializationFilePath
     * @return
     */
    private DownloadPartObj download(DownloadPartObj partThreadObj,String serializationFilePath){

        try {
            partThreadObj.setResult(true);
            //向子線程池中submit單個文件所有分塊下載線程
            for (int i=0 ;i<partThreadObj.getDownloadPartThreads().size();i++) {
                if (partThreadObj.getDownloadPartThreads().get(i).geteTag() == null)
                    pool.submit(partThreadObj.getDownloadPartThreads().get(i));
            }
            //shutdown子線程池，池內所下載任務執行結束後停止當前線程池
            pool.shutdown();
            //循環檢查線程池，同時在此序列化partThreadObj
            while (!pool.isTerminated()) {
                ObjectSerializableUtil.save(partThreadObj,serializationFilePath);
                pool.awaitTermination(Constant.SERIALIZATION_TIME, TimeUnit.SECONDS);
            }
            //判斷下載結果
            for (DownloadPartThread downloadPartThread: partThreadObj.getDownloadPartThreads()) {
                if (downloadPartThread.geteTag() == null){
                    partThreadObj.setResult(false);
                }
            }
            //下載成功 刪除序列化文件
            if (partThreadObj.isResult()==true) 
                ObjectSerializableUtil.delSerlzFile(serializationFilePath);

        } catch (Exception e) {
            LOGGER.info("=="+e.getMessage());
        }
        return partThreadObj;
    }

    /**
     * 獲取分塊數
     * @param fileLength
     * @param partSize
     * @return
     */
    private static int calPartCount(long fileLength,long partSize) {
        int partCount = (int) (fileLength / partSize);
        if (fileLength % partSize != 0){
            partCount++;
        }
        return partCount;
    }

    public String getLocalFilePath() {
        return localFilePath;
    }

    public void setLocalFilePath(String localFilePath) {
        this.localFilePath = localFilePath;
    }

    public String getBucketName() {
        return bucketName;
    }

    public void setBucketName(String bucketName) {
        this.bucketName = bucketName;
    }

    public String getKey() {
        return key;
    }

    public void setKey(String key) {
        this.key = key;
    }
}

此處引用了多個類
cloudStorage.basis.Constant;//定義程序中使用的變量
cloudStorage.basis.Global;//定義了全局的靜態值，錯誤狀態值
cloudStorage.basis.OSSClientFactory;//OSSClient工廠
cloudStorage.oss.download.DownloadPartObj;//分塊下載線程類封裝
cloudStorage.oss.download.DownloadPartThread;//分塊下載線程
cloudStorage.util.ObjectSerializableUtil;//序列工具類

調下載方法是阻塞式，需要給調用者返回下結果，所以使用Callable和Future返回int型狀態值，下面是子線程池pool中的分塊下載線程DownloadPartThread的代碼清單

DownloadPartThread 代碼：

package cloudStorage.oss.download;

import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.io.Serializable;
import java.util.Date;
import java.util.concurrent.Callable;
import org.apache.log4j.Logger;
import cloudStorage.basis.OSSClientFactory;
import com.aliyun.oss.common.utils.IOUtils;
import com.aliyun.oss.model.GetObjectRequest;
import com.aliyun.oss.model.OSSObject;

/**
 * @Description: 用於上傳每個part的線程類 可序列化 用於上傳的斷點續傳
 * @author: zrk
 * @time: 2015年4月1日 上午10:35:34
 */
public class DownloadPartThread implements Callable<DownloadPartThread>,Serializable {

    private static final long serialVersionUID = 1L;
    public static final Logger LOGGER = Logger.getLogger(DownloadPartThread.class);
    // 當前線程的下載開始位置
    private long startPos;

    // 當前線程的下載結束位置
    private long endPos;

    // 保存文件路徑
    private String localFilePath;

    private String bucketName;
    private String fileKey;
    private String eTag;
    private String accessId;
    private String accessKey;

    public DownloadPartThread(long startPos, long endPos, String localFilePath,
            String bucketName, String fileKey, String accessId,
            String accessKey) {
        this.startPos = startPos;
        this.endPos = endPos;
        this.localFilePath = localFilePath;
        this.bucketName = bucketName;
        this.fileKey = fileKey;
        this.accessId = accessId;
        this.accessKey = accessKey;
    }

    @Override
    public DownloadPartThread call() {
        RandomAccessFile file = null;
        OSSObject ossObject = null;
        try {
            File pFile = new File(localFilePath);
            if(!pFile.getParentFile().exists())
                pFile.getParentFile().mkdirs();
            file = new RandomAccessFile(localFilePath, "rw");
            //調用ossapi
            GetObjectRequest getObjectRequest = new GetObjectRequest(bucketName, fileKey);
            getObjectRequest.setRange(startPos, endPos);
            ossObject = OSSClientFactory.getInstance().getObject(getObjectRequest);
            file.seek(startPos);
            int bufSize = 1024;
            byte[] buffer = new byte[bufSize];
            int bytesRead;
            while ((bytesRead = ossObject.getObjectContent().read(buffer)) > -1) {
                file.write(buffer, 0, bytesRead);
                //更新開始位置，保證在出錯後重下載是從上次結束的地方開始下，而不是下載整個塊
                startPos += bytesRead; 
            }
            this.eTag = ossObject.getObjectMetadata().getETag();
        } catch (Exception e) {
            LOGGER.info("=="+e.getMessage());
        } finally{
            if(ossObject!=null)IOUtils.safeClose(ossObject.getObjectContent());
            try {if(file!=null)file.close();} catch (IOException e) {e.printStackTrace();}
            return this;
        }

    }
}

每個DownloadPartThread下載線程執行結束都會將自己作爲返回值返回，當前文件是否下載完整有每個線程的返回值決定。

ObjectSerializableUtil 代碼 點擊查看

DownloadPartObj 代碼：

package cloudStorage.oss.download;

import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;

/**
 * @Description: 單個文件的下載線程集合
 * @author: zrk
 * @time: 2015年5月5日 上午10:15:11
 */
public class DownloadPartObj implements Serializable{
    private static final long serialVersionUID = 1L;
    /**
     * 下載線程集合
     */
    List<DownloadPartThread> downloadPartThreads = Collections.synchronizedList(new ArrayList<DownloadPartThread>());
    /**
     * 下載結果
     */
    boolean result = true;

    public List<DownloadPartThread> getDownloadPartThreads() {
        return downloadPartThreads;
    }
    public void setDownloadPartThreads(List<DownloadPartThread> downloadPartThreads) {
        this.downloadPartThreads = downloadPartThreads;
    }
    public boolean isResult() {
        return result;
    }
    public void setResult(boolean result) {
        this.result = result;
    }

}

所有下載當前文件的線程都會操作downloadPartThreads，所以downloadPartThreads使用集合Collections.synchronizedList將其轉換爲一個線程安全的類。DownloadPartObj封裝了downloadPartThreads和一個用於標識下載成功與失敗的boolean值，定時保存序列化文件就直接序列化DownloadPartObj，DownloadPartObj的線程安全是至關重要的。

OSSClientFactory 代碼：

package cloudStorage.basis;
import com.aliyun.oss.OSSClient;
public class OSSClientFactory {
    private static OSSClient ossClient = null;
    private OSSClientFactory() {
    }
    public static OSSClient getInstance() {
        if (ossClient == null) {
            // 可以使用ClientConfiguration對象設置代理服務器、最大重試次數等參數。
            // ClientConfiguration config = new ClientConfiguration();
            ossClient = new OSSClient(Constant.OSS_ENDPOINT,Constant.ACCESS_ID, Constant.ACCESS_KEY);
        }
        return ossClient;
    }
}

Constant 代碼：

package cloudStorage.basis;

import cloudStorage.service.OSSConfigService;

/**
 * @Description: 
 * @author: zrk
 * @time: 2015年4月1日 下午5:22:28
 */
public class Constant {
    public static String OSS_ENDPOINT = "http://oss.aliyuncs.com/";
    public static String ACCESS_ID;
    public static String ACCESS_KEY;
    public static Integer DOWNLOAD_PART_SIZE ; // 每個下載Part的大小
    public static Integer UPLOAD_PART_SIZE ; // 每個上傳Part的大小
    public static int CONCURRENT_FILE_NUMBER ; // 併發文件數。
    public static int SINGLE_FILE_CONCURRENT_THREADS ; // 單文件併發線程數。
    public static int RETRY ;//失敗重試次數
    public static int SERIALIZATION_TIME;//斷點保存時間間隔(秒)
    //。。。
}

Constant中數值是加載外部配置文件，也可在這兒直接配置，個別參數值斷點上傳時使用，只看下載的話請忽略。

Global代碼：

package cloudStorage.basis;
/**
 * @Description: TODO
 * @author: zrk
 * @time: 2015年4月1日 下午5:22:46
 */
public class Global {
    public static final int SUCCESS = 1;
    public static final int ERROR = 10;
    public static final int FILE_NOT_FOUND_ERROR = 11;
    public static final int THREAD_ERROR = 12;
    public static final int NETWORK_ERROR = 13;
    public static final int OSS_SUBMIT_ERROR = 14;
//  META
    public static final String X_OSS_META_MY_MD5 = "x-oss-meta-my-md5";
}

調用下載方式：

實例化OSSDownloadFile後調用downloadFile方法：
return new OSSDownloadFile(localFilePath,bucketName, key).downloadFile();

OSS SDK文件的分塊下載支持的很好，OSS官方的SDK裏面也提供了一個多線程下載功能的實現，所以在實現文件的分塊下載並沒有什麼難度。這塊兒多線程下載僅供大家參考，後面自己在使用過程中繼續優化。基於OSS-SDK的分塊斷點上傳請參考OSS實現多文件多線程的斷點上傳（java） 。

本文地址：http://blog.csdn.net/zrk1000/article/details/46286767

---