多線程斷點續傳--java

以前看過一個前輩寫的斷點續傳的文章，記得當時沒看懂，就扔那了。昨天翻了出來，自己也仿照寫了一個，不過感覺沒人家寫的好，有點亂，希望大家能來批評、指正，給點意見！

 

    功能很簡單，就是啓動多個線程分別從給定的地址下載數據，用RandomAccessFile寫到目標文件。實現思路是：

    1、獲得連接的長度（即要下載的文件大小），除以設定的線程數，即得到每個線程要下載的大小。

    2、記錄臨時文件，文件中記錄每個線程的編號（id），該線程要下載的起始位置、終止位置和當前位置（當前位置在首次下載時與起始位置相同）。

    3、啓動具體執行下載任務的線程，並等待其結束。

    4、下載完成，刪除臨時文件。

 

代碼如下:

主線程及測試的main方法

package com.why.download.test;

import java.io.DataOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.math.BigDecimal;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.util.UUID;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

/**
 *
 * @author why
 *
 */
public class DownLoad {
    //文件目錄、文件名
    public String fileDir = "E:/MyDownLoad";
    public String fileName;
    //超時重連時間
    public long reconnectTime = 5;
    //線程數
    private int poolSize = 5;
    //每個線程的緩衝區大小
    public int bufferSize = 1024;
    //url地址
    private String urlLocation = null;

    public DownLoad(){}
    public DownLoad(String url){
        this.urlLocation = url;
    }
    public void downLoad(){
        if(this.urlLocation == null || "".equals(this.urlLocation))return;
        downLoad(this.urlLocation);
    }
    public void downLoad(String urlLocation){
        File file = null;
        File tempFile = null;
        CountDownLatch latch;
        URL url = null;
        ExecutorService pool = Executors.newCachedThreadPool();
        long contentLength = 0;
        long threadLength = 0;
        try {
            //如果未指定名稱，則從url中獲得下載的文件格式與名字
            if(fileName == null || "".equals(fileName)){
                this.fileName = urlLocation.substring(urlLocation.lastIndexOf("/") + 1,
                        urlLocation.lastIndexOf("?") > 0 ? urlLocation.lastIndexOf("?")
                                : urlLocation.length());
                if ("".equalsIgnoreCase(this.fileName)) {
                    this.fileName = UUID.randomUUID().toString();
                }
            }
            new File(fileDir).mkdirs();
            file = new File(fileDir + File.separator + fileName);
            tempFile = new File(fileDir + File.separator + fileName + "_temp");

            url = new URL(urlLocation);
            HttpURLConnection conn = (HttpURLConnection) url.openConnection();
            setHeader(conn);
            //得到content的長度
            contentLength = conn.getContentLength();

            System.out.println("total length=" + contentLength);

            //把context分爲poolSize段，計算每段的長度。
//          threadLength = contentLength / this.poolSize;
            BigDecimal b1 = new BigDecimal(Double.toString(contentLength));
            BigDecimal b2 = new BigDecimal(Double.toString(this.poolSize));
            threadLength = b1.divide(b2, 0, BigDecimal.ROUND_HALF_UP).longValue();

            if(file.exists() && tempFile.exists()){
                //如果文件已存在，根據臨時文件中記載的線程數量，繼續上次的任務
                latch = new CountDownLatch((int)tempFile.length()/28);
                for(int i=0;i<tempFile.length()/28;i++){
                    pool.submit(new DownLoadTask(file, tempFile, url, i+1,latch,reconnectTime,bufferSize));
                }
            }else{
                //如果下載的目標文件不存在，則創建新文件
                latch = new CountDownLatch(poolSize);
                file.createNewFile();
                tempFile.createNewFile();
                DataOutputStream os = new DataOutputStream(new FileOutputStream(tempFile));
                for(int i=0;i<this.poolSize;i++){
                    os.writeInt(i+1);
                    os.writeLong(i*threadLength);
                    if(i==this.poolSize-1){//最後一個線程的結束位置應爲文件末端
                        os.writeLong(contentLength);
                    }else{
                        os.writeLong((i+1)*threadLength);
                    }
                    os.writeLong(i*threadLength);
                    pool.submit(new DownLoadTask(file, tempFile, url, i+1,latch,reconnectTime,bufferSize));
                }
                os.close();
            }
            //等待下載任務完成
            latch.await();
            //刪除臨時文件
            tempFile.delete();
        } catch (IOException e) {
            e.printStackTrace();
        } catch (InterruptedException e) {
            e.printStackTrace();
        } finally{
            pool.shutdown();
        }
    }

    private void setHeader(URLConnection conn) {
        conn.setRequestProperty("User-Agent",
                        "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.3) Gecko/2008092510 Ubuntu/8.04 (hardy) Firefox/3.0.3");
        conn.setRequestProperty("Accept-Language", "en-us,en;q=0.7,zh-cn;q=0.3");
        conn.setRequestProperty("Accept-Encoding", "aa");
        conn.setRequestProperty("Accept-Charset","ISO-8859-1,utf-8;q=0.7,*;q=0.7");
        conn.setRequestProperty("Keep-Alive", "300");
        conn.setRequestProperty("Connection", "keep-alive");
        conn.setRequestProperty("If-Modified-Since", "Fri, 02 Jan 2009 17:00:05 GMT");
        conn.setRequestProperty("If-None-Match", "\"1261d8-4290-df64d224\"");
        conn.setRequestProperty("Cache-Control", "max-age=0");
        conn.setRequestProperty("Referer","http://www.skycn.com/soft/14857.html");
    }


    public String getFileDir() {
        return fileDir;
    }
    public void setFileDir(String fileDir) {
        this.fileDir = fileDir;
    }
    public String getFileName() {
        return fileName;
    }
    public void setFileName(String fileName) {
        this.fileName = fileName;
    }
    public long getReconnectTime() {
        return reconnectTime;
    }
    public void setReconnectTime(long reconnectTime) {
        this.reconnectTime = reconnectTime;
    }
    public int getPoolSize() {
        return poolSize;
    }
    public void setPoolSize(int poolSize) {
        this.poolSize = poolSize;
    }
    public int getBufferSize() {
        return bufferSize;
    }
    public void setBufferSize(int bufferSize) {
        this.bufferSize = bufferSize;
    }
    /**
     * @param args
     */
    public static void main(String[] args) {
        DownLoad dl = new DownLoad();
        dl.setFileDir("E:/MyDownLoad/music/");
        dl.setFileName("大笑江湖.mp3");
        dl.setPoolSize(20);
        long beginTime = System.currentTimeMillis();
        dl.downLoad("http://mh.163k.com/UploadFile/video/2010/12-13/201012131213448942190.mp3");
        long endTime = System.currentTimeMillis();
        BigDecimal b1 = new BigDecimal(endTime - beginTime);
        BigDecimal b2 = new BigDecimal(1000);
        double cost = b1.divide(b2, 2, BigDecimal.ROUND_HALF_UP).doubleValue();
        System.out.println("Time cost:" + cost + "s");
    }

}

 

 

執行下載任務的線程：

package com.why.download.test;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.RandomAccessFile;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.util.concurrent.Callable;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;

/**
 *
 * @author why
 *
 */
public class DownLoadTask implements Callable<String>{
    //超時重連時間
    private long reconnectTime = 5;
    //緩衝區大小
    private int bufferSize = 1024;

    private CountDownLatch latch;
    private RandomAccessFile file = null;
    private RandomAccessFile tempFile = null;
    private URL url = null;
    private int id;
    private long startPosition;
    private long endPosition;
    private long currentPosition ;

    public DownLoadTask(File file,File tempFile,URL url,int id,CountDownLatch latch,long reconnectTime,int bufferSize){
        try {
            this.file = new RandomAccessFile(file, "rw");
            this.tempFile = new RandomAccessFile(tempFile, "rw");
        } catch (FileNotFoundException e) {
            e.printStackTrace();
        }
        this.url = url;
        this.id = id;
        this.latch = latch;
    }

    public String call(){

        try {
            tempFile.seek((id-1)*28);
            tempFile.readInt();
            this.startPosition = tempFile.readLong();
            this.endPosition = tempFile.readLong();
            this.currentPosition = tempFile.readLong();
        } catch (IOException e) {
            e.printStackTrace();
        }

        System.out.println("Thread " + id + " begin!");

        HttpURLConnection conn = null;
        InputStream inputStream = null;

        while(true){
            try {
                tempFile.seek(id*28 - 8);
                // 打開URLConnection
                conn = (HttpURLConnection) this.url.openConnection();
                setHeader(conn);
                // 設置連接超時時間爲10000ms
                conn.setConnectTimeout(10000);
                // 設置讀取數據超時時間爲10000ms
                conn.setReadTimeout(10000);

                if (currentPosition < endPosition) {
                    // 設置下載數據的起止區間
                    conn.setRequestProperty("Range", "bytes=" + currentPosition + "-" + endPosition);

                    System.out.println("Thread " + id + " startPosition=" + startPosition
                            + ",endPosition=" + endPosition + ",currentPosition=" + currentPosition);

                    file.seek(currentPosition);

                    // 判斷http status是否爲HTTP/1.1 206 Partial Content或者200 OK
                    // 如果不是以上兩種狀態，把status改爲STATUS_HTTPSTATUS_ERROR
                    if (conn.getResponseCode() != HttpURLConnection.HTTP_OK
                            && conn.getResponseCode() != HttpURLConnection.HTTP_PARTIAL) {
                        System.out.println("Thread " + id + ": code = " + conn.getResponseCode() + ", status = " + conn.getResponseMessage());
                        file.close();
                        conn.disconnect();
                        System.out.println("Thread " + id + " finished.");
                        break;
                    }

                    inputStream = conn.getInputStream();
                    int len = 0;
                    byte[] b = new byte[bufferSize];
                    while ((len = inputStream.read(b)) != -1) {
                        file.write(b, 0, len);

                        currentPosition += len;
                        // set tempFile now position
                        tempFile.seek(id*28 - 8);
                        tempFile.writeLong(currentPosition);
                    }

                    file.close();
                    tempFile.close();
                    inputStream.close();
                    conn.disconnect();
                }

                System.out.println("Thread " + id + " finished.");
                break;
            } catch (IOException e) {
                try {
                    TimeUnit.SECONDS.sleep(getReconnectTime());
                } catch (InterruptedException e1) {
                    e1.printStackTrace();
                }
                continue;
            }
        }
        latch.countDown();
        return "finish";
    }

    private void setHeader(URLConnection conn) {
        conn.setRequestProperty("User-Agent",
                        "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.3) Gecko/2008092510 Ubuntu/8.04 (hardy) Firefox/3.0.3");
        conn.setRequestProperty("Accept-Language", "en-us,en;q=0.7,zh-cn;q=0.3");
        conn.setRequestProperty("Accept-Encoding", "aa");
        conn.setRequestProperty("Accept-Charset","ISO-8859-1,utf-8;q=0.7,*;q=0.7");
        conn.setRequestProperty("Keep-Alive", "300");
        conn.setRequestProperty("Connection", "keep-alive");
        conn.setRequestProperty("If-Modified-Since", "Fri, 02 Jan 2009 17:00:05 GMT");
        conn.setRequestProperty("If-None-Match", "\"1261d8-4290-df64d224\"");
        conn.setRequestProperty("Cache-Control", "max-age=0");
        conn.setRequestProperty("Referer","http://www.skycn.com/soft/14857.html");
    }

    public long getReconnectTime() {
        return reconnectTime;
    }

    public void setReconnectTime(long reconnectTime) {
        this.reconnectTime = reconnectTime;
    }

    public int getBufferSize() {
        return bufferSize;
    }

    public void setBufferSize(int bufferSize) {
        this.bufferSize = bufferSize;
    }


}

 

    最近一直在測試，寫文檔，好久沒敲代碼了。手有點癢，寫着玩的，代碼寫的不是很工整，純屬娛樂！

轉載自：http://wuhongyu.iteye.com/blog/869109