利用JUnit實現對hadoop中javaAPI的測試

package gorilla.test;

import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.io.IOUtils;
import org.junit.Before;
import org.junit.Test;

public class HDFSTest02 {
    FileSystem fSystem;
    /**
     * 連接hdfs
     * @throws IOException
     * @throws InterruptedException
     * @throws URISyntaxException
     */
    @Before
    public void init() throws IOException, InterruptedException, URISyntaxException {
        Configuration configuration = new Configuration();
        fSystem = FileSystem.get(new URI("hdfs://192.168.15.134:9000"), configuration, "hadoop");
        
    }
    @Test
    public void test() {
        System.out.println("test");
    }
    
    /**
     * 將本地文件上傳
     * @throws IllegalArgumentException
     * @throws IOException
     */
    @Test
    public void addFileToHdfs() throws IllegalArgumentException, IOException {
        /*
         *  比如/eee 而根目錄下不存在eee這個文件夾，就會認爲是要將文件傳到/ 下面的eee文件裏
         *  如果eee是一個文件夾就會在eee文件夾下使用原名字上傳
         *  如果是/eee/ff.md就知道是要上傳到/eee這個文件夾下的ff.md這個文件中
         */
        fSystem.copyFromLocalFile(new Path("C:/Users/Administrator/Desktop/input/block.rar"), new Path("/user/block.rar"));
    }
    
    /**
     * 將hdfs文件下載到本地
     * @throws IllegalArgumentException
     * @throws IOException
     */
    @Test
    public void addFileFromHdfs() throws IllegalArgumentException, IOException {
        fSystem.copyToLocalFile(false, new Path("/source.txt"), new Path("G:/HDFSDownload"), true);
    }
    
    // 刪除一個參數的方法以及棄用,兩個參數的第二個參數表示是否要使用遞歸操作,如果刪除的是文件夾，只要不是空文件夾，就必須設置成true允許遞歸操作才能刪除
    @Test
    public void delete() throws IllegalArgumentException, IOException {
        fSystem.delete(new Path(""), true);
    }
    // 查看目錄下的文件信息
    @Test
    public void testListFiles() throws FileNotFoundException, IllegalArgumentException, IOException {
        RemoteIterator<LocatedFileStatus> listFiles = fSystem.listFiles(new Path("/user"), true);
        while (listFiles.hasNext()) {
            LocatedFileStatus next = listFiles.next();
            System.out.println(next);
            System.out.println(next.getBlockSize());
            System.out.println(next.getPath().getName());
            BlockLocation[] blockLocations = next.getBlockLocations();
            System.out.println("開始展示block塊");
            for (BlockLocation blockLocation : blockLocations) {
                System.out.println(blockLocation);
                System.out.println("---------");
                // offset爲0是因爲是第一個block
                String[] hosts = blockLocation.getHosts();
                for (String string : hosts) {
                    System.out.println(string);
                }
            }
        }
    }
    
    // 向HDFS上傳文件
    @Test
    public void uploadByStream() throws IllegalArgumentException, IOException {
        // input 把數據讀入
        FileInputStream fis = new FileInputStream("G:/HDFSDownload/source.txt");
        // output 把輸入進來的數據寫到某個地方去
        FSDataOutputStream hdfsOutputStream = fSystem.create(new Path("/gmj/source.txt"),true);
        IOUtils.copyBytes(fis, hdfsOutputStream, 1024);
    }
    
    // 從HDFS下載文件或文件夾
    @Test
    public void downloadByStream() throws IllegalArgumentException, IOException {
        FSDataInputStream inputStream = fSystem.open(new Path("/gmj/source.txt"));
        FileOutputStream fos = new FileOutputStream("C:/Users/Administrator/Desktop/output/source2.txt");
        org.apache.commons.io.IOUtils.copy(inputStream, fos);
    }
    
    @Test
    public void downloadByStreamUseSeek() throws IllegalArgumentException, IOException {
        FSDataInputStream inputStream = fSystem.open(new Path("/user/jdk-8u131-windows-x64.exe"));
        inputStream.seek(1024*1024*128);
        FileOutputStream fos = new FileOutputStream("C:/Users/Administrator/Desktop/output/outseek1.exe");
        org.apache.commons.io.IOUtils.copy(inputStream, fos);
    }
    
    @Test
    public void testListAll() throws FileNotFoundException, IllegalArgumentException, IOException {
        FileStatus[] listStatus = fSystem.listStatus(new Path("/"));
        for (FileStatus fileStatus : listStatus) {
            System.out.println(fileStatus.getPath().getName());
        }
    }
    
    // 讀取一個文件大小介於129M~256M，切塊中的第二個block塊的內容
    @Test
    public void readSecondBlock() throws FileNotFoundException, IllegalArgumentException, IOException {
        FSDataInputStream inputStream = fSystem.open(new Path("/user/jdk-8u131-windows-x64.exe"));
        RemoteIterator<LocatedFileStatus> listFiles = fSystem.listFiles(new Path("/user/jdk-8u131-windows-x64.exe"), false);
        
        LocatedFileStatus next = listFiles.next();
        BlockLocation[] blockLocations = next.getBlockLocations();
        for (int j = 0; j < blockLocations.length; j++) {
            System.out.println(blockLocations[j].getOffset());
            if (j == 1) {
                inputStream.seek(blockLocations[j].getOffset());
                break;
            }
        }
        
        FileOutputStream fos = new FileOutputStream("C:/Users/Administrator/Desktop/output/outseek3.exe");
        org.apache.commons.io.IOUtils.copy(inputStream, fos);
    }
    
    // 讀取一個文件大於256M且切塊中的第二個block塊的內容
    @Test
    public void readSecondBlockAgain() throws FileNotFoundException, IllegalArgumentException, IOException {
        FSDataInputStream inputStream = fSystem.open(new Path("/user/block.rar"));
        RemoteIterator<LocatedFileStatus> listFiles = fSystem.listFiles(new Path("/user/block.rar"), false);

        LocatedFileStatus next = listFiles.next();
        BlockLocation[] blockLocations = next.getBlockLocations();
        for (int j = 0; j < blockLocations.length; j++) {
            System.out.println(blockLocations[j].getOffset());
            if (j == 1) {
                long length = blockLocations[j].getLength();
                FileOutputStream fos = new FileOutputStream("C:/Users/Administrator/Desktop/output/bigblock2.rar");
                org.apache.commons.io.IOUtils.copyLarge(inputStream, fos, blockLocations[j].getOffset(), length);
                break;
            }
        }
    }
    
    // 將一個超過兩個block塊的大文件以每一個文件如block0，block1， 下載下來
    @Test
    public void readSecondBlockAgainToo() throws FileNotFoundException, IllegalArgumentException, IOException {
        
        RemoteIterator<LocatedFileStatus> listFiles = fSystem.listFiles(new Path("/user/block.rar"), false);

        LocatedFileStatus next = listFiles.next();
        BlockLocation[] blockLocations = next.getBlockLocations();
        for (int j = 0; j < blockLocations.length; j++) {
            FSDataInputStream inputStream = fSystem.open(new Path("/user/block.rar"));
            System.out.println(blockLocations[j].getOffset());
            System.out.println(blockLocations.length);
            FileOutputStream fos = new FileOutputStream("C:/Users/Administrator/Desktop/output/bigdata" + j + ".rar");
            org.apache.commons.io.IOUtils.copyLarge(inputStream, fos, blockLocations[j].getOffset(), blockLocations[j].getLength());
            
        }
    }   
}