HDFSAPI代碼

    這裏主要是與hdfs系統的交互，包括上傳、下載文件到hdfs系統中，代碼如下：

import java.io.IOException;  
import java.net.URI;  
import java.net.URISyntaxException;  
  
import org.apache.hadoop.conf.Configuration;  
import org.apache.hadoop.fs.FSDataInputStream;  
import org.apache.hadoop.fs.FileStatus;  
import org.apache.hadoop.fs.FileSystem;  
import org.apache.hadoop.fs.Path;  
import org.apache.hadoop.io.IOUtils;  
  
public class hdfsGYT {  
      
    private static  final String HDFS = "hdfs://127.0.0.1:9000/";  
  
    public hdfsGYT(String hdfs,  Configuration conf ){  
        this.hdfsPath = hdfs;  
        this.conf = conf;  
    }  
  
    public hdfsGYT() {  
        // TODO Auto-generated constructor stub  
    }  
  
    private String hdfsPath;  
    private Configuration conf = new Configuration() ;  
      
    public static void main(String[] args) throws IOException, URISyntaxException{  
        hdfsGYT hdfsgyt = new hdfsGYT();  
        String folder = HDFS + "mr/groom_system/small2.csv";  
        String local = "/home/thinkgamer/Java/hadoop_shizhan/src/user_thing_tuijian/small2.csv";  
        String local1 = "/home/thinkgamer/Java/hadoop_shizhan/src/user_thing_tuijian";  
        //判斷某個文件夾是否存在  
        //hdfsgyt.isExist(folder);  
        //創建文件夾  
        //hdfsgyt.mkdir(folder);  
        //刪除文件夾  
        //hdfsgyt.rmr(folder);  
        //列出所有文件夾  
        //hdfsgyt.ls(folder);  
        //遞歸列出所有文件夾  
        //hdfsgyt.lsr(folder);  
        //上傳文件  
        //hdfsgyt.put(local, folder);  
        //下載文件  
        //hdfsgyt.get(folder,local1);  
        //刪除文件  
        //hdfsgyt.rm(folder);  
        //顯示文件  
        //hdfsgyt.cat(folder);  
    }  
  
    //顯示文件  
    private void cat(String folder) throws IOException, URISyntaxException {  
        // 與hdfs建立聯繫  
        FileSystem fs = FileSystem.get(new URI(HDFS),new Configuration());  
        Path path = new Path(folder);  
         FSDataInputStream fsdis = null;  
            System.out.println("cat: " + folder);  
            try {    
                fsdis =fs.open(path);  
                IOUtils.copyBytes(fsdis, System.out, 4096, false);    
              } finally {    
                IOUtils.closeStream(fsdis);  
                fs.close();  
              }  
    }  
  
    //刪除文件  
    private void rm(String folder) throws IOException, URISyntaxException {  
        //與hdfs建立聯繫  
        FileSystem fs = FileSystem.get(new URI(HDFS),new Configuration());  
        Path path = new Path(folder);  
        if(fs.deleteOnExit(path)){  
            fs.delete(path);  
            System.out.println("delete:" + folder);  
        }else{  
            System.out.println("The fiel is not exist!");  
        }  
        fs.close();  
    }  
      
    //下載文件  
    private void get(String remote,  String local) throws IllegalArgumentException, IOException, URISyntaxException {  
        // 建立聯繫  
        FileSystem fs = FileSystem.get(new URI(HDFS), new Configuration());  
        fs.copyToLocalFile(new Path(remote), new Path(local));  
        System.out.println("Get From :   " + remote  + "   To :" + local);  
        fs.close();  
    }  
      
    //上傳文件  
    private void put(String local, String remote) throws IOException, URISyntaxException {  
        // 建立聯繫  
        FileSystem fs = FileSystem.get(new URI(HDFS), new Configuration());  
        fs.copyFromLocalFile(new Path(local), new Path(remote));  
        System.out.println("Put :" + local  + "   To : " + remote);  
        fs.close();  
    }  
  
    //遞歸列出所有文件夾  
    private void lsr(String folder) throws IOException, URISyntaxException {  
        //與hdfs建立聯繫  
        FileSystem fs = FileSystem.get(new URI(HDFS),new Configuration());  
        Path path = new Path(folder);  
        //得到該目錄下的所有文件  
        FileStatus[] fileList = fs.listStatus(path);  
        for (FileStatus f : fileList) {  
            System.out.printf("name: %s   |   folder: %s  |   size: %d\n", f.getPath(),  f.isDir() , f.getLen());  
            try{  
                FileStatus[] fileListR = fs.listStatus(f.getPath());  
                for(FileStatus fr:fileListR){  
                    System.out.printf("name: %s   |   folder: %s  |   size: %d\n", fr.getPath(),  fr.isDir() , fr.getLen());  
                }  
            }finally{  
                continue;  
            }  
        }  
        fs.close();  
    }  
      
    //列出所有文件夾  
    private void ls(String folder) throws IOException, URISyntaxException {  
        //與hdfs建立聯繫  
        FileSystem fs = FileSystem.get(new URI(HDFS),new Configuration());  
        Path path = new Path(folder);  
        //得到該目錄下的所有文件  
        FileStatus[] fileList = fs.listStatus(path);  
        for (FileStatus f : fileList) {  
            System.out.printf("name: %s   |   folder: %s  |   size: %d\n", f.getPath(),  f.isDir() , f.getLen());  
        }  
        fs.close();  
    }  
  
    //刪除文件夾  
    private void rmr(String folder) throws IOException, URISyntaxException {  
        //與hdfs建立聯繫  
        FileSystem fs = FileSystem.get(new URI(HDFS),new Configuration());  
        Path path = new Path(folder);  
        fs.delete(path);  
        System.out.println("delete:" + folder);  
        fs.close();  
    }  
      
    //創建文件夾  
    public void mkdir(String folder) throws IOException, URISyntaxException {  
        //與hdfs建立聯繫  
        FileSystem fs = FileSystem.get(new URI(HDFS),new Configuration());  
        Path path = new Path(folder);  
        if (!fs.exists(path)) {  
            fs.mkdirs(path);  
            System.out.println("Create: " + folder);  
        }else{  
            System.out.println("it is have exist:" + folder);  
        }  
        fs.close();   
    }  
      
    //判斷某個文件夾是否存在  
    private void isExist(String folder) throws IOException, URISyntaxException {  
        //與hdfs建立聯繫  
        FileSystem fs = FileSystem.get(new URI(HDFS),new Configuration());  
        Path path = new Path(folder);  
        if(fs.exists(path)){  
            System.out.println("it is have exist:" + folder);  
        }else{  
            System.out.println("it is not exist:" + folder);  
        }     
        fs.close();  
    }  
      
}