Java API實現Hadoop HDFS操作

一，依賴

<dependency>
    <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-common</artifactId>
    <version>2.7.3</version>
</dependency>
<dependency>
    <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-hdfs</artifactId>
    <version>2.7.3</version>
</dependency>
<dependency>
    <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-client</artifactId>
    <version>2.7.3</version>
</dependency>

二，Java API

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;
import java.io.*;
import java.util.ArrayList;
import java.util.List;

/**
 * HDFS工具類
 */
public class HDFSUtil {

    /**
     * 創建文件夾
     * @param conf
     * @param path
     * @throws IOException
     */
    public static void mkdir(Configuration conf, String path) throws IOException {
        FileSystem fs = FileSystem.get(conf);
        fs.mkdirs(new Path(path));
        fs.close();
    }

    /**
     * 刪除
     * @param conf
     * @param path
     * @throws IOException
     */
    public static void delete(Configuration conf, String path) throws IOException {
        FileSystem fs = FileSystem.get(conf);
        fs.delete(new Path(path), true); // 爲true表示目標路徑爲文件夾，則遞歸刪除
        fs.close();
    }

    /**
     * 查看文件列表
     * listFiles查看的是文件，不包括文件夾。支持遞歸
     * @param conf
     * @param path
     * @return
     * @throws IOException
     */
    public static List<LocatedFileStatus> fileList(Configuration conf, String path) throws IOException {
        FileSystem fs = FileSystem.get(conf);
        RemoteIterator<LocatedFileStatus> iterator =  fs.listFiles(new Path(path), true); // true爲遞歸
        List<LocatedFileStatus> resultList = new ArrayList<>();
        while (iterator.hasNext()) {
            LocatedFileStatus locatedFileStatus  = iterator.next();
            resultList.add(locatedFileStatus);
        }
        fs.close();
        return resultList;
    }

    /**
     * 查看文件列表
     * listStatus查看的是文件夾和文件，不支持遞歸。需要自己實現遞歸。
     * @param conf
     * @param path
     * @return
     * @throws IOException
     */
    public static List<FileStatus> fileList2(Configuration conf, String path, List<FileStatus> resultList) throws IOException {
        FileSystem fs = FileSystem.get(conf);
        FileStatus[] fileStatus =  fs.listStatus(new Path(path));
        if (resultList == null) {
            resultList = new ArrayList<>();
        }
        for(FileStatus s : fileStatus) {
            // 遞歸查找子目錄文件
            if (s.isDirectory()) {
                //System.out.println("目錄："  + s.getPath());
                fileList2(conf, s.getPath().toUri().getPath(), resultList);
            } else {
                //System.out.println("文件：" + s.getPath());
                resultList.add(s);
            }
        }
        fs.close();
        return resultList;
    }

    /**
     * 上傳
     * @param conf
     * @param filePath
     * @param hdfsPath
     * @throws IOException
     */
    public static void upload(Configuration conf, String filePath, String hdfsPath) throws IOException {
        FileSystem fs = FileSystem.get(conf);
        /*InputStream input = new FileInputStream(filePath);
        OutputStream out = fs.create(new Path(hdfsPath));
        IOUtils.copyBytes(input, out, 1024);*/
        fs.copyFromLocalFile(new Path(filePath), new Path(hdfsPath));
    }

    /**
     * 下載
     * @param conf
     * @param hdfsPath
     * @param savePath
     * @throws IOException
     */
    public static void download(Configuration conf, String hdfsPath, String savePath) throws IOException {
        FileSystem fs = FileSystem.get(conf);
        /*InputStream input = fs.open(new Path(hdfsPath));
        OutputStream out = new FileOutputStream(savePath);
        IOUtils.copyBytes(input, out, 1024);*/
        fs.copyToLocalFile(new Path(hdfsPath), new Path(savePath));
    }

    public static void main(String[] args) throws IOException {
        // 建立客戶端
        System.setProperty("HADOOP_USER_NAME","root");
        //System.setProperty(" hadoop.home.dir","D:\\hadoop-2.7.3");

        // 設置參數，指定NameNode地址
        Configuration conf = new Configuration();
        conf.set("fs.defaultFS","hdfs://10.1.255.121:9000");
        // 設置塊大小
        conf.set("dfs.blocksize","256m");
        // 設置副本數
        conf.set("dfs.replication","2");

        //mkdir(conf, "/abc12323");

        //delete(conf, "/test");

        //upload(conf, "E:/jar/fastjson-1.2.7.jar", "/abc123/fastjson-1.2.7.jar");

        //download(conf, "/2020/a.txt", "E:/test/a.txt");

        /*List<LocatedFileStatus> list = fileList(conf, "/");
        for (LocatedFileStatus locatedFileStatus: list) {
            System.out.println(locatedFileStatus.getPath());
        }*/

       /* System.out.println("----------------");
        List<FileStatus> list2 = fileList2(conf, "/", null);
        for(FileStatus s : list2) {
            System.out.println(s.getPath());
        }*/


     }
}