Java實現對Hadoop HDFS的API操作

1.配置Hadoop的Windows客戶端

Hadoop 配置Windows 客戶端

2.新建Maven項目[略]

3.添加依賴

        <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-client -->
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>3.3.6</version>
        </dependency>

        <!-- https://mvnrepository.com/artifact/junit/junit -->
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.13.2</version>
        </dependency>

        <!-- https://mvnrepository.com/artifact/org.slf4j/slf4j-reload4j -->
        <dependency>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-reload4j</artifactId>
            <version>2.0.9</version>
        </dependency>

4.在項目的 src/main/resources目錄下,新建日誌配置文件,文件名爲“log4j.properties”

log4j.rootLogger=INFO, stdout
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d %p [%c] - %m%n
log4j.appender.logfile=org.apache.log4j.FileAppender
log4j.appender.logfile.File=target/spring.log
log4j.appender.logfile.layout=org.apache.log4j.PatternLayout
log4j.appender.logfile.layout.ConversionPattern=%d %p [%c] - %m%n

5.新建測試類,用於在Hadoop中創建一個文件夾

package cn.coreqi.hdfs;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

/**
 * 客戶端代碼步驟
 * 1.首先獲取一個客戶端對象
 * 2.執行相關的操作命令
 * 3.關閉資源
 */
public class HdfsClient {

    /**
     * 客戶端對象
     */
    private FileSystem fs;

    /**
     * 初始化客戶端對象
     * @throws IOException
     * @throws InterruptedException
     * @throws URISyntaxException
     */
    @Before
    public void init() throws IOException, InterruptedException, URISyntaxException {
        // 需要連接的Hadoop NameNode地址
        URI uri = new URI("hdfs://192.168.58.130:8020");
        // 創建一個配置文件
        Configuration configuration = new Configuration();
        // 用戶
        String user = "root";
        // 獲取客戶端對象
        fs = FileSystem.get(uri, configuration,user);
    }

    /**
     * 釋放客戶端資源
     * @throws IOException
     */
    @After
    public void close() throws IOException {
        // 關閉資源
        fs.close();
    }

    @Test
    public void testMkdirs() throws IOException{
        // 創建一個文件夾
        fs.mkdirs(new Path("/coreqi"));
    }
}
發表評論
所有評論
還沒有人評論,想成為第一個評論的人麼? 請在上方評論欄輸入並且點擊發布.
相關文章