大數據實戰04-Java操作hadoop的常用的API

在前面的文章中介紹了在windows環境中開發hadoop的入門程序，並實現了在windows環境中連接虛擬機來模擬真實的遠程連接的情況，接下來通過同樣的連接方式來講解hadoop的常用api的使用的方式

hadoop的使用方式

package com.kkcl.hadoop;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.Progressable;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.net.URI;


public class API{

    public static final String HDFS_PATH = "hdfs://192.168.93.10:9000";
    FileSystem  fileSystem = null;
    Configuration configuration = null;

    @Before
    public void setUp() throws Exception{
        System.out.println("-------setUp--------");
        configuration = new Configuration();
        fileSystem = FileSystem.get(new URI(HDFS_PATH),configuration,"hadoop");
    }

    /**
     創建文件夾
     **/
    @Test
    public void mkdir() throws Exception{
        fileSystem.mkdirs(new Path("/hdfsapi/output"));
    }

    //查看hdfs內容
    @Test
    public void text() throws Exception{
        FSDataInputStream in = fileSystem.open(new Path("/cdh_version.properties"));
        IOUtils.copyBytes(in, System.out, 1024);
    }

    //創建文件
    @Test
    public void create() throws Exception{
        FSDataOutputStream out = fileSystem.create(new Path("/hdfsapi/test/a.txt"));
        out.writeUTF("hello world!");
        out.flush();
        out.close();
    }

    //測試文件名的更改
    @Test
    public void rename()throws Exception{
        Path oldPath = new Path("/hdfsapi/test/a.txt");
        Path newPath = new Path("/hdfsapi/test/c.txt");
        boolean result = fileSystem.rename(oldPath,newPath);
    }

    //拷貝本地文件到hdfs文件系統
    @Test
    public void copyFromLocalFile()throws Exception{
        Path src = new Path("C:/Users/CHENG/Desktop/hello.txt");//本地文件的目錄
        Path dist = new Path("/hdfsapi/test/");
        fileSystem.copyFromLocalFile(src,dist);
    }

    //拷貝大文件帶進度條
    public void copyFromBigLocalFile()throws Exception{
        InputStream in = new BufferedInputStream(new FileInputStream(new File("/Users/xxx")));
        FSDataOutputStream out = fileSystem.create(new Path("/hdfsapi/test"), new Progressable() {
            @Override
            public void progress() {
                System.out.println(".");
            }
        });

        IOUtils.copyBytes(in,out,4096);
    }

    //將HDFS文件拷貝到本地：下載
    @Test
    public void copyToLocalFile() throws Exception{
        Path src = new Path("/hdfsapi/text/xxx.txt");
        Path dist = new Path("/Users/xss");
        fileSystem.copyToLocalFile(src,dist);
    }

    //查看文件夾下的所有的文件
    @Test
    public void  listFiles() throws Exception{
        FileStatus [] statuses = fileSystem.listStatus(new Path("/"));
        for(FileStatus f : statuses){
            String isDir = f.isDirectory() ? "文件夾" : "文件";
            String permission = f.getPermission().toString();
            short replication = f.getReplication();
            long length= f.getLen();
            String path = f.getPath().toString();
            System.out.println(isDir+"\t"+permission+"\t"+replication+"\t"+length+"\t"+path+"\t");
        }
    }

    //查看文件快的信息
    @Test
    public void getFileBlockLocations() throws Exception{
        FileStatus fileStatus = fileSystem.getFileStatus(new Path("/hdfsapi/test/xxx"));
        BlockLocation [] blockLocations = fileSystem.getFileBlockLocations(fileStatus,0,fileStatus.getLen());
        for(BlockLocation block : blockLocations){
            for(String name : block.getNames()){
                System.out.println(name + ":" + block.getOffset() + ":" + block.getLength() +":"+ block.getHosts());
            }
        }
    }

    //刪除文件
    @Test
    public void delete() throws Exception{
        boolean result = fileSystem.delete(new Path("/hdfsapi"));
        System.out.println(result);
    }

    @After
    public void tearDown(){
        configuration = null;
        fileSystem = null;
        System.out.println("---------tearDown----------");
    }
}

開發過程中可能遇到的問題

Exception in thread "main" java.lang.UnsatisfiedLinkError: org.apache.hadoop.io.nativeio.NativeIO$Windows.createDirectoryWithMode0(Ljava/lang/String;I)V
	at org.apache.hadoop.io.nativeio.NativeIO$Windows.createDirectoryWithMode0(Native Method)
	at org.apache.hadoop.io.nativeio.NativeIO$Windows.createDirectoryWithMode(NativeIO.java:299)
	at org.apache.hadoop.fs.RawLocalFileSystem.mkOneDirWithMode(RawLocalFileSystem.java:465)
	at org.apache.hadoop.fs.RawLocalFileSystem.mkdirsWithOptionalPermission(RawLocalFileSystem.java:518)
	at org.apache.hadoop.fs.RawLocalFileSystem.mkdirs(RawLocalFileSystem.java:496)
	at org.apache.hadoop.fs.FilterFileSystem.mkdirs(FilterFileSystem.java:316)
	at org.apache.hadoop.mapreduce.JobSubmissionFiles.getStagingDir(JobSubmissionFiles.java:133)
	at org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:148)
	at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1307)
	at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1304)
	at java.security.AccessController.doPrivileged(Native Method)
	at javax.security.auth.Subject.doAs(Subject.java:422)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1924)
	at org.apache.hadoop.mapreduce.Job.submit(Job.java:1304)
	at org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:1325)
	at com.kkcl.mr.wc.WordCountApp.main(WordCountApp.java:40)

解決的方式

• 第一步下載對應hadoop版本的winutils並配置環境變量
• 刪除掉第一步中安裝的軟件的bin下的hadoop.dll(或者電腦上的所有的hadoop.dll)
• 重寫org.apache.hadoop.io.nativeio.NativeIO該類並修改部分源代碼

public static class Windows {
        public static final long GENERIC_READ = 2147483648L;
        public static final long GENERIC_WRITE = 1073741824L;
        public static final long FILE_SHARE_READ = 1L;
        public static final long FILE_SHARE_WRITE = 2L;
        public static final long FILE_SHARE_DELETE = 4L;
        public static final long CREATE_NEW = 1L;
        public static final long CREATE_ALWAYS = 2L;
        public static final long OPEN_EXISTING = 3L;
        public static final long OPEN_ALWAYS = 4L;
        public static final long TRUNCATE_EXISTING = 5L;
        public static final long FILE_BEGIN = 0L;
        public static final long FILE_CURRENT = 1L;
        public static final long FILE_END = 2L;
        public static final long FILE_ATTRIBUTE_NORMAL = 128L;

        public Windows() {
        }

        public static void createDirectoryWithMode(File path, int mode) throws IOException {
            createDirectoryWithMode0(path.getAbsolutePath(), mode);
        }

        private static native void createDirectoryWithMode0(String var0, int var1) throws NativeIOException;

        public static native FileDescriptor createFile(String var0, long var1, long var3, long var5) throws IOException;

        public static FileOutputStream createFileOutputStreamWithMode(File path, boolean append, int mode) throws IOException {
            long desiredAccess = 1073741824L;
            long shareMode = 3L;
            long creationDisposition = append ? 4L : 2L;
            return new FileOutputStream(createFileWithMode0(path.getAbsolutePath(), desiredAccess, shareMode, creationDisposition, mode));
        }

        private static native FileDescriptor createFileWithMode0(String var0, long var1, long var3, long var5, int var7) throws NativeIOException;

        public static native long setFilePointer(FileDescriptor var0, long var1, long var3) throws IOException;

        private static native String getOwner(FileDescriptor var0) throws IOException;

        private static native boolean access0(String var0, int var1);

        public static boolean access(String path, AccessRight desiredAccess) throws IOException {
            //return access0(path, desiredAccess.accessRight());
            return true;    //修改這裏
    }