HDFS JAVA API開發的思路及工具類代碼

前面剛剛介紹了hdfs的架構及運行原理，有想看的見https://blog.csdn.net/weixin_42231373/article/details/85005667

現在我們來說說如何使用java API 開發HDFS，下面是些簡單的思路 後面會有代碼

 

建項目時修改pom.xml 文件           增加Hadoop的依賴  強調一下我說的是maven項目，普通項目自行下載依賴並添加到類路徑,如果不知道的話見之前的博客，略有說明 https://blog.csdn.net/weixin_42231373/article/details/84654576

  maven的依賴爲

        <dependency>
			<groupId>org.apache.hadoop</groupId>
			<artifactId>hadoop-client</artifactId>
			<version>2.7.4</version>
			<scope>provided</scope>
		</dependency>

獲得日誌對象                                  Logger logger = Logger.get(cls);

建立hdfs配置單例                          Configrutaion conf = new Configrutaion()；

通過conf傳遞參數                          conf.set(name, value);                conf.get(name)

獲得hdfs文件系統                          FileSystem fs = FileSystem.get(conf);

獲得hdfs處理(輸入輸出流，各種操作)

   FSDataInputStream fsis = fs.open(new Path(“hdfs路徑”));

實際處理邏輯      （讀取輸入流，寫入 等）

開發完後打包上傳，測試運行

下面是操縱HDFS文件具體的代碼 以工具類的方式呈現

import java.io.IOException;

import org.apache.commons.io.output.ByteArrayOutputStream;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.log4j.Logger;

public class HdfsFileOperatorUtil {
	Logger loger = Logger.getLogger(HdfsFileOperatorUtil.class);
	static Configuration hadoopConf = new Configuration();

	public static byte[] readFileToByteArray(String filePath) throws Exception {
		byte[] result = null;
		if (filePath == null || filePath.trim().length() == 0) {
			throw new Exception("文件路徑不對" + filePath + "，請檢查");
		}
		FSDataInputStream hdfsIS = null;
		ByteArrayOutputStream baos = null;
		try {
			FileSystem fs = FileSystem.get(hadoopConf);
			Path hdfsPath = new Path(filePath);
			hdfsIS = fs.open(hdfsPath);
			byte[] b = new byte[65536];
			baos = new ByteArrayOutputStream();
			int flag = -1;
			while ((flag = hdfsIS.read(b)) != -1) {
				baos.write(b);
				// baos.write(flag);//不成功
				b = new byte[65536];
			}
			result = baos.toByteArray();
		} catch (Exception e) {
			e.printStackTrace();
		} finally {
			CloseUtil.close(hdfsIS);
			CloseUtil.close(baos);
		}

		return result;
	}

	public static String readFileToString(String filePath) throws Exception {
		String result = null;
		result = new String(readFileToByteArray(filePath), "utf-8");
		return result;
	}

	public static boolean writeFile(byte[] content, String toFilePath)
			throws Exception {
		boolean isFinish = false;
		if (toFilePath == null || toFilePath.trim().length() == 0) {
			throw new Exception("文件路徑不對" + toFilePath + "，請檢查");
		}
		FSDataOutputStream hdfsOS = null;
		try {
			FileSystem fs = FileSystem.get(hadoopConf);
			Path hdfsPath = new Path(toFilePath);
			hdfsOS = fs.create(hdfsPath);
			hdfsOS.write(content);
			isFinish = true;
		} catch (Exception e) {
			isFinish = false;
			e.printStackTrace();
		} finally {
			CloseUtil.close(hdfsOS);
		}
		return isFinish;
	}

	/**
	 * 查看某個目錄下的所有文件
	 * 
	 * @throws IOException
	 *
	 */

	public static String readFilesListToString(String filesPath)
			throws IOException {
		StringBuffer sb = new StringBuffer();
		FileSystem fs = FileSystem.get(hadoopConf);
		FileStatus[] fileStatuses = fs.listStatus(new Path(filesPath));
		for (FileStatus fileStatus : fileStatuses) {
			String path = fileStatus.getPath().toString();
			sb.append(path + " ");
		}

		return sb.toString();
	}

}

集羣本地文件操作的工具類

package cn.tl.util;

import java.io.BufferedInputStream;
import java.io.FileInputStream;

import org.apache.commons.io.output.ByteArrayOutputStream;

public class LocalFileOperatorUtil {
	public static byte[] readFileToByte(String filePath) throws Exception{
		byte[] result = null;
		if (filePath == null || filePath.trim().length() == 0) {
			throw new Exception("文件路徑不對" + filePath + "，請檢查");
		}
		BufferedInputStream bis = null;
		ByteArrayOutputStream baos = null;
		try {
			bis = new BufferedInputStream(new FileInputStream(filePath));
			baos = new ByteArrayOutputStream();
			int reads = -1;
			while((reads = bis.read()) != -1){
				baos.write(reads);
			}
			result = baos.toByteArray();
		} catch (Exception e) {
			e.printStackTrace();
		} finally{
			CloseUtil.close(baos);
			CloseUtil.close(bis);
		}
		
		return result;
	}

	public static String readFileToString(String filePath) throws Exception{
		String result = null;
		if (filePath == null || filePath.trim().length() == 0) {
			throw new Exception("文件路徑不對" + filePath + "，請檢查");
		}
		result = new String(readFileToByte(filePath), "utf-8");
		return result;
	}

}

附CloseUtil工具類

package cn.tl.util;

public class CloseUtil {

	public static void close(AutoCloseable obj) {
		if (obj != null) {
			try {
				obj.close();
			} catch (Exception e) {
				e.printStackTrace();
			}
		}
	}
}

測試類

package cn.tl.dataCirculation;

import java.io.File;

import cn.tl.util.HdfsFileOperatorUtil;
import cn.tl.util.LocalFileOperatorUtil;

public class readHtml {

	public static void main(String[] args) throws Exception {
		String filePath = File.separator + "home" + File.separator + "zhangxin"
				+ File.separator + "index.html";// 本地路徑
		String toFilePath = File.separator + "user" + File.separator
				+ "zhangxin" + File.separator + "index.html";// hdfs路徑
		System.out.println("reading.....");
		byte[] readToByte = LocalFileOperatorUtil.readFileToByte(filePath);
		System.out.println("readEnd,writing....");
		boolean flag = HdfsFileOperatorUtil.writeFile(readToByte, toFilePath);
		String result = HdfsFileOperatorUtil.readFileToString(toFilePath);
		System.out.println(result);
		System.out.println("done");
		System.out.println(flag);
	}

}