hdfs基本操作，上傳、下載、刪除

hadoop程序中對hdfs的操作必不可少，粘貼一個工具類，分享給大家。

import java.io.File;
import java.io.FileInputStream;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

/**
 * @author hadoop
 *
 */
public class HdfsUtil {
	
	/**
	 * 如果localFilePath是一個文件夾，則目標路徑hdfsFilePath也會看成一個文件夾，然後把localFilePath下的所有文件，移動到
	 * hdfsFilePath文件夾下。如果localFilePath是一個文件，則目標路徑hdfsFilePath也會被當成一個文件路徑，就算在路徑後面加
	 * 上了顯式的“/”。
	 * @param conf
	 * @param localFilePath
	 * @param hdfsFilePath
	 * @throws Exception
	 */
	public static void upload(Configuration conf, String localFilePath, String hdfsFilePath) throws Exception
	{
		FileSystem fs = FileSystem.get(conf);
		Path localPath = new Path(localFilePath);
		Path dstPath = new Path(hdfsFilePath);
		fs.copyFromLocalFile(localPath, dstPath);
	}
	
	/**
	 * 如果hdfsFilePath上一個文件夾，則會刪除整個文件夾
	 * @param conf
	 * @param hdfsFilePath
	 * @throws Exception
	 */
	public static void delete(Configuration conf, String hdfsFilePath) throws Exception
	{
		FileSystem fs = FileSystem.get(conf);
		Path dstPath = new Path(hdfsFilePath);
		fs.delete(dstPath, true);
	}
	
	/**
	 * 從hdfs上下載文件，到本地。
	 * 如果hdfsFilePath是一個文件夾，則目標路徑localFilePath也會看成一個文件夾，然後把hdfsFilePath下的所有文件，移動到
	 * localFilePath文件夾下。如果hdfsFilePath是一個文件，則目標路徑localFilePath也會被當成一個文件路徑，就算在路徑後面加
	 * 上了顯式的“/”。
	 * @param conf
	 * @param hdfsFilePath
	 * @param localFilePath
	 * @throws Exception
	 */
	public static void download(Configuration conf, String hdfsFilePath, String localFilePath) throws Exception
	{
		FileSystem fs = FileSystem.get(conf);
		Path localPath = new Path(localFilePath);
		Path hdfsPath = new Path(hdfsFilePath);
		fs.copyToLocalFile(hdfsPath, localPath);
	}
	
	/**
	 * 把localFilePath文件內容追加到hdfsFilePath文件上
	 * @param conf
	 * @param hdfsFilePath
	 * @param localFilePath
	 * @throws Exception
	 */
	public static void append(Configuration conf, String hdfsFilePath, String localFilePath, boolean fromNewLine) throws Exception
	{
		FileSystem fs = FileSystem.get(conf);
		File localFile = new File(localFilePath);
		if(localFile.isDirectory())
		{
			throw new Exception("不支持文件夾的append操作");
		}
		Path hdfsPath = new Path(hdfsFilePath);
		FSDataOutputStream outputStream = fs.append(hdfsPath);
		if(fromNewLine)
		{
			String newLine = "\n";
			outputStream.write(newLine.getBytes());
		}
		int bufferSize = 4096;
		byte[] buffer = new byte[bufferSize];
		FileInputStream inputStream = new FileInputStream(new File(localFilePath));
		int size = -1;
		while( (size = inputStream.read(buffer)) > 0)
		{
			outputStream.write(buffer, 0, size);
		}
		inputStream.close();
		outputStream.close();
	}

/**
	 * @param hdfsFolder hdfs的父文件夾
	 * @param filePreffix 文件的前綴
	 * @throws Exception
	 */
	public static boolean isFileExist(Configuration conf, String hdfsFolder, String filePreffix) throws Exception
	{
		boolean exist = false;
		Path parentPath = new Path(hdfsFolder);
    	FileSystem fileSystem = FileSystem.get(conf);
		FileStatus[] fileStatus = fileSystem.listStatus(parentPath);
		for(int i = 0; i < fileStatus.length; i++)
		{
			String name = fileStatus[i].getPath().getName();
			if(name.startsWith(filePreffix))
			{
				exist = true;
				break;
			}
		}
		return exist;
	}
	
	/**
	 * 在hdfs上創建一個空文件
	 * @param conf
	 * @param hdfsFileName
	 */
	public static void createEmptyFile(Configuration conf, String hdfsFileName) throws Exception
	{
		Path filePath = new Path(hdfsFileName);
    	FileSystem fileSystem = FileSystem.get(conf);
    	FSDataOutputStream stream = fileSystem.create(filePath);
    	stream.write("".getBytes());
    	stream.close();
	}

	/**
	 * @param args
	 */
	public static void main(String[] args) throws Exception {
		Configuration conf = new Configuration();
		conf.set("fs.default.name", Prop.FS_DEFAULT_NAME);
		conf.set("dfs.support.append", Prop.DFS_SUPPORT_APPEND);
		String localFilePath = "hadoop/test/file1";
		String hdfsFilePath = "/user/test/file0";
		HdfsUtil.upload(conf, localFilePath, hdfsFilePath);
		//HdfsUtil.delete(conf, hdfsFilePath);
		//HdfsUtil.download(conf, hdfsFilePath, localFilePath);
		//HdfsUtil.append(conf, hdfsFilePath, localFilePath, true);
	}
}