一.文件操作
1.上傳本地文件到hadood
2.在hadoop中新建文件，並寫入
3.刪除hadoop上的文件
4.讀取文件
5.文件修改時間
二.目錄操作
1.在hadoop上創建目錄
2.刪除目錄
3.讀取某個目錄下的所有文件
三.hdfs信息
1.查找某個文件在HDFS集羣中位置
2.獲取HDFS集羣上所有名稱節點信息
一.文件操作1.上傳本地文件到hadoodpackage com.hdfs.file;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
public class CopyFileToHDFS {
/**
* @param args
* @throws IOException
*/
public static void main(String[] args) throws IOException {
// TODO Auto-generated method stub
//copyFromLocal
//step1
Configuration conf=new Configuration();
//step2
FileSystem fs=FileSystem.get(conf);
//step3
fs.copyFromLocalFile(new Path(args[0]), new Path(args[1]));
fs.close();
}
}
把當前java文件打包成copyFileToHdfs.jar包，（注意，接下來的所有代碼都先需要打包）
通過命令:hadoop jar copyFileToHdfs.jar路徑 輸入文件路徑 輸出目錄路徑
2.在hadoop中新建文件，並寫入package com.hdfs.file;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
public class CreateNewFileAndWrite {
/**
* @param args
* @throws IOException
*/
public static void main(String[] args) throws IOException {
// TODO Auto-generated method stub
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(conf);
FSDataOutputStream fsOutput = fs.create(new Path(args[0]));
fsOutput.write(args[1].getBytes("UTF-8"));
fsOutput.close();
fs.close();
}
}
通過命令:hadoop jar createNFAW.jar路徑 創建的文件路徑(是hdfs中的文件路徑) 要寫入的內容。
3.刪除hadoop上的文件package com.hdfs.file;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
public class DeleteFile {
/**
* @param args
* @throws IOException
*/
public static void main(String[] args) throws IOException {
// TODO Auto-generated method stub
Configuration conf=new Configuration();
FileSystem fs=FileSystem.get(conf);
fs.delete(new Path(args[0]));
fs.close();
}
}
通過命令:hadoop jar deleteFile.jar路徑 需要刪除的文件路徑。
4.讀取文件package com.hdfs.file;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
public class ReadFile {
/**
* @param args
* @throws Exception
*/
public static void main(String[] args) throws Exception {
// TODO Auto-generated method stub
Configuration conf=new Configuration();
FileSystem fs=FileSystem.get(conf);
Path path=new Path(args[0]);
if(fs.exists(path)){
FSDataInputStream fsIn=fs.open(path);
FileStatus status=fs.getFileStatus(path);
byte[] buffer=new byte[Integer.parseInt(String.valueOf(status.getLen()))];
fsIn.readFully(0,buffer);
fsIn.close();
fs.close();
System.out.println("讀取完成!");
System.out.println(new String(buffer));
}else{
throw new Exception("the file is not found!");
}
}
}
通過命令:hadoop jar readFile.jar路徑 需要讀取的文件路徑。
5.查看文件修改時間package com.hdfs.file;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
public class ModifyFileTime {
/**
* @param args
* @throws IOException
*/
public static void main(String[] args) throws IOException {
// TODO Auto-generated method stub
Configuration conf=new Configuration();
FileSystem fs=FileSystem.get(conf);
Path path=new Path(args[0]);
FileStatus status=fs.getFileStatus(path);
System.out.println("時間:"+status.getAccessTime());
fs.close();
}
}
通過命令:hadoop jar modifyFileTime.jar路徑 需要查看的文件路徑
二.目錄操作1.在hadoop上創建目錄package com.hdfs.directory;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
public class CreateDir {
/**
* @param args
* @throws IOException
*/
public static void main(String[] args) throws IOException {
// TODO Auto-generated method stub
Configuration conf=new Configuration();
FileSystem fs=FileSystem.get(conf);
fs.mkdirs(new Path("/user/long1657/20130908"));
fs.close();
// FileStatus fileStatus=fs.getFileStatus(new Path("/user/long1657/20130908"));
}
}
通過命令:hadoop jar createDir.jar路徑
2.刪除目錄package com.hdfs.directory;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
public class DeleteDir {
/**
* @param args
* @throws IOException
*/
public static void main(String[] args) throws IOException {
// TODO Auto-generated method stub
Configuration conf=new Configuration();
FileSystem fs=FileSystem.get(conf);
fs.deleteOnExit(new Path(args[0]));
fs.close();
}
}
通過命令:hadoop jar deleteDir.jar路徑 需要刪除的目錄路徑
3.讀取某個目錄下的所有文件package com.hdfs.directory;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
public class ReadAllFileInDir {
/**
* @param args
* @throws IOException
*/
public static void main(String[] args) throws IOException {
// TODO Auto-generated method stub
Configuration conf=new Configuration();
FileSystem fs=FileSystem.get(conf);
Path path=new Path(args[0]);
FileStatus[] status=fs.listStatus(path);
for(int i=0;i<status.length;i++){
System.out.println(status[i].getPath().toString());
if(status[i].isDir()){
FileStatus[] ss=fs.listStatus(status[i].getPath());
for(int j=0;j<ss.length;j++){
System.out.println(ss[j].getPath().toString());
}
}
}
fs.close();
}
}
通過命令:hadoop jar deleteDir.jar路徑 需要讀取的目錄路徑
三.hdfs信息1.查找某個文件在HDFS集羣中位置public static void getFileLocal(String pathStr) throws IOException{
Configuration conf=new Configuration();
FileSystem fs=FileSystem.get(conf);
Path path=new Path(pathStr);
//獲取文件系統裏面的文件信息
FileStatus fileStatus=fs.getFileStatus(path);
//獲取文件的塊信息
BlockLocation[] blocks=fs.getFileBlockLocations(fileStatus, 0, fileStatus.getBlockSize());
//循環打印信息
int blockSize=blocks.length;
for(int i=0;i<blockSize;i++){
String[] hosts=blocks[i].getHosts();
System.out.println("block_"+i+"_location:"+hosts[0]);
}
fs.close();
}2.獲取HDFS集羣上所有名稱節點信息
/**
* 集羣節點數
* */
public static void getHDFSNodes()throws IOException{
Configuration conf=new Configuration();
FileSystem fs=FileSystem.get(conf);
DistributedFileSystem df=(DistributedFileSystem)fs;
DatanodeInfo[] dataNode=df.getDataNodeStats();
//循環答應節點信息
for(int i=0;i<dataNode.length;i++){
System.out.println("DataNode_"+i+"_Name:"+dataNode[i].getHostName());
}
fs.close();
}
