hadoop學習5-HDFS API學習

複習上一次課總結

1）HDFS讀過程

DistributedFileSystem =>FSDataInputStream =>DFSClient.open（RPC通信機制)=>NN.open

2)HDFS寫過程

DistributeFileSystem => FSDataOutputStream => DFSClient.create（RPC通信機制)=>NN.create

3)SecondaryNamenode的作用與機制

SNN不是完全意義上的NN的備份

拉取Fsimage和edits文件的SNN的內存中進行合併

fs.checkpoint.period

fs.checkpoint.size

fs.checkpoint.dir

hadoop2.x以及hadoop0.23後

checkpoint node 與secondaryNamenode完全一樣

backup node 完全意義上的namenode備份

4）一旦丟失NN或者元數據信息，可以通過SNN檢查點目錄恢復元數據信息

hadoop namenode -importCheckpoint  導入檢查點信息

hadoop-daemon.sh start namenode  啓動namenode

5)機架感知

默認情況下所有DN被認爲是出於同一個機架上，不管是否物理上屬於同一個機架

/default-rack

topology.script.file.name屬性值是一個腳本文件可以是python或shell，這個腳本里面寫的是真正意義上的網絡拓撲結構圖

/d1/rack1/dn1 

HDFS API

地址:hadoop.apache.org/docs/current1/api

下面通過一些testcase來展示HDFS API部分代碼

public static String hdfsUrl = "hdfs://192.168.1.201:9100";
	
	//create HDFS folder
	@Test
	public void testHDFSMKdir() throws IOException{
		Configuration conf = new Configuration();
		FileSystem fs = FileSystem.get(URI.create(hdfsUrl),conf);
		Path path = new Path("/test");
		fs.mkdirs(path);		
	}
	
	//create HDFS file
	@Test
	public void testCreateFile() throws IOException{
		Configuration conf = new Configuration();
		FileSystem fs = FileSystem.get(URI.create(hdfsUrl), conf);
		Path path = new Path("/test/a.txt");
		FSDataOutputStream out = fs.create(path);
		out.write("hello hadoop!".getBytes());
	}
	
	//rename a file name
	@Test
	public void testRenameFile() throws IOException{
		Configuration conf = new Configuration();
		FileSystem fs = FileSystem.get(URI.create(hdfsUrl), conf);
		Path path = new Path("/test/a.txt");
		Path newPath = new Path("/test/b.txt");
		System.out.print(fs.rename(path, newPath));
	}
	
	//upload a local file
	@Test
	public void testUploadLocalFile1() throws IOException{
		Configuration conf = new Configuration();
		FileSystem fs =FileSystem.get(URI.create(hdfsUrl), conf);
		Path src = new Path("/home/hadoop-1.2.1/bin/rcc");
		Path dst = new Path("/test");
		fs.copyFromLocalFile(src, dst);
	}
	
	//other way upload a local file
		@Test
		public void testUploadLocalFile2() throws IOException{
			Configuration conf = new Configuration();
			FileSystem fs =FileSystem.get(URI.create(hdfsUrl), conf);
			InputStream in = new BufferedInputStream(new FileInputStream(new File("/home/hadoop-1.2.1/bin/rcc")));
			FSDataOutputStream out = fs.create(new Path("/test/rcc"));
			IOUtils.copyBytes(in, out, 1024);
		}
	
	//other way upload a local file
	@Test
	public void testUploadLocalFile3() throws IOException{
		Configuration conf = new Configuration();
		FileSystem fs =FileSystem.get(URI.create(hdfsUrl), conf);
		InputStream in = new BufferedInputStream(new FileInputStream(new File("/home/hadoop-1.2.1/bin/rcc")));
		FSDataOutputStream out = fs.create(new Path("/test/data"),new Progressable(){
			
			@Override
			public void progress(){
				System.out.println(".");
			}
		});
		IOUtils.copyBytes(in, out, 1024);
	}
	
	//List files under folder
	@Test
	public void testListFiles() throws IOException{
		Configuration conf = new Configuration();
		FileSystem fs = FileSystem.get(URI.create(hdfsUrl),conf);
		Path dst = new Path("/test");
		FileStatus[] files = fs.listStatus(dst);
		for(FileStatus file:files){
			System.out.println(file.getPath().toString());
		}
	}
	
	//List block under folder
		@Test
		public void testGetBlockInfo() throws IOException{
			Configuration conf = new Configuration();
			FileSystem fs = FileSystem.get(URI.create(hdfsUrl),conf);
			Path dst = new Path("/test/data");
			FileStatus fileStatus = fs.getFileStatus(dst);
			BlockLocation[] blkLoc = fs.getFileBlockLocations(fileStatus, 0, fileStatus.getLen());
			
			for(BlockLocation loc:blkLoc){
				for(int i = 0;i<loc.getHosts().length;i++){
					System.out.println(loc.getHosts()[i]);
				}
			}
		}