package com.atguigu.hdfs;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.junit.Test;
import org.mortbay.util.IO;
/**通過IO流操作HDFS
* @author huangyu
*
*/
public class IOToHdfs {
@Test
// 文件的上傳 從本地通過IO流的方式上傳到HDFS服務器
public void putFileToHDFS() throws IOException, InterruptedException,
URISyntaxException {
Configuration conf = new Configuration();
// 1,獲取文件系統
FileSystem fileSystem = FileSystem.get(
new URI("hdfs://hadoop103:9000"), conf, "atguigu");
// 2,獲取輸入流
FileInputStream fsInputStream = new FileInputStream(new File(
"e:/hadoop-2.7.2.tar.gz"));
// 3,獲取輸出流
FSDataOutputStream fsDataOutputStream = fileSystem.create(new Path(
"/user/atguigu/hadoop-2.7.2.tar.gz"));
// 4,流的拷貝
try {
IOUtils.copyBytes(fsInputStream, fsDataOutputStream, conf);
} catch (Exception e) {
// TODO: handle exception
} finally {
// 5,關閉資源
IOUtils.closeStream(fsInputStream);
IOUtils.closeStream(fsDataOutputStream);
}
}
@Test
// 文件的下載
public void getFileFromHDFS() throws IOException, InterruptedException,
URISyntaxException {
Configuration conf = new Configuration();
// 1,獲取文件系統
FileSystem fileSystem = FileSystem.get(
new URI("hdfs://hadoop103:9000"), conf, "atguigu");
// 2,獲取輸入流
FSDataInputStream fsDataInputStream = fileSystem.open(new Path(
"/user/atguigu/test/a.txt"));
// 3,獲取輸出流
FileOutputStream fos = new FileOutputStream(new File("e:/huangyu.txt"));
// 5,關閉資源
try {
// 4,流的拷貝
IOUtils.copyBytes(fsDataInputStream, fos, conf);
} catch (Exception e) {
// TODO: handle exception
} finally {
IOUtils.closeStream(fsDataInputStream);
IOUtils.closeStream(fos);
}
}
@Test
//下載大文件的第一塊數據
public void getFileFromHDFSSeek1() throws IOException, InterruptedException, URISyntaxException{
Configuration conf = new Configuration();
//1,獲取文件系統
FileSystem fileSystem = FileSystem.get(new URI("hdfs://hadoop103:9000"), conf, "atguigu");
//2,獲取輸入流
FSDataInputStream fsDataInputStream = fileSystem.open(new Path("/user/atguigu/hadoop-2.7.2.tar.gz"));
//3,獲取輸出流
FileOutputStream fos = new FileOutputStream(new File("e:/1/hadoop-2.7.2.tar.gz.part1"));
try {
//4,流的拷貝
byte[] buf = new byte[1024 * 1024];
for(int i = 0;i < 128;i++){
fsDataInputStream.read(buf);
fos.write(buf);
}
} catch (Exception e) {
// TODO: handle exception
}finally{
IOUtils.closeStream(fsDataInputStream);
IOUtils.closeStream(fos);
}
}
@Test
//下載大文件的第二塊數據
public void getFileFromHDFSSeek2() throws IOException, InterruptedException, URISyntaxException{
Configuration conf = new Configuration();
//1,獲取文件系統
FileSystem fileSystem = FileSystem.get(new URI("hdfs://hadoop103:9000"), conf, "atguigu");
//2,獲取輸入流
FSDataInputStream fsDataInputStream = fileSystem.open(new Path("/user/atguigu/hadoop-2.7.2.tar.gz"));
//3,獲取輸出流
FileOutputStream fos = new FileOutputStream(new File("e:/1/hadoop-2.7.2.tar.gz.part2"));
//4,流的對接,定位到128M
fsDataInputStream.seek(1024*1024*128);
//6,關閉資源
try {
//5,流的對接
IOUtils.copyBytes(fsDataInputStream, fos, conf);
} catch (Exception e) {
// TODO: handle exception
}finally{
IOUtils.closeStream(fsDataInputStream);
IOUtils.closeStream(fos);
}
}
}