- pom.xml
<!-- hadoop --> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-client</artifactId> <version>2.7.6</version> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-hdfs</artifactId> <version>2.7.6</version> </dependency>
- HadoopConfig.java
package com.hahashujia.hadoop.config; import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.fs.FileSystem; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; /** * @author yanan.wang * @describe * @createTime 2019-12-19 15:13 */ @Configuration @Slf4j public class HadoopConfig { @Value("${hadoop.user}") private String user; @Value("${hadoop.password}") private String password; @Value("${hdfs.hdfs-site}") private String hdfsSite; @Value("${hdfs.core-site}") private String coreSite; @Bean("fileSystem") public FileSystem createFs() throws Exception { System.setProperty("HADOOP_USER_NAME", user); // System.setProperty("HADOOP_USER_PASSWORD", password); //讀取配置文件 org.apache.hadoop.conf.Configuration conf = new org.apache.hadoop.conf.Configuration(); conf.addResource(coreSite); conf.addResource(hdfsSite); conf.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem"); log.info("===============【hadoop configuration info start.】==============="); log.info("【hadoop conf】: size:{}, {}", conf.size(), conf.toString()); log.info("【fs.defaultFS】: {}", conf.get("fs.defaultFS")); log.info("【fs.hdfs.impl】: {}", conf.get("fs.hdfs.impl")); FileSystem fs = FileSystem.newInstance(conf); log.info("【fileSystem scheme】: {}", fs.getScheme()); log.info("===============【hadoop configuration info end.】==============="); return fs; } }
- HadoopTemplate.java
package com.hahashujia.hadoop.config; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IOUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; import org.springframework.stereotype.Component; import java.io.IOException; import java.io.InputStream; /** * @author yanan.wang * @describe * @createTime 2019-12-19 15:15 */ @Component @ConditionalOnBean(FileSystem.class) @Slf4j public class HadoopTemplate { @Autowired private FileSystem fileSystem; public void uploadFile(String srcFile, String destPath) { copyFileToHDFS(false, true, srcFile, destPath); } public void uploadFile(boolean del, String srcFile, String destPath) { copyFileToHDFS(del, true, srcFile, destPath); } public void delDir(String path) { rmdir(path, null); } public void download(String fileName, String savePath) { getFile(fileName, savePath); } /** * 創建目錄 * * @param filePath * @param create * @return */ public boolean existDir(String filePath, boolean create) throws IOException { boolean flag = false; if (StringUtils.isEmpty(filePath)) { throw new IllegalArgumentException("filePath不能爲空"); } Path path = new Path(filePath); if (create) { if (!fileSystem.exists(path)) { fileSystem.mkdirs(path); } } if (fileSystem.isDirectory(path)) { flag = true; } return flag; } /** * 創建目錄 * * @param filePath * @return */ public boolean existFile(String filePath) throws IOException { if (StringUtils.isEmpty(filePath)) { throw new IllegalArgumentException("filePath不能爲空"); } Path path = new Path(filePath); return fileSystem.exists(path); } /** * 文件上傳至 HDFS * * @param delSrc 指是否刪除源文件,true 爲刪除,默認爲 false * @param overwrite * @param srcFile 源文件,上傳文件路徑 * @param destPath hdfs的目的路徑 */ public void copyFileToHDFS(boolean delSrc, boolean overwrite, String srcFile, String destPath) { // 源文件路徑是Linux下的路徑,如果在 windows 下測試,需要改寫爲Windows下的路徑,比如D://hadoop/djt/weibo.txt Path srcPath = new Path(srcFile); Path dstPath = new Path(destPath); // 實現文件上傳 try { // 獲取FileSystem對象 fileSystem.copyFromLocalFile(delSrc, overwrite, srcPath, dstPath); } catch (IOException e) { log.error("", e); } } /** * 刪除文件或者文件目錄 * * @param path */ public void rmdir(String path, String fileName) { try { if (StringUtils.isNotBlank(fileName)) { path = path + "/" + fileName; } // 刪除文件或者文件目錄 delete(Path f) 此方法已經棄用 fileSystem.delete(new Path(path), true); } catch (IllegalArgumentException | IOException e) { log.error("", e); } } /** * 從 HDFS 下載文件 * * @param hdfsFile * @param destPath 文件下載後,存放地址 */ public void getFile(String hdfsFile, String destPath) { Path hdfsPath = new Path(hdfsFile); Path dstPath = new Path(destPath); try { // 下載hdfs上的文件 fileSystem.copyToLocalFile(hdfsPath, dstPath); } catch (IOException e) { log.error("", e); } } public void writer(String destPath, InputStream in) { try { FSDataOutputStream out = fileSystem.create(new Path(destPath)); IOUtils.copyBytes(in, out, fileSystem.getConf()); } catch (IOException e) { e.printStackTrace(); } } }
- 調用(代碼片段)
@Autowired private HadoopTemplate hadoopTemplate; if (hadoopTemplate.existDir(hdfsPath, true)) { hadoopTemplate.copyFileToHDFS(false, overwrite, srcFile, hdfsPath); } else { log.error("==============dir create fail.=============="); }
下一篇:是另一種方式調用Hadoop
Hadoop系列——SpringBoot集成Hadoop、實現hdfs上傳下載文件(2)
Hadoop系列——SpringBoot集成Hadoop、實現hdfs上傳下載文件(1)
發表評論
所有評論
還沒有人評論,想成為第一個評論的人麼? 請在上方評論欄輸入並且點擊發布.