Amazon aws s3上传文件，并设置为公共可读
直接上硬菜：
1.依赖
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
<version>1.11.625</version>
</dependency>
2.编码
public static String uploadToS3(MultipartFile file) throws IOException {
String perfix = "https://xxxxx.s3-us-west-1.amazonaws.com/";
String bucketName = "txxxxx";
if (file.isEmpty()) {
return "上传文件不能为空";
}
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentType(file.getContentType());
metadata.setContentLength(file.getSize());
String key = UUID.randomUUID().toString().replaceAll("-", "") + "." + getFileType(file.getOriginalFilename());
String fileTypeByContentType = getFileTypeByContentType(file.getContentType());
if ("image".equals(fileTypeByContentType)) {
bucketName = "xxx-img";
perfix = "https://xxx-img.s3-us-west-1.amazonaws.com/";
} else if ("video".equals(fileTypeByContentType)) {
bucketName = "xxx-video1";
perfix = "https://xxx-video1.s3-us-west-1.amazonaws.com/";
} else {
bucketName = "xxxx-other";
perfix = "https://xxx-other.s3-us-west-1.amazonaws.com/";
}
try {
//验证名称为bucketName的bucket是否存在，不存在则创建
if (!checkBucketExists(s3Client, bucketName)) {
s3Client.createBucket(bucketName);
}
zhi
/*之前被误导，一直上传上此的文件，返回一个链接带有有效期，而且最七天，各种想办法，其实是写法就错误了，
应该用下面的这种写法withCannedAcl，设置ACL权限就好，希望大家避坑*/
//开始上传文件
s3Client.putObject(new PutObjectRequest(bucketName, key, file.getInputStream(), metadata)
.withCannedAcl(CannedAccessControlList.PublicRead));
String url = perfix + key;
if (url == null) {
throw new BizException(GlobalExceptionEnum.SERVER_ERROR.getCode(), " can't get s3 file url!");
}
return url.toString();
} catch (AmazonServiceException ase) {
ase.printStackTrace();
log.info("====================================AWS S3 UPLOAD ERROR START======================================");
log.info("Caught an AmazonServiceException, which means your request made it "
+ "to Amazon S3, but was rejected with an error response for some reason.");
log.info("Caught an AmazonServiceException, which means your request made it "
+ "to Amazon S3, but was rejected with an error response for some reason.");
log.info("Error Message: " + ase.getMessage());
log.info("HTTP Status Code: " + ase.getStatusCode());
log.info("AWS Error Code: " + ase.getErrorCode());
log.info("Error Type: " + ase.getErrorType());
log.info("Request ID: " + ase.getRequestId());
log.info(ase.getMessage(), ase);
log.info("====================================AWS S3 UPLOAD ERROR END======================================");
throw new BizException(GlobalExceptionEnum.SERVER_ERROR.getCode(), "error occurs during upload to s3!");
} catch (AmazonClientException ace) {
log.info("====================================AWS S3 UPLOAD ERROR START======================================");
log.info("Caught an AmazonClientException, which means the client encountered "
+ "a serious internal problem while trying to communicate with S3, "
+ "such as not being able to access the network.");
log.info("Error Message: " + ace.getMessage());
log.info("====================================AWS S3 UPLOAD ERROR END======================================");
throw new BizException(GlobalExceptionEnum.SERVER_ERROR.getCode(), "error occurs during upload to s3!");
} finally {
}
}
就是这么硬
在给定bucket新建文件夹
其实非常检查，简单，只需要在key设置的时候，加上文件夹名，再加个斜杆/即可
核心代码在这里，
记得上传改为用线程池，多线程去异步执行就好，如果需要返回结果，则调用callable方法实现，把结果返回给主线程，
以实现线程间通讯
今天想记录一下工作中一个场景：
上传一个zip包（是网站的一个主题活动页面，里面是只能有一个html文件和多个css/js/image等之类的资源文件）
后台服务器解压，并上上传到asw的s3存储。
解决步骤
1.upload文件到服务器本地
2.结果zip文件
3.用多线程 实现批量上传文件到s3，并把上传返回的结果记录到内存map中，用以在第4步中替换
4.本来想着服务端去I/O html文件，readline,然后根据step3中的map，遍历该map再替换，发现其实挺慢
5.于是做了决定，让前端自己去修改html中资源路径，直接告诉她替换规则为http://xxxxx/压缩文件名字/资源名
这样就带来的好处
1.可以直接异步去上传这个一个html文件，立刻返回个前端结果
2.步骤3中不需要存每一个上传的返回路径了，他可以完全异步去做，
3.步骤4中也不要去I/O html文件，readline，再去遍历了
好处大大的啊！就这么愉快的决定了
代码如下
public static String generatorPage(File zipFile) {
try {
String zipFileOriginalFilename = zipFile.getName();
String zipfileSuffix = FileUtil.getFileType(zipFileOriginalFilename);
if (!zipfileSuffix.equals("zip")){
throw new BizException(GlobalExceptionEnum.UPLOAD_FILE_ONLY_ZIP);
}
//服务器保存路径-------配置文件读取
String serverFilePath = PropertyUtils.getftpProp("upload_zipfile_path");
/**
* step1.上传文件到应用服务器
*/
byte[] zipFileByteArrays = getFileByteArray(zipFile);
FileUtil.uploadFile(zipFileByteArrays, serverFilePath, zipFileOriginalFilename);
String uploadFilePath = serverFilePath + zipFileOriginalFilename;
/**
* step2.将该文件解压
*/
ZipUtil.unZipFiles(uploadFilePath, serverFilePath);
String unzipFilePath = uploadFilePath.substring(0, uploadFilePath.lastIndexOf("."));
File file = new File(unzipFilePath);
String pageName = file.getName();
List<File> allUnZipFiles = FileUtil.getAllFiles(unzipFilePath);
File htmlFile = null;
/**
* step3.上传解压后的html文件到s3
*/
// Map replaceMap = new HashMap();
int htmlfileNumber = 0;
for (File unzipFile : allUnZipFiles) {
String unzipfileName = unzipFile.getName();
String unzipfileSuffix1 = FileUtil.getFileType(unzipfileName);
if (unzipfileSuffix1.equals("html") || unzipfileSuffix1.equals("htm")) {//如果是html文件
htmlFile = unzipFile;
htmlfileNumber++;
}
}
if (htmlfileNumber == 0 || htmlfileNumber > 1) {
throw new BizException(GlobalExceptionEnum.UPLOAD_FILE_HTML_NUMBER);
}
long start = System.currentTimeMillis();
String s3Path = uploadS3ThreadPoolAsync(pageName, htmlFile).get();
long end1 = System.currentTimeMillis();
System.out.println("上传hmtl页面完毕，耗时：" + (end1 - start));
/**
* step4.上传非html文件到s3
*/
for (File unzipFile : allUnZipFiles) {
String unzipfileName = unzipFile.getName();
String unzipfileSuffix = FileUtil.getFileType(unzipfileName);
if (!"zip".equals(unzipfileSuffix) && !"DS_Store".equals(unzipfileSuffix) && !"html".equals(unzipfileSuffix) && !"htm".equals(unzipfileSuffix)) {
uploadS3ThreadPoolAsync(pageName, unzipFile);
}
}
//step4.替换html文件中资源路径[这一步省去，让前端同学处理，把所有资源类的文件路径都替换为：
// https://xxxxx.amazonaws.com/压缩包文件名/14769441969842358.png]
//FileUtil.alterStringToCreateNewFile(htmlFile.getAbsolutePath(), replaceMap);
long end2 = System.currentTimeMillis();
System.out.println(" ----多线程，异步发布任务结束(后台异步去运行)-----：" + (end2 - end1));
System.out.println("主线程运行结束，耗时：" + (end2 - start));
return s3Path;
} catch (Exception e) {
e.printStackTrace();
throw new BizException(GlobalExceptionEnum.UPLOAD_GENERATOR_PAGE_FAIL);
}
}
private static Future<String> uploadS3ThreadPoolAsync(String pageName, File htmlFile) throws InterruptedException, ExecutionException {
Future<String> submit = executorService.submit(() -> {
try {
String htmlS3Path = AmazonS3Util.uploadToS3Async(htmlFile, false, pageName);
return htmlS3Path;
} catch (IOException e) {
e.printStackTrace();
return "";
}
});
return submit;
}
/**
*
* @param file 要上传的文件,文件为普通文件
* @param issinglefile 是否是单个文件上传
* @param pageName 活动页压缩名称
* @return
* @throws IOException
*/
public static String uploadToS3Async(File file, boolean issinglefile, String pageName) throws IOException {
if (file.length() == 0) {
throw new BizException(GlobalExceptionEnum.UPLOAD_FILE_IS_EMPTY);
}
long size = file.length();
String path = file.getPath();
String originalFilename = file.getName();
InputStream inputStream = new FileInputStream(file);
String contentType = FileUtil.getFileContentTypeByPath(path);
return getUploadS3Path(contentType, originalFilename, size, inputStream, issinglefile, pageName);
}
/**
*
* @param multipartFile 要上传的文件，文件为Multipart文件
* @param issinglefile 是否是单个文件上传
* @param pageName 活动页压缩名称
* @return
* @throws IOException
*/
public static String uploadToS3Async(MultipartFile multipartFile, boolean issinglefile, String pageName) throws IOException {
if (multipartFile.isEmpty()) {
throw new BizException(GlobalExceptionEnum.UPLOAD_FILE_IS_EMPTY);
}
String contentType = multipartFile.getContentType();
String originalFilename = multipartFile.getOriginalFilename();
long size = multipartFile.getSize();
InputStream inputStream = multipartFile.getInputStream();
return getUploadS3Path(contentType, originalFilename, size, inputStream, issinglefile, pageName);
}
/**
* 普通单个文件上传
*
* @param contentType
* @param originalFilename
* @param size
* @param inputStream
* @return
*/
private static String getUploadS3Path(String contentType, String originalFilename, long size, InputStream inputStream, boolean issinglefile, String pageName) {
if (!issinglefile) {
bucketName = "tfc-page";
perfix = "https://tfc-page.s3-us-west-1.amazonaws.com/";
} else {
String fileTypeByContentType = getFileTypeByContentType(contentType);
if ("image".equals(fileTypeByContentType)) {
bucketName = "tfc-img";
perfix = "https://tfc-img.s3-us-west-1.amazonaws.com/";
} else if ("video".equals(fileTypeByContentType)) {
bucketName = "tfc-video1";
perfix = "https://tfc-video1.s3-us-west-1.amazonaws.com/";
} else {
bucketName = "tfc-other";
perfix = "https://tfc-other.s3-us-west-1.amazonaws.com/";
}
}
return uploadToS3Asyn(contentType, originalFilename, size, inputStream, pageName);
}
private static String uploadToS3Asyn(String contentType, String originalFilename, long size, InputStream inputStream, String pageName) {
Future<String> submit = executorService.submit(() -> {
String key = "";
try {
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentType(contentType);
metadata.setContentLength(size);
if (pageName == null || "".equals(pageName)) {
key = DateUtil.getDays()+"/"+UUID.randomUUID().toString().replaceAll("-", "") + "." + getFileType(originalFilename);
} else {
key = pageName + "/" + originalFilename;
}
//验证名称为bucketName的bucket是否存在，不存在则创建
if (!checkBucketExists(s3Client, bucketName)) {
s3Client.createBucket(bucketName);
}
System.out.format("线程%s 正在上传 %s",Thread.currentThread().getName(),key);
System.out.println();
//开始上传文件
s3Client.putObject(new PutObjectRequest(bucketName, key, inputStream, metadata)
.withCannedAcl(CannedAccessControlList.PublicRead));
System.out.format("线程%s 上传完毕",Thread.currentThread().getName());
System.out.println();
String url = perfix + key;
return url;
} catch (AmazonServiceException ase) {
System.out.format("-----__------------上传key=%s时异常",key);
ase.printStackTrace();
log.info("====================================AWS S3 UPLOAD ERROR START======================================");
log.info("Caught an AmazonServiceException, which means your request made it "
+ "to Amazon S3, but was rejected with an error response for some reason.");
log.info("Caught an AmazonServiceException, which means your request made it "
+ "to Amazon S3, but was rejected with an error response for some reason.");
log.info("Error Message: " + ase.getMessage());
log.info("HTTP Status Code: " + ase.getStatusCode());
log.info("AWS Error Code: " + ase.getErrorCode());
log.info("Error Type: " + ase.getErrorType());
log.info("Request ID: " + ase.getRequestId());
log.info(ase.getMessage(), ase);
log.info("====================================AWS S3 UPLOAD ERROR END======================================");
throw new BizException(GlobalExceptionEnum.SERVER_ERROR.getCode(), "error occurs during upload to s3!");
} catch (AmazonClientException ace) {
log.info("====================================AWS S3 UPLOAD ERROR START======================================");
log.info("Caught an AmazonClientException, which means the client encountered "
+ "a serious internal problem while trying to communicate with S3, "
+ "such as not being able to access the network.");
log.info("Error Message: " + ace.getMessage());
log.info("====================================AWS S3 UPLOAD ERROR END======================================");
throw new BizException(GlobalExceptionEnum.SERVER_ERROR.getCode(), "error occurs during upload to s3!");
} finally {
}
});
String s3Path = null;
try {
s3Path = submit.get();
} catch (InterruptedException e) {
e.printStackTrace();
} catch (ExecutionException e) {
e.printStackTrace();
}
return s3Path;
}