amazonS3文件操作知识点汇总
1.文件上传时，对于不存在路径会是啥结果？
如果上传的存储桶不存在，会报错。如果是存储桶下面的文件夹不存在，则会创建一个并保存进去你上传的文件。第二次上传同样的路径，应为已经存在，则直接保存。如果上传的路径的 一级目录 前面多了斜杠“/”，也会报错。举个例子：
指定的文件上传的key为：/temp/20200723/小小的船-2129566950.pptx，则就会报错。
The specified key does not exist. (Service: Amazon S3; Status Code: 404; Error Code: NoSuchKey; Request ID: DE8C45718259ACD7;
1
2.amazonS3能够实现文件操作
上传，下载，复制，删除，但是没有直接的移动操作。并且通过在进入amazonS3进行配置生命周期的规则支持后，就可以实现对上传文件的过期时间的设置。这个过期时间的实现必须的前提条件就是要开启对存储桶指定目录的生命周期。如下图是对temp目录设置好的生命周期规则。其中前面这个transient_rule即是ruleId。
3.剩下一些工具类的实现
package com.util;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.core.io.Resource;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.ResponseEntity;
import org.springframework.web.client.RestTemplate;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.Delete;
import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse;
import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
import software.amazon.awssdk.services.s3.model.S3Exception;
import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Iterable;
import java.io.*;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
/**
* AmazonS3文件操作工具类
*
* @author zhanglifeng
* @date 2020-07-01
*/
public class AmazonS3Util {
private static final Logger LOGGER = LoggerFactory.getLogger(AmazonS3Util.class);
/**
* access_key_id 你的亚马逊S3服务器访问密钥ID
*/
private static final String ACCESS_KEY = "AIEKIAF3TXCLQT6M7A7L";
/**
* secret_key 你的亚马逊S3服务器访问密钥
*/
private static final String SECRET_KEY = "3wyLbaxZ62XnMMz517IB36c63jeRev6e8HhxemSS";
/**
* end_point 你的亚马逊S3服务器连接路径和端口(新版本不再需要这个,直接在创建S3对象的时候根据桶名和Region自动获取)
*
* 格式: https://桶名.s3-你的Region名称.amazonaws.com
* 示例: https://xxton.s3-cn-north-1.amazonaws.com
*/
//private static final String END_POINT = "https://xxton.s3-cn-north-1.amazonaws.com";
/**
* bucketname 你的亚马逊S3服务器创建的桶名
*/
private static final String S3_BUCKET_NAME = "media.fenglizhang.com";
/**
* 创建访问凭证对象
*/
private static final BasicAWSCredentials awsCreds = new BasicAWSCredentials(ACCESS_KEY, SECRET_KEY);
/**
* 创建s3对象
*/
private static final AmazonS3 s3Client = AmazonS3ClientBuilder.standard()
.withCredentials(new AWSStaticCredentialsProvider(awsCreds))
//设置服务器所属地区
.withRegion(Regions.US_WEST_1)
.build();
/**
* 经过测试，文件上传的方法这个要比下面的uploadToS3的快。
*
* @param key 文件在bucket中的存储文件名
* @param env 当前项目的环境：dev,test,stage,prod
* @param filePath 待上传的文件存放位置
* @param s3CdnBaseUrl cdn在不同环境的url
* @return
*/
public static String uploadToS3Fast(String key, String env, String filePath, String s3CdnBaseUrl) {
long startTime = System.currentTimeMillis();
S3Client client = S3Client.builder().region(Region.US_WEST_1).credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create(ACCESS_KEY, SECRET_KEY))).build();
String bucketName = getS3BucketName(env);
software.amazon.awssdk.services.s3.model.PutObjectRequest putObjectRequest = software.amazon.awssdk.services.s3.model.PutObjectRequest.builder().bucket(bucketName).key(key).build();
client.putObject(putObjectRequest, Path.of(filePath));
String downloadUrl = String.format("https://%s/%s", s3CdnBaseUrl, key);
long endTime = System.currentTimeMillis();
LOGGER.info("上传到S3耗时：{}", endTime - startTime);
return downloadUrl;
}
/**
* @param key 文件在bucket中的存储文件名
* @param env 当前项目的环境：dev,test,stage,prod
* @param filePath 待上传的文件存放位置
* @param s3CdnBaseUrl cdn在不同环境的url
* @return
*/
public static String uploadToS3(String key, String env, String filePath, String s3CdnBaseUrl) {
long startTime = System.currentTimeMillis();
String bucketName = getS3BucketName(env);
PutObjectRequest request = new PutObjectRequest(bucketName, key, new File(filePath));
s3Client.putObject(request);
String downloadUrl = String.format("https://%s/%s", s3CdnBaseUrl, key);
long endTime = System.currentTimeMillis();
LOGGER.info("上传到S3耗时：{}", endTime - startTime);
return downloadUrl;
}
/**
* @param key 文件在存储桶中的目录
* @param env 当前项目的环境：dev,test,stage,prod
* @param targetFilePath 缓存到本地的文件名
*/
public static void amazonS3Download(String key, String env, String targetFilePath) {
long startTime = System.currentTimeMillis();
String bucketName = getS3BucketName(env);
S3Object object = s3Client.getObject(new GetObjectRequest(bucketName, key));
if (object != null) {
LOGGER.info("Content-Type: " + object.getObjectMetadata().getContentType());
InputStream input = null;
FileOutputStream fileOutputStream = null;
byte[] data = null;
try {
//获取文件流
input = object.getObjectContent();
data = new byte[input.available()];
int len = 0;
fileOutputStream = new FileOutputStream(targetFilePath);
while ((len = input.read(data)) != -1) {
fileOutputStream.write(data, 0, len);
}
LOGGER.info("下载文件成功");
} catch (IOException e) {
LOGGER.error("文件流转换发生异常：{}", e);
} finally {
try {
if (fileOutputStream != null) {
fileOutputStream.flush();
fileOutputStream.close();
}
if (input != null) {
input.close();
}
long endTime = System.currentTimeMillis();
LOGGER.info("amazonS3Download方法从S3上下载文件耗时：{}", endTime - startTime);
} catch (IOException e) {
LOGGER.error("关闭文件流发生异常：{}", e);
}
}
}
}
private static String getS3BucketName(String env) {
String bucketName;
if ("prod".equals(env)) {
bucketName = S3_BUCKET_NAME;
} else if ("dev".equals(env)) {
bucketName = String.format("%s.%s", env, S3_BUCKET_NAME);
} else {
bucketName = String.format("%s.%s", "stage", S3_BUCKET_NAME);
}
return bucketName;
}
/**
* @param url 能够通过浏览器打开的资源链接。比如：https://cdn.fenglizhang.com/cw/1495159251-0000.png
* @param saveFile 要保存的文件地址
* @param restTemplate 需要注入实例的rest客户端
*/
public static void downloadFileFromS3(String url, File saveFile, RestTemplate restTemplate) {
try {
HttpHeaders headers = new HttpHeaders();
HttpEntity<Resource> httpEntity = new HttpEntity<>(headers);
ResponseEntity<byte[]> response = restTemplate.exchange(url, HttpMethod.GET, httpEntity, byte[].class);
FileOutputStream fos = new FileOutputStream(saveFile);
fos.write(response.getBody());
fos.flush();
fos.close();
} catch (Exception e) {
LOGGER.error(String.format("downloadFile url {} exception", url), e);
}
}
/**
* 本复制文件的方法适用于在同一个bucket中
*
* @param sourceKey Source object key
* @param env 当前应用的环境
* @param destinationKey Destination object key
*/
public static String copyObjectFromS3ToS3(String sourceKey, String env, String destinationKey, String s3CdnBaseUrl) {
String bucketName = getS3BucketName(env);
// Copy the object into a new object in the same bucket.
CopyObjectRequest copyObjRequest = new CopyObjectRequest(bucketName, sourceKey, bucketName, destinationKey);
/* ObjectMetadata objectMetadata = new ObjectMetadata();
objectMetadata.setExpirationTime();
objectMetadata.setExpirationTimeRuleId();
copyObjRequest.setNewObjectMetadata(objectMetadata);*/
CopyObjectResult copyObjectResult = s3Client.copyObject(copyObjRequest);
return String.format("https://%s/%s", s3CdnBaseUrl, destinationKey);
}
/**
* 上传文件到暂存区并且设置文件的过期时间
* @param key 文件在bucket中的存储文件名
* @param env 当前项目的环境：dev,test,stage,prod
* @param filePath 待上传的文件存放位置
* @param s3CdnBaseUrl cdn在不同环境的url
* @return
*/
public static String uploadToS3TransientAndSetExpiredTime(String key, String env, String filePath, String s3CdnBaseUrl) {
long startTime = System.currentTimeMillis();
String bucketName = getS3BucketName(env);
File file = new File(filePath);
PutObjectRequest request = new PutObjectRequest(bucketName, key, file);
String downloadUrl = null;
int index = filePath.lastIndexOf(".");
String contentType = String.format("%s/%s", "application", filePath.substring(index + 1));
InputStream inputStream = null;
try {
inputStream = new FileInputStream(file);
ObjectMetadata objectMetadata = new ObjectMetadata();
// 必须设置ContentLength
objectMetadata.setContentLength(file.length());
objectMetadata.setExpirationTimeRuleId("transient_rule");
objectMetadata.setContentType(contentType);
request.setMetadata(objectMetadata);
s3Client.putObject(bucketName, key, inputStream, objectMetadata);
downloadUrl = String.format("https://%s/%s", s3CdnBaseUrl, key);
long endTime = System.currentTimeMillis();
LOGGER.info("上传到S3耗时：{}", endTime - startTime);
} catch (FileNotFoundException e) {
LOGGER.error(e.getMessage());
} finally {
try {
inputStream.close();
} catch (IOException e) {
LOGGER.error("inputStream关闭异常，异常信息：{}",e.getMessage());
}
}
return downloadUrl;
}
/**
* 遍历存储桶指定文件夹下的所有文件路径
*
* @param s3
* @param bucket
* @param prefix
* @return
*/
public static List<String> listKeysInDirectory(S3Client s3, String bucket, String prefix) {
String delimiter = "/";
if (!prefix.endsWith(delimiter)) {
prefix += delimiter;
}
// Build the list objects request
software.amazon.awssdk.services.s3.model.ListObjectsV2Request listReq = software.amazon.awssdk.services.s3.model.ListObjectsV2Request.builder()
.bucket(bucket)
.prefix(prefix)
.delimiter(delimiter)
.maxKeys(1)
.build();
ListObjectsV2Iterable listRes = s3.listObjectsV2Paginator(listReq);
List<String> keyList = new ArrayList<>();
final String folder = prefix;
listRes.contents().stream()
.forEach(content -> {
if (!folder.equals(content.key())) {
keyList.add(content.key());
}
});
return keyList;
}
/**
* 删除指定目录下的所有文件
*
* @param env 环境
* @param folderPath
*/
public static void deleteS3Folder(String env, String folderPath) {
String bucketName = getS3BucketName(env);
S3Client s3 = S3Client.builder().
region(Region.US_WEST_1).
credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create(ACCESS_KEY, SECRET_KEY)))
.build();
ArrayList<ObjectIdentifier> to_delete = new ArrayList<ObjectIdentifier>();
List<String> object_keys = listKeysInDirectory(s3, bucketName, folderPath);
if (null == object_keys || object_keys.size() == 0) {
return;
}
for (String k : object_keys) {
to_delete.add(ObjectIdentifier.builder().key(k).build());
}
try {
software.amazon.awssdk.services.s3.model.DeleteObjectsRequest dor = software.amazon.awssdk.services.s3.model.DeleteObjectsRequest.builder()
.bucket(bucketName)
.delete(Delete.builder().objects(to_delete).build())
.build();
DeleteObjectsResponse response = s3.deleteObjects(dor);
while (!response.sdkHttpResponse().isSuccessful()) {
Thread.sleep(100);
}
} catch (S3Exception | InterruptedException e) {
LOGGER.error(e.getMessage());
}
LOGGER.info("delete folder successfully--->" + folderPath);
}
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
4.amazon上面工具类的依赖
<!-- aws依赖 -->
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>s3</artifactId>
<version>2.10.4</version>
<exclusions>
<exclusion>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>ses</artifactId>
<version>2.10.4</version>
<exclusions>
<exclusion>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
<version>1.11.233</version>
</dependency>
————————————————
版权声明：本文为CSDN博主「万米高空」的原创文章，遵循CC 4.0 BY-SA版权协议，转载请附上原文出处链接及本声明。
原文链接：https://blog.csdn.net/zhanglf02/article/details/107559433
