minio冷热分布部署
需求列表：
- 两个MinIO服务器，一个存放旧项目，一个存放新项目
- yaml文件中设置关于MinIO的三个项目：旧项目MinIO的url前缀，新项目MinIO的url前缀，新旧分界日期
- 每个物探项目在MinIO中创建一个文件夹，文件夹名称以创建项目的日期开头
- 系统中对所有MinIO保存的文件，仅记录后缀，从项目文件夹名称开始
- 运维人员会确保：早于新旧分界日期的项目，文件夹已经被迁移到旧项目MinIO
- 系统对于早于新旧分界日期的项目，使用旧项目MinIO，晚于分界的使用新项目MinIO
0 minio中的bucket和object
- bucket是存储object的逻辑空间，每个bucket之间是相互隔离的，相当于是存储文件的顶层文件夹
- object是存储到minio的基本对象，对用户而言，相当于是文件
1 minio的基本使用
- 引入依赖
<!--minio大文件上传-->
<dependency>
<groupId>io.minio</groupId>
<artifactId>minio</artifactId>
<version>8.0.3</version>
</dependency>
- 配置minio
# minio 配置（大文件分片上传使用）
minio:
endpoint: ${MINIO_ENDPOINT:http://minio服务IP:端口号}
accesskey: ${MINIO_ASSESSKEY:密钥}
secretkey: ${MINIO_SECRETKEY:密匙}
bucket: ${MINIO_BUCKET:桶名称}
- 配置minio客户端到spring容器中
@Bean
public MinioClient minioClient() {
return MinioClient.builder()
.endpoint(endpoint)
.credentials(accessKey, secretKey)
.build();
}
- 工具类
public class MinioUtil {
@Resource
private MinioProperties properties;
@Resource
private MinioClient minioClient;
/**
* 文件上传
*
* @param file 文件
* @return Boolean
*/
public String upload(MultipartFile file) {
String originalFilename = file.getOriginalFilename();
if (StringUtils.isBlank(originalFilename)) {
throw new RuntimeException("上传文件文件名为空");
}
String preName = originalFilename.substring(0,originalFilename.lastIndexOf("."));
String pastName = originalFilename.substring(originalFilename.lastIndexOf("."));
String objectName = preName + UUID.randomUUID() + pastName;
try {
PutObjectArgs objectArgs = PutObjectArgs.builder().bucket(properties.getBucketName()).object(objectName)
.stream(file.getInputStream(), file.getSize(), -1).contentType(file.getContentType()).build();
minioClient.putObject(objectArgs);
} catch (Exception e) {
e.printStackTrace();
return null;
}
return objectName;
}
/**
* 获取预览的路径
*/
public String preview(String objectName) {
// 查看文件地址
GetPresignedObjectUrlArgs build = new GetPresignedObjectUrlArgs().builder().bucket(properties.getBucketName()).object(objectName).method(Method.GET).build();
try {
return minioClient.getPresignedObjectUrl(build);
} catch (Exception e) {
e.printStackTrace();
}
return null;
}
/**
* 文件下载
*
*/
public InputStream download(String objectName) {
try {
GetObjectArgs objectArgs = GetObjectArgs.builder().bucket(properties.getBucketName()).object(objectName).build();
return minioClient.getObject(objectArgs);
} catch (Exception e) {
e.printStackTrace();
return null;
}
}
/**
* 根据对象名删除对象
* @param objectName
* @return
*/
public boolean remove(String objectName) {
try {
minioClient.removeObject(RemoveObjectArgs.builder().bucket(properties.getBucketName()).object(objectName).build());
} catch (Exception e) {
return false;
}
return true;
}
/**
* 复制对象
* @param src 复制源对象名
* @return
*/
public String copy(String src){
// 处理对象名
String target;
String pastName = src.substring(src.lastIndexOf('.'));
String preName = src.substring(0, src.lastIndexOf('.')-36);// 减去36个uuid的位置
target = preName + UUID.randomUUID() + pastName;
try {
minioClient.copyObject(CopyObjectArgs.builder()
.source(CopySource.builder().bucket(properties.getBucketName()).object(src).build())
.bucket(properties.getBucketName())
.object(target)
.build());
} catch (Exception e) {
e.printStackTrace();
throw new RuntimeException("复制对象失败...");
}
return target;
}
/**
* 查看存储bucket是否存在
* @return boolean
*/
public Boolean bucketExist() {
Boolean found;
try {
found = minioClient.bucketExists(BucketExistsArgs.builder().bucket(properties.getBucketName()).build());
} catch (Exception e) {
e.printStackTrace();
return false;
}
return found;
}
/**
* 创建存储bucket
*
* @return Boolean
*/
public Boolean makeBucket() {
try {
minioClient.makeBucket(MakeBucketArgs.builder()
.bucket(properties.getBucketName())
.build());
} catch (Exception e) {
e.printStackTrace();
return false;
}
return true;
}
/**
* 删除存储bucket
*
* @return Boolean
*/
public Boolean removeHotBucket(String bucketName) {
try {
minioClient.removeBucket(RemoveBucketArgs.builder()
.bucket(bucketName)
.build());
} catch (Exception e) {
e.printStackTrace();
return false;
}
return true;
}
/**
* 获取全部bucket
*/
public List<Bucket> getAllHotBuckets() {
try {
List<Bucket> buckets = minioClient.listBuckets();
return buckets;
} catch (Exception e) {
e.printStackTrace();
}
return null;
}
}
- 常用接口
/** 判断桶是否存在 */
Boolean bucketExists(BucketExistsArgs args);// 参数通过BucketExistsArgs.builder().bucket(name).build()构建
/** 创建桶 */
void makeBucket(MakeBucketArgs args);// 参数通过MakeBucketArgs.builder().bucket(name).build()创建，如果返回异常说明创建失败
/** 查询桶列表 */
List<Bucket> listBuckets(void);
/** 删除桶 */
void removeBucket(RemoveBucketArgs args); // 参数通过 RemoveBucketArgs.builder().bucket(name).build()创建，如果返回异常说明删除失败
/** 上传文件到指定的桶 */
ObjectWriteResponse pubObject(PutObjectArgs args); // args eg:PutObjectArgs.builder().bucket(bucketName).object(objectName).stream(new FileInputSteam(url), objectSize, -1).build()
/** 上传文件 */
ObjectWriteResponse uploadObject(UploadObjectArgs args);// args eg:UploadObjectArgs.builder().bucket(bucketName).object(objectName).filename(localUrl).build()
...
如果上传的文件过大（GB级别），可以采用分片上传的方式，提高上传效率和稳定性
具体需要：
- 初始化分片
- 上传
- 合并分片
需要的接口：
- 上传：接收multipartFile存入服务器同时返回url
- 删除：通过是否抛出异常判断是否删除成功
- 下载：根据url从服务器获取文件对象
2 冷热分布解决方案
- 因为需要分布到两个服务器上，所以创建两个不同的bean对象同时通过name进行区分
- 由于common模块中没有依赖项目中的其他模块，所以只能通过传入具体的时间的方式来判断存储的定向
2.1 配置yaml
#MINIO对象存储信息
minio:
enabled: true
endpointHot: ******
accessKeyHot: ******
secretKeyHot: ******
endpointCold: ******
accessKeyCold: ******
secretKeyCold: ******
separateTime: yyyy-MM-dd HH:mm:ss
bucketName: phys-pros
2.2 修改配置类
注：此处需要注入两个不同服务器上的客户端，所以配置时需要通过指定不同名称来进行区分
@Component
@ConfigurationProperties(prefix = "minio")
public class MinioConfig {
@Value("${minio.endpointHot}")
private String endpointHot;
@Value("${minio.accessKeyHot}")
private String accessKeyHot;
@Value("${minio.secretKeyHot}")
private String secretKeyHot;
@Value("${minio.endpointCold}")
private String endpointCold;
@Value("${minio.accessKeyCold}")
private String accessKeyCold;
@Value("${minio.secretKeyCold}")
private String secretKeyCold;
@Bean(name = "minioClientHot")
public MinioClient minioClientHot() {
return MinioClient.builder()
.endpoint(endpointHot)
.credentials(accessKeyHot, secretKeyHot)
.build();
}
@Bean(name = "minioClientCold")
public MinioClient minioClientCold() {
return MinioClient.builder()
.endpoint(endpointCold)
.credentials(accessKeyCold, secretKeyCold)
.build();
}
}
2.3 修改工具类
上传时存储对象的命名格式：项目创建日期--项目id/唯一文件名
@Component
public class MinioUtils {
@Autowired
private MinioConfig prop;
@Resource
@Qualifier("minioClientHot")
private MinioClient minioClientHot;
@Resource
@Qualifier("minioClientCold")
private MinioClient minioClientCold;
@Value("${minio.bucketName}")
private String bucketName;
@Value("${minio.separateTime}")
private String separateTime;
/**
* 文件上传 传到项目文件夹中
*
* @param file 文件
* @param createDate 项目创建日期
* @param projectId 项目id
* @return String 对象名
*/
public String upload(MultipartFile file, Date createDate, Long projectId, String subDir) {
String originalFilename = file.getOriginalFilename();
if (StringUtils.isBlank(originalFilename)) {
throw new RuntimeException();
}
String preName = originalFilename.substring(0,originalFilename.lastIndexOf("."));
String pastName = originalFilename.substring(originalFilename.lastIndexOf("."));
String fileName = preName + IdUtils.randomUUID() + pastName;
SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd");
// 对象名格式：项目创建日期_项目id/唯一文件名
String objectName;
if(StringUtils.isEmpty(subDir)) objectName = simpleDateFormat.format(createDate) + "_" + projectId + "/" + fileName;
else objectName = simpleDateFormat.format(createDate) + "_" + projectId + "/" + subDir + "/" + fileName;
try {
PutObjectArgs objectArgs = PutObjectArgs.builder().bucket(bucketName).object(objectName)
.stream(file.getInputStream(), file.getSize(), -1).contentType(file.getContentType()).build();
simpleDateFormat.applyPattern("yyyy-MM-dd HH:mm:ss");
if(createDate.before(simpleDateFormat.parse(separateTime))){
minioClientCold.putObject(objectArgs);
}else{
minioClientHot.putObject(objectArgs);
}
} catch (Exception e) {
e.printStackTrace();
return null;
}
return objectName;
}
/**
* 通用文件的上传
* @param file
* @return 对象名
*/
public String upload(MultipartFile file, String prefix) {
String originalFilename = file.getOriginalFilename();
if (StringUtils.isBlank(originalFilename)) {
throw new RuntimeException();
}
String preName = originalFilename.substring(0,originalFilename.lastIndexOf("."));
String pastName = originalFilename.substring(originalFilename.lastIndexOf("."));
String fileName = preName + IdUtils.randomUUID() + pastName;
String objectName = prefix + "/" + fileName;
try {
PutObjectArgs objectArgs = PutObjectArgs.builder().bucket(bucketName).object(objectName)
.stream(file.getInputStream(), file.getSize(), -1).contentType(file.getContentType()).build();
minioClientHot.putObject(objectArgs);
} catch (Exception e) {
e.printStackTrace();
return null;
}
return objectName;
}
/**
* 预览图片 获取预览的路径
*/
public String preview(String objectName, Date createDate) {
// 查看文件地址
GetPresignedObjectUrlArgs build = new GetPresignedObjectUrlArgs().builder().bucket(bucketName).object(objectName).method(Method.GET).build();
try {
String url;
SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
if(createDate.before(simpleDateFormat.parse(separateTime))){
url = minioClientCold.getPresignedObjectUrl(build);
}else{
url = minioClientHot.getPresignedObjectUrl(build);
}
return url;
} catch (Exception e) {
e.printStackTrace();
}
return null;
}
/**
* 获取预览地址
* @param objectName
* @return
*/
public String preview(String objectName) {
GetPresignedObjectUrlArgs build = new GetPresignedObjectUrlArgs().builder().bucket(bucketName).object(objectName).method(Method.GET).build();
try {
String url;
url = minioClientHot.getPresignedObjectUrl(build);
return url;
} catch (Exception e) {
e.printStackTrace();
}
return null;
}
/**
* 文件下载
*
*/
public InputStream download(String objectName, Date createDate) {
try {
GetObjectArgs objectArgs = GetObjectArgs.builder().bucket(bucketName).object(objectName).build();
SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
if(createDate.before(simpleDateFormat.parse(separateTime))){
return minioClientCold.getObject(objectArgs);
}else{
return minioClientHot.getObject(objectArgs);
}
} catch (ErrorResponseException | InsufficientDataException | InternalException | InvalidKeyException |
InvalidResponseException | IOException | NoSuchAlgorithmException | ServerException |
XmlParserException e) {
e.printStackTrace();
return null;
} catch (ParseException e) {
e.printStackTrace();
return null;
}
}
/**
* 文件下载
* @param objectName
* @return
*/
public InputStream download(String objectName) {
try {
GetObjectArgs objectArgs = GetObjectArgs.builder().bucket(bucketName).object(objectName).build();
return minioClientHot.getObject(objectArgs);
} catch (ErrorResponseException | InsufficientDataException | InternalException | InvalidKeyException |
InvalidResponseException | IOException | NoSuchAlgorithmException | ServerException |
XmlParserException e) {
e.printStackTrace();
return null;
}
}
/**
* 删除文件
*
*/
public boolean remove(String objectName, Date createDate) {
try {
SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
if(createDate.before(simpleDateFormat.parse(separateTime))){
minioClientCold.removeObject(RemoveObjectArgs.builder().bucket(bucketName).object(objectName).build());
}else{
minioClientHot.removeObject(RemoveObjectArgs.builder().bucket(bucketName).object(objectName).build());
}
} catch (Exception e) {
return false;
}
return true;
}
/**
* 根据对象名删除对象
* @param objectName
* @return
*/
public boolean remove(String objectName) {
try {
minioClientHot.removeObject(RemoveObjectArgs.builder().bucket(bucketName).object(objectName).build());
} catch (Exception e) {
return false;
}
return true;
}
/**
* 复制对象
* @param src 复制源对象名
* @param createTime 所属项目创建时间
* @return
*/
public String copy(String src, Date createTime){
// 处理对象名
String dest;
String pastName = src.substring(src.lastIndexOf('.'));
String preName = src.substring(0, src.lastIndexOf('.')-36);// 减去36个uuid的位置
dest = preName + IdUtils.randomUUID() + pastName;
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
try {
Date date = format.parse(separateTime);
if(createTime.before(date)){// cold
minioClientCold.copyObject(CopyObjectArgs.builder()
.source(CopySource.builder().bucket(bucketName).object(src).build())
.bucket(bucketName)
.object(dest)
.build());
}else{
minioClientHot.copyObject(CopyObjectArgs.builder()
.source(CopySource.builder().bucket(bucketName).object(src).build())
.bucket(bucketName)
.object(dest)
.build());
}
} catch (Exception e) {
e.printStackTrace();
throw new RuntimeException();
}
return dest;
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////
/**
* 查看存储bucket是否存在
* @return boolean
*/
public Boolean hotBucketExists() {
Boolean found;
try {
found = minioClientHot.bucketExists(BucketExistsArgs.builder().bucket(bucketName).build());
} catch (Exception e) {
e.printStackTrace();
return false;
}
return found;
}
/**
* 创建存储bucket
*
* @return Boolean
*/
public Boolean makeHotBucket() {
try {
minioClientHot.makeBucket(MakeBucketArgs.builder()
.bucket(bucketName)
.build());
} catch (Exception e) {
e.printStackTrace();
return false;
}
return true;
}
/**
* 删除存储bucket
*
* @return Boolean
*/
public Boolean removeHotBucket(String bucketName) {
try {
minioClientHot.removeBucket(RemoveBucketArgs.builder()
.bucket(bucketName)
.build());
} catch (Exception e) {
e.printStackTrace();
return false;
}
return true;
}
/**
* 获取全部bucket
*/
public List<Bucket> getAllHotBuckets() {
try {
List<Bucket> buckets = minioClientHot.listBuckets();
return buckets;
} catch (Exception e) {
e.printStackTrace();
}
return null;
}
/**
* 查看存储bucket是否存在
* @return boolean
*/
public Boolean coldBucketExists() {
Boolean found;
try {
found = minioClientCold.bucketExists(BucketExistsArgs.builder().bucket(bucketName).build());
} catch (Exception e) {
e.printStackTrace();
return false;
}
return found;
}
/**
* 创建存储bucket
*
* @return Boolean
*/
public Boolean makeColdBucket() {
try {
minioClientCold.makeBucket(MakeBucketArgs.builder()
.bucket(bucketName)
.build());
} catch (Exception e) {
e.printStackTrace();
return false;
}
return true;
}
/**
* 删除存储bucket
*
* @return Boolean
*/
public Boolean removeColdBucket(String bucketName) {
try {
minioClientCold.removeBucket(RemoveBucketArgs.builder()
.bucket(bucketName)
.build());
} catch (Exception e) {
e.printStackTrace();
return false;
}
return true;
}
/**
* 获取全部bucket
*/
public List<Bucket> getAllColdBuckets() {
try {
List<Bucket> buckets = minioClientCold.listBuckets();
return buckets;
} catch (Exception e) {
e.printStackTrace();
}
return null;
}
}
2.4 编写接口
- 对于文件的操作共四种:上传 下载 预览 删除
- 由于需要根据项目的创建时间定位minio服务器,所以每个接口都需要明确文件所属项目
- 接口中不可能都是直接接收到创建时间的参数,而是接收项目的id,通过id再进行查询
- 而在common模块中,是没有其他模块的依赖的,所以接口类不能写在common模块中
- 由于是对文件进行查询,所以接口定义在system模块中(SupProject的相关逻辑在system中)
