第2-3-3章 文件处理策略-文件存储服务系统-nginx/fastDFS/minio/阿里云oss/七牛云oss
5.2 文件处理策略
在开发fastDFS和minio实现类之前,需要提前安装部署好fastDFS和minio。搭建教程可参考前面的章节。
第2-1-2章 传统方式安装FastDFS-附FastDFS常用命令
第2-1-3章 docker-compose安装FastDFS,实现文件存储服务
第2-1-5章 docker安装MinIO实现文件存储服务-springboot整合minio-minio全网最全的资料
全套代码及资料全部完整提供,点此处下载
由于我们当前的文件服务需要给客户端提供统一的服务接口,这就需要文件服务屏蔽底层的具体文件存储方式,所以对文件处理策略进行如下类和接口的设计:
5.2.1 FileStrategy
FileStrategy是文件处理策略顶层接口,是对文件处理的顶层抽象,具体代码如下:
package com.itheima.pinda.file.strategy;
import com.itheima.pinda.file.domain.FileDeleteDO;
import com.itheima.pinda.file.entity.File;
import org.springframework.web.multipart.MultipartFile;
import java.util.List;
/**
* 文件策略接口
*/
public interface FileStrategy {
/**
* 文件上传
*
* @param file 文件
* @return 文件对象
*/
File upload(MultipartFile file);
/**
* 删除文件
*
* @param list 列表
*/
boolean delete(List<FileDeleteDO> list);
}
5.2.2 AbstractFileStrategy
AbstractFileStrategy是抽象文件策略处理类,实现了FileStrategy接口。
AbstractFileStrategy实现主要的文件上传、删除的处理流程,例如异常情况的判断,文件对象的封装等,但是真正上传的处理过程需要其子类来完成,因为不同的存储方案处理方式是不同的。
package com.itheima.pinda.file.strategy.impl;
import com.itheima.pinda.exception.BizException;
import com.itheima.pinda.file.domain.FileDeleteDO;
import com.itheima.pinda.file.entity.File;
import com.itheima.pinda.file.enumeration.IconType;
import com.itheima.pinda.file.properties.FileServerProperties;
import com.itheima.pinda.file.strategy.FileStrategy;
import com.itheima.pinda.file.utils.FileDataTypeUtil;
import com.itheima.pinda.utils.DateUtils;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.multipart.MultipartFile;
import java.time.LocalDateTime;
import java.util.List;
import static com.itheima.pinda.exception.code.ExceptionCode.BASE_VALID_PARAM;
/**
* 文件抽象策略处理类
*/
@Slf4j
public abstract class AbstractFileStrategy implements FileStrategy {
private static final String FILE_SPLIT = ".";
@Autowired
protected FileServerProperties fileProperties;
protected FileServerProperties.Properties properties;
/**
* 上传文件
*
* @param multipartFile
* @return
*/
@Override
public File upload(MultipartFile multipartFile) {
try {
if (!multipartFile.getOriginalFilename().contains(FILE_SPLIT)) {
throw BizException.wrap(BASE_VALID_PARAM.build("缺少后缀名"));
}
File file = File.builder()
.isDelete(false).
submittedFileName(multipartFile.getOriginalFilename())
.contextType(multipartFile.getContentType())
.dataType(FileDataTypeUtil.getDataType(multipartFile.getContentType()))
.size(multipartFile.getSize())
.ext(FilenameUtils.getExtension(multipartFile.getOriginalFilename()))
.build();
file.setIcon(IconType.getIcon(file.getExt()).getIcon());
setDate(file);
uploadFile(file, multipartFile);
return file;
} catch (Exception e) {
log.error("e={}", e);
throw BizException.wrap(BASE_VALID_PARAM.build("文件上传失败"));
}
}
/**
* 获取下载地址前缀
*/
protected String getUriPrefix() {
if (StringUtils.isNotEmpty(properties.getUriPrefix())) {
return properties.getUriPrefix();
} else {
return properties.getEndpoint();
}
}
/**
* 具体类型执行上传操作
*
* @param file
* @param multipartFile
* @throws Exception
*/
protected abstract void uploadFile(File file, MultipartFile multipartFile) throws Exception;
private void setDate(File file) {
LocalDateTime now = LocalDateTime.now();
file.setCreateMonth(DateUtils.formatAsYearMonthEn(now))
.setCreateWeek(DateUtils.formatAsYearWeekEn(now))
.setCreateDay(DateUtils.formatAsDateEn(now));
}
@Override
public boolean delete(List<FileDeleteDO> list) {
if (list.isEmpty()) {
return true;
}
boolean flag = false;
for (FileDeleteDO file : list) {
try {
delete(file);
flag = true;
} catch (Exception e) {
log.error("删除文件失败", e);
}
}
return flag;
}
/**
* 具体执行删除方法, 无需处理异常
*/
protected abstract void delete(FileDeleteDO file);
}
5.2.3 LocalServiceImpl
LocalServiceImpl是AbstractFileStrategy的子类,负责处理存储策略为本地时的文件上传和删除操作。为了使程序能够动态选择具体的策略处理类,可以提供一个配置类,在配置类中定义LocalServiceImpl,具体代码如下:
package com.itheima.pinda.file.storage;
import cn.hutool.core.util.StrUtil;
import com.itheima.pinda.file.domain.FileDeleteDO;
import com.itheima.pinda.file.entity.File;
import com.itheima.pinda.file.properties.FileServerProperties;
import com.itheima.pinda.file.strategy.impl.AbstractFileStrategy;
import com.itheima.pinda.utils.StrPool;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Configuration;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import java.nio.file.Paths;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.UUID;
import static com.itheima.pinda.utils.DateUtils.DEFAULT_MONTH_FORMAT_SLASH;
/**
* 本地上传配置
*/
@EnableConfigurationProperties(FileServerProperties.class)
@Configuration
@ConditionalOnProperty(name = "pinda.file.type", havingValue = "LOCAL")
@Slf4j
public class LocalAutoConfigure {
/**
* 本地文件策略处理类
*/
@Service
public class LocalServiceImpl extends AbstractFileStrategy {
private void buildClient() {
properties = fileProperties.getLocal();
}
/**
* 上传文件
* @param file
* @param multipartFile
* @throws Exception
*/
@Override
protected void uploadFile(File file, MultipartFile multipartFile) throws Exception {
buildClient();
//生成文件名
String fileName = UUID.randomUUID().toString() + StrPool.DOT + file.getExt();
//日期文件夹,例如:2020\04
String relativePath = Paths.get(LocalDate.now().format(DateTimeFormatter.ofPattern(DEFAULT_MONTH_FORMAT_SLASH))).toString();
// web服务器存放的绝对路径,例如:D:\\uploadFiles\\oss-file-service\\2020\\04
String absolutePath = Paths.get(properties.getEndpoint(), properties.getBucketName(), relativePath).toString();
//目标输出文件
java.io.File outFile = new java.io.File(Paths.get(absolutePath, fileName).toString());
//向目标文件写入数据
org.apache.commons.io.FileUtils.writeByteArrayToFile(outFile, multipartFile.getBytes());
String url = new StringBuilder(getUriPrefix())
.append(StrPool.SLASH)
.append(properties.getBucketName())
.append(StrPool.SLASH)
.append(relativePath)
.append(StrPool.SLASH)
.append(fileName)
.toString();
//替换掉windows环境的\路径
url = StrUtil.replace(url, "\\\\", StrPool.SLASH);
url = StrUtil.replace(url, "\\", StrPool.SLASH);
file.setUrl(url);
file.setFilename(fileName);
file.setRelativePath(relativePath);
}
/**
* 文件删除
* @param file
*/
@Override
protected void delete(FileDeleteDO file) {
java.io.File ioFile =
new java.io.File(Paths.get(properties.getEndpoint(),
properties.getBucketName(),
file.getRelativePath(),
file.getFileName()).toString());
org.apache.commons.io.FileUtils.deleteQuietly(ioFile);
}
}
}
通过上面的代码可以看到,在进行文件上传和文件删除时都会使用到配置文件中的配置项,关于本地文件处理策略的配置如下:
pinda:
mysql:
database: pd_files
nginx:
ip: ${spring.cloud.client.ip-address} #正式环境要将该ip设置成nginx对应的公网ip
port: 10000 #正式环境需要将该ip设置成nginx对应的公网端口
file:
type: LOCAL
local:
uriPrefix: http://${pinda.nginx.ip}:${pinda.nginx.port}
bucket-name: oss-file-service
endpoint: D:\soft\nginx-1.23.0\uploadFiles
5.2.4 FastDfsServiceImpl
FastDfsServiceImpl是AbstractFileStrategy的子类,负责处理存储策略为FastDFS时的文件上传和删除操作。为了使程序能够动态选择具体的策略处理类,可以提供一个配置类,在配置类中定义FastDfsServiceImpl,具体代码如下:
package com.itheima.pinda.file.storage;
import com.github.tobato.fastdfs.domain.fdfs.StorePath;
import com.github.tobato.fastdfs.service.FastFileStorageClient;
import com.itheima.pinda.file.domain.FileDeleteDO;
import com.itheima.pinda.file.entity.File;
import com.itheima.pinda.file.properties.FileServerProperties;
import com.itheima.pinda.file.strategy.impl.AbstractFileStrategy;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Configuration;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
/**
* FastDFS配置
*/
@EnableConfigurationProperties(FileServerProperties.class)
@Configuration
@Slf4j
@ConditionalOnProperty(name = "pinda.file.type", havingValue = "FAST_DFS")
public class FastDfsAutoConfigure {
/**
* FastDFS文件策略处理类
*/
@Service
public class FastDfsServiceImpl extends AbstractFileStrategy {
@Autowired
private FastFileStorageClient storageClient; //操作FastDFS的客户端
/**
* 上传文件
* @param file
* @param multipartFile
* @throws Exception
*/
@Override
protected void uploadFile(File file, MultipartFile multipartFile)
throws Exception {
//调用FastDFS客户端将文件上传到FastDFS
StorePath storePath =
storageClient.uploadFile(multipartFile.getInputStream(),
multipartFile.getSize(),
file.getExt(),
null);
file.setUrl(fileProperties.getUriPrefix() +
storePath.getFullPath());
file.setGroup(storePath.getGroup());
file.setPath(storePath.getPath());
}
/**
* 文件删除
* @param file
*/
@Override
protected void delete(FileDeleteDO file) {
//调用FastDFS客户端删除文件
storageClient.deleteFile(file.getGroup(), file.getPath());
}
}
}
通过上面代码可以看到要使用FastDFS提供的客户端FastFileStorageClient来实现文件的上传和删除,这就需要在文件服务对应的配置文件中进行如下配置:
pinda:
mysql:
database: pd_files
nginx:
ip: ${spring.cloud.client.ip-address} #正式环境要将该ip设置成nginx对应的公网ip
port: 10000 #正式环境需要将该ip设置成nginx对应的公网端口
file:
type: FAST_DFS
uriPrefix: http://172.17.0.115:8188/ #存储类型为FAST_DFS时使用
#FAST_DFS配置
fdfs:
soTimeout: 1500
connectTimeout: 600
thumb-image:
width: 150
height: 150
tracker-list:
- 172.17.0.115:22122
pool:
#从池中借出的对象的最大数目
max-total: 153
max-wait-millis: 102
jmx-name-base: 1
jmx-name-prefix: 1
5.2.5 AliServiceImpl
AliServiceImpl是AbstractFileStrategy的子类,负责处理存储策略为阿里云OSS时的文件上传和删除操作。为了使程序能够动态选择具体的策略处理类,可以提供一个配置类,在配置类中定义AliServiceImpl,可以参照阿里云OSS官方提供的示例代码(https://help.aliyun.com/document_detail/32011.html?spm=a2c4g.11186623.6.769.652763282djHGw)进行文件的上传和删除。
具体代码如下:
package com.itheima.pinda.file.storage;
import cn.hutool.core.util.StrUtil;
import com.alibaba.fastjson.JSONObject;
import com.aliyun.oss.OSS;
import com.aliyun.oss.OSSClientBuilder;
import com.aliyun.oss.model.*;
import com.itheima.pinda.file.domain.FileDeleteDO;
import com.itheima.pinda.file.entity.File;
import com.itheima.pinda.file.properties.FileServerProperties;
import com.itheima.pinda.file.strategy.impl.AbstractFileStrategy;
import com.itheima.pinda.utils.StrPool;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Configuration;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import java.nio.file.Paths;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.UUID;
import static com.itheima.pinda.utils.DateUtils.DEFAULT_MONTH_FORMAT_SLASH;
/**
* 阿里云OSS配置
*/
@EnableConfigurationProperties(FileServerProperties.class)
@Configuration
@Slf4j
@ConditionalOnProperty(name = "pinda.file.type", havingValue = "ALI")
public class AliOssAutoConfigure {
/**
* 阿里云OSS文件策略处理类
*/
@Service
public class AliServiceImpl extends AbstractFileStrategy {
/**
* 构建阿里云OSS客户端
* @return
*/
private OSS buildClient() {
properties = fileProperties.getAli();
return new OSSClientBuilder().
build(properties.getEndpoint(),
properties.getAccessKeyId(),
properties.getAccessKeySecret());
}
protected String getUriPrefix() {
if (StringUtils.isNotEmpty(properties.getUriPrefix())) {
return properties.getUriPrefix();
} else {
String prefix = properties.
getEndpoint().
contains("https://") ? "https://" : "http://";
return prefix + properties.getBucketName() + "." +
properties.getEndpoint().replaceFirst(prefix, "");
}
}
/**
* 上传文件
* @param file
* @param multipartFile
* @throws Exception
*/
@Override
protected void uploadFile(File file, MultipartFile multipartFile)
throws Exception {
OSS client = buildClient();
//获得OSS空间名称
String bucketName = properties.getBucketName();
if (!client.doesBucketExist(bucketName)) {
//创建存储空间
client.createBucket(bucketName);
}
//生成文件名
String fileName = UUID.randomUUID().toString() +
StrPool.DOT +
file.getExt();
//日期文件夹,例如:2020\04
String relativePath =
Paths.get(LocalDate.now().
format(DateTimeFormatter.
ofPattern(DEFAULT_MONTH_FORMAT_SLASH))).
toString();
// web服务器存放的相对路径
String relativeFileName = relativePath + StrPool.SLASH + fileName;
relativeFileName = StrUtil.replace(relativeFileName, "\\\\",
StrPool.SLASH);
relativeFileName = StrUtil.replace(relativeFileName, "\\",
StrPool.SLASH);
//对象元数据
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentDisposition("attachment;fileName=" +
file.getSubmittedFileName());
metadata.setContentType(file.getContextType());
//上传请求对象
PutObjectRequest request =
new PutObjectRequest(bucketName, relativeFileName,
multipartFile.getInputStream(),
metadata);
//上传文件到阿里云OSS空间
PutObjectResult result = client.putObject(request);
log.info("result={}", JSONObject.toJSONString(result));
String url = getUriPrefix() + StrPool.SLASH + relativeFileName;
url = StrUtil.replace(url, "\\\\", StrPool.SLASH);
url = StrUtil.replace(url, "\\", StrPool.SLASH);
// 写入文件表
file.setUrl(url);
file.setFilename(fileName);
file.setRelativePath(relativePath);
file.setGroup(result.getETag());
file.setPath(result.getRequestId());
//关闭阿里云OSS客户端
client.shutdown();
}
/**
* 文件删除
* @param file
*/
@Override
protected void delete(FileDeleteDO file) {
OSS client = buildClient();
//获得OSS空间名称
String bucketName = properties.getBucketName();
// 删除文件
client.deleteObject(bucketName, file.getRelativePath() +
StrPool.SLASH + file.getFileName());
//关闭阿里云OSS客户端
client.shutdown();
}
}
}
通过上面代码可以看到要使用阿里云OSS提供的客户端OSS来实现文件的上传和删除,这就需要在文件服务对应的配置文件中进行如下配置:
pinda:
mysql:
database: pd_files
nginx:
ip: ${spring.cloud.client.ip-address} #正式环境要将该ip设置成nginx对应的公网ip
port: 10000 #正式环境需要将该ip设置成nginx对应的公网端口
file:
type: ALI
ali:
# 请填写自己的阿里云存储配置
bucket-name: bladex-loan
endpoint: http://oss-cn-qingdao.aliyuncs.com
access-key-id: LTAI4FhtimFAiz6iLGJSiJui
access-key-secret: SsU15qaPwpF1x5xMqwc0XzGuY92fnc
5.2.6 MinioServiceImpl
MinioServiceImpl是AbstractFileStrategy的子类,负责处理存储策略为Minio时的文件上传和删除操作。为了使程序能够动态选择具体的策略处理类,可以提供一个配置类,在配置类中定义MinioServiceImpl,具体代码如下:
package com.itheima.pinda.file.storage;
import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.json.JSONUtil;
import com.itheima.pinda.base.R;
import com.itheima.pinda.file.domain.FileDeleteDO;
import com.itheima.pinda.file.dto.BucketPolicyConfigDTO;
import com.itheima.pinda.file.dto.chunk.FileChunksMergeDTO;
import com.itheima.pinda.file.entity.File;
import com.itheima.pinda.file.properties.FileServerProperties;
import com.itheima.pinda.file.strategy.impl.AbstractFileChunkStrategy;
import com.itheima.pinda.file.strategy.impl.AbstractFileStrategy;
import com.itheima.pinda.utils.DateUtils;
import com.itheima.pinda.utils.StrPool;
import io.minio.BucketExistsArgs;
import io.minio.ComposeObjectArgs;
import io.minio.ComposeSource;
import io.minio.GetObjectArgs;
import io.minio.GetPresignedObjectUrlArgs;
import io.minio.ListObjectsArgs;
import io.minio.MakeBucketArgs;
import io.minio.MinioClient;
import io.minio.ObjectWriteArgs;
import io.minio.PutObjectArgs;
import io.minio.RemoveObjectArgs;
import io.minio.Result;
import io.minio.SetBucketPolicyArgs;
import io.minio.StatObjectArgs;
import io.minio.StatObjectResponse;
import io.minio.http.Method;
import io.minio.messages.Item;
import lombok.extern.slf4j.Slf4j;
import org.jetbrains.annotations.Nullable;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Configuration;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.FileInputStream;
import java.io.InputStream;
import java.io.SequenceInputStream;
import java.nio.ByteBuffer;
import java.nio.file.Paths;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import java.util.Vector;
import java.util.concurrent.TimeUnit;
/**
* @Author: 郭浩伟 qq:912161367
* @Date: 2022/11/6 0006 19:44
* @Description: minio
*/
@Configuration
@Slf4j
@EnableConfigurationProperties(FileServerProperties.class)
@ConditionalOnProperty(name = "pinda.file.type", havingValue = "MINIO")
public class MinioAutoConfigure {
private MinioClient minioClient;
private String bucketName;
private String endpoint;
private static final int DEFAULT_EXPIRY_TIME = 7 * 24 * 3600;
/**
* 构建minioClient
*
* @return
*/
private void buildClient(FileServerProperties fileProperties) {
//加载配置文件相关信息
FileServerProperties.Properties properties = fileProperties.getMinio();
endpoint = properties.getEndpoint();
String accessKey = properties.getAccessKey();
String secretKey = properties.getSecretKey();
this.bucketName = properties.getBucketName();
//创建一个MinIO的Java客户端
minioClient = MinioClient.builder()
.endpoint(endpoint)
.credentials(accessKey, secretKey)
.build();
}
/**
* 本地文件策略处理类
*/
@Service
public class MinioServiceImpl extends AbstractFileStrategy {
/**
* 文件上传抽象方法,需要由当前类的子类来实现
*
* @param file
* @param multipartFile
* @return
*/
@Override
public void uploadFile(File file, MultipartFile multipartFile) throws Exception {
MinioAutoConfigure.this.buildClient(fileProperties);
//生成文件名 此处并未用原始文件名multipartFile.getOriginalFilename(),因为同名文件会覆盖
String fileName = UUID.randomUUID() + StrPool.DOT + file.getExt();
String objectName = doReName(fileName, file);
MinioAutoConfigure.this.putObject(bucketName, multipartFile, objectName);
log.info("文件上传成功!");
}
/**
* 文件删除抽象方法,需要当前类的子类来实现
*
* @param fileDeleteDO
*/
@Override
public void delete(FileDeleteDO fileDeleteDO) throws Exception {
MinioAutoConfigure.this.buildClient(fileProperties);
// 设置存储对象名称
String objectName = Paths.get(fileDeleteDO.getRelativePath(), fileDeleteDO.getFileName()).toString();
// 替换掉windows环境的\路径
objectName = StrUtil.replace(objectName, "\\\\", StrPool.SLASH);
objectName = StrUtil.replace(objectName, "\\", StrPool.SLASH);
// 执行删除操作
if (bucketExists(bucketName)) {
minioClient.removeObject(RemoveObjectArgs.builder().bucket(bucketName).object(objectName).build());
}
}
}
}
通过上面的代码可以看到,在进行文件上传和文件删除时都会使用到配置文件中的配置项,关于Minio文件处理策略的配置如下:
pinda:
mysql:
database: pd_files
nginx:
ip: ${spring.cloud.client.ip-address} #正式环境要将该ip设置成nginx对应的公网ip
port: 10000 #正式环境需要将该ip设置成nginx对应的公网端口
file:
type: MINIO
minio:
endpoint: http://192.168.86.101:9000 #MinIO服务所在地址
bucketName: file #存储桶名称
accessKey: admin #访问的key
secretKey: admin123456 #访问的秘钥