Java 大文件分片上传
原理：前端通过js读取文件，并将大文件按照指定大小拆分成多个分片，并且计算每个分片的MD5值。前端将每个分片分别上传到后端，后端在接收到文件之后验证当前分片的MD5值是否与上传的MD5一致，待所有分片上传完成之后后端将多个分片合并成一个大文件，并校验该文件的MD5值是否与上传时传入的MD5值一致；
首先是交互的控制器
支持文件分片上传，查询当前已经上传的分片信息，取消文件上传
package com.aimilin.component.system.service.modular.file.controller;
import com.aimilin.common.core.pojo.base.param.BaseParam;
import com.aimilin.common.core.pojo.response.ResponseData;
import com.aimilin.common.log.annotation.BusinessLog;
import com.aimilin.common.log.enums.LogOpTypeEnum;
import com.aimilin.common.security.annotation.Permission;
import com.aimilin.component.system.service.modular.file.param.SysPartFileParam;
import com.aimilin.component.system.service.modular.file.result.SysPartFileResult;
import com.aimilin.component.system.service.modular.file.service.SysPartFileService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RestController;
import javax.annotation.Resource;
/**
* 系统大文件上传
*
* @version V1.0
* @date 2022/5/24 11:22
*/
@Slf4j
@RestController
public class SysPartFileController {
@Resource
private SysPartFileService sysPartFileService;
/**
* 上传大文件
*
*/
@Permission
@PostMapping("/sysFileInfo/partUpload")
public ResponseData<SysPartFileResult> partUpload(@Validated(BaseParam.add.class) SysPartFileParam partFile) {
return ResponseData.success(sysPartFileService.partUpload(partFile));
}
/**
* 获取文件上传状态
*
*/
@Permission
@GetMapping("/sysFileInfo/partUpload/status")
public ResponseData<SysPartFileResult> getPartUploadStatus(@Validated(BaseParam.detail.class) SysPartFileParam partFile) {
return ResponseData.success(sysPartFileService.getPartUploadStatus(partFile));
}
/**
* 获取文件上传状态
*
*/
@Permission
@GetMapping("/sysFileInfo/partUpload/cancel")
@BusinessLog(title = "文件_上传大文件_取消", opType = LogOpTypeEnum.OTHER)
public ResponseData<SysPartFileResult> cancelUpload(@Validated(BaseParam.detail.class) SysPartFileParam partFile) {
return ResponseData.success(sysPartFileService.cancelUpload(partFile));
}
}
上传文件分片参数接收
如果按照分片方式上传文件需要指定当前大文件的MD5、分片MD5、分片内容、分片大小、当前文件名称、文件总大小等信息；另外对于每个文件前端都需要生成一个唯一编码用于确定当前上传的分片属于统一文件。
package com.aimilin.component.system.service.modular.file.param;
import java.io.Serializable;
import java.util.Objects;
import com.baomidou.mybatisplus.core.toolkit.StringUtils;
import com.aimilin.common.core.pojo.base.param.BaseParam;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
import org.springframework.web.multipart.MultipartFile;
import javax.validation.constraints.NotNull;
/**
* 大文件断点续传
*
* @version V1.0
* @date 2022/5/24 10:52
*/
@Getter
@Setter
@ToString
public class SysPartFileParam extends BaseParam implements Serializable {
/**
* 文件上传Id, 前端传入的值
*/
@NotNull(message = "uid不能为空", groups = {BaseParam.detail.class, BaseParam.add.class})
private String uid;
/**
* 上传文件名称
*/
private String filename;
/**
* 当前文件块，从1开始
*/
@NotNull(message = "partNumber不能为空", groups = {BaseParam.add.class})
private Integer partNumber;
/**
* 当前分块Md5
*/
@NotNull(message = "partMd5不能为空", groups = {BaseParam.add.class})
private String partMd5;
/**
* 分块大小，根据 totalSize 和这个值你就可以计算出总共的块数。注意最后一块的大小可能会比这个要大。
*/
@NotNull(message = "partSize不能为空", groups = {BaseParam.add.class})
private Long partSize;
/**
* 总大小
*/
@NotNull(message = "totalSize不能为空", groups = {BaseParam.add.class})
private Long totalSize;
/**
* 文件标识,MD5指纹
*/
@NotNull(message = "fileMd5不能为空", groups = {BaseParam.add.class})
private String fileMd5;
/**
* 二进制文件
*/
@NotNull(message = "file不能为空", groups = {BaseParam.add.class})
private MultipartFile file;
/**
* 总块数, （int）totalSize / partSize 最后一个模块要大一点；
*
* @return 结果
*/
public Integer getTotalParts() {
if (Objects.isNull(totalSize) || Objects.isNull(partSize)) {
return 0;
}
return new Double(Math.ceil(totalSize * 1.0 / partSize)).intValue();
}
public String getFilename() {
if (StringUtils.isBlank(this.filename) && Objects.isNull(this.file)) {
return null;
}
return StringUtils.isBlank(this.filename) ? this.file.getOriginalFilename() : this.filename;
}
}
至于代码中的 BaseParam 类，只是定义了一些验证的分组，类似以下代码：
/**
* 参数校验分组：分页
*/
public @interface page {
}
/**
* 参数校验分组：列表
*/
public @interface list {
}
/**
* 参数校验分组：下拉
*/
public @interface dropDown {
}
/**
* 参数校验分组：增加
*/
public @interface add {
}
大文件分片上传服务类实现
也是定义了三个接口，分片上传、查询当前已上传的分片、取消文件上传
package com.aimilin.component.system.service.modular.file.service;
import com.aimilin.component.system.service.modular.file.param.SysPartFileParam;
import com.aimilin.component.system.service.modular.file.result.SysPartFileResult;
/**
* 块文件上传
*
* @version V1.0
* @date 2022/5/24 10:59
*/
public interface SysPartFileService {
/**
* 文件块上传公共前缀
*/
public static final String PART_FILE_KEY = "PART_FILE";
/**
* 文件块上传
* 1. 将上传文件按照partSize拆分成多个文件块
* 2. 判断当前文件块是否已经上传
* 3. 未上传，则上传当前文本块
* 4. 已上传则不处理
* 5. 统计当前文本块上传进度信息
* 6. 判断所有文本块是否已经上传完成，如果上传完成则触发文件合并
*/
public SysPartFileResult partUpload(SysPartFileParam partFile);
/**
* 获取文件上传状态
*
* @param partFile 上传文件信息
* @return 文件上传状态结果
*/
public SysPartFileResult getPartUploadStatus(SysPartFileParam partFile);
/**
* 取消文件上传
*
* @param partFile 上传文件信息
* @return 文件上传状态结果
*/
public SysPartFileResult cancelUpload(SysPartFileParam partFile);
}
服务实现类：
package com.aimilin.component.system.service.modular.file.service.impl;
import cn.hutool.core.io.FileUtil;
import com.baomidou.mybatisplus.core.toolkit.IdWorker;
import com.aimilin.common.base.file.FilePartOperator;
import com.aimilin.common.base.file.param.AbortMultipartUploadResult;
import com.aimilin.common.base.file.param.CompleteFileUploadPart;
import com.aimilin.common.base.file.param.FileUploadPart;
import com.aimilin.common.base.file.param.FileUploadPartResult;
import com.aimilin.common.cache.RedisService;
import com.aimilin.common.core.consts.CommonConstant;
import com.aimilin.common.core.context.login.LoginContextHolder;
import com.aimilin.common.core.exception.ServiceException;
import com.aimilin.component.system.service.modular.file.convert.SysPartFileConvert;
import com.aimilin.component.system.service.modular.file.entity.SysFileInfo;
import com.aimilin.component.system.service.modular.file.enums.SysFileInfoExceptionEnum;
import com.aimilin.component.system.service.modular.file.enums.SysPartFileEnum;
import com.aimilin.component.system.service.modular.file.param.SysPartFileParam;
import com.aimilin.component.system.service.modular.file.result.SysPartFileCache;
import com.aimilin.component.system.service.modular.file.result.SysPartFileCache.FileInfo;
import com.aimilin.component.system.service.modular.file.result.SysPartFileCache.SysFilePart;
import com.aimilin.component.system.service.modular.file.result.SysPartFileResult;
import com.aimilin.component.system.service.modular.file.service.SysFileInfoService;
import com.aimilin.component.system.service.modular.file.service.SysPartFileService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FilenameUtils;
import org.redisson.api.RLock;
import org.redisson.api.RedissonClient;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import javax.annotation.Resource;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.TimeUnit;
import static com.aimilin.component.system.service.config.FileConfig.DEFAULT_BUCKET;
/**
* 大文件上传功能服务实现
*
* @version V1.0
* @date 2022/5/24 11:53
*/
@Slf4j
@Service
public class SysPartFileServiceImpl implements SysPartFileService {
@Resource
private FilePartOperator fileOperator;
@Resource
private RedisService redisService;
@Resource
private SysFileInfoService sysFileInfoService;
@Resource
private RedissonClient redisson;
/**
* 文件块上传
* 1. 将上传文件按照partSize拆分成多个文件块
* 2. 判断当前文件块是否已经上传
* 3. 未上传，则上传当前文本块
* 4. 已上传则不处理
* 5. 统计当前文本块上传进度信息
* 6. 判断所有文本块是否已经上传完成，如果上传完成则触发文件合并
*
* @param partFile 上传文件
* @return SysPartFileResult 文件上传结果
*/
@Override
public SysPartFileResult partUpload(SysPartFileParam partFile) {
MultipartFile file = partFile.getFile();
log.info("分块上传文件:{}, partNumber:{}/{}, partSize:{}/{}",
partFile.getFilename(), partFile.getPartNumber(), partFile.getTotalParts(), file.getSize(), partFile.getPartSize());
SysPartFileResult partUploadStatus = this.getPartUploadStatus(partFile);
// 已经上传该部分则直接返回当前文件状态
if (SysPartFileEnum.SUCCESS.getCode().equals(partUploadStatus.getPartState())) {
return partUploadStatus;
}
// 上传分片文件
FileUploadPart fileUploadPart = this.getFileUploadPart(partFile);
try {
FileUploadPartResult uploadPartResult = fileOperator.uploadPart(fileUploadPart);
this.setPartUploadStatus(partFile, uploadPartResult);
} catch (Exception e) {
log.error("文件分片上传失败，请求：{}：{}", partFile, e.getMessage(), e);
throw new ServiceException(SysFileInfoExceptionEnum.FILE_OSS_ERROR);
}
return this.getPartUploadStatus(partFile);
}
/**
* 获取文件上传状态
*
* @param partFile 上传文件信息
* @return 文件上传状态结果
*/
@Override
public SysPartFileResult getPartUploadStatus(SysPartFileParam partFile) {
SysPartFileCache fileCache = redisService.getCacheObject(getPartFileKey(partFile.getUid()));
SysPartFileResult result;
// 如果没有上传过则返回默认值
if (Objects.isNull(fileCache)) {
result = SysPartFileConvert.INSTANCE.toSysPartFileResult(partFile);
result.setFileState(SysPartFileEnum.NOT_EXISTS.getCode());
result.setPartState(SysPartFileEnum.NOT_EXISTS.getCode());
} else {
result = SysPartFileConvert.INSTANCE.toSysPartFileResult(fileCache, fileCache.getFilePart(partFile.getPartNumber()));
}
return result;
}
/**
* 取消文件上传
*
* @param partFile 上传文件信息
* @return 文件上传状态结果
*/
@Override
public SysPartFileResult cancelUpload(SysPartFileParam partFile) {
String cacheKey = getPartFileKey(partFile.getUid());
SysPartFileCache fileCache = redisService.getCacheObject(cacheKey);
if (Objects.isNull(fileCache)) {
throw new ServiceException(SysFileInfoExceptionEnum.NOT_EXISTED_FILE);
}
SysPartFileCache.FileInfo fileInfo = fileCache.getFileInfo();
fileOperator.abortMultipartUpload(fileInfo.getBucketName(), fileInfo.getObjectName(), fileInfo.getUploadId());
log.info("取消文件上传：{}", partFile.getUid());
SysPartFileResult sysPartFileResult = SysPartFileConvert.INSTANCE.toSysPartFileResult(partFile);
sysPartFileResult.setFileState(SysPartFileEnum.CANCELED.getCode());
redisService.deleteObject(cacheKey);
return sysPartFileResult;
}
/**
* 文件分片上传，设置文件分片信息
*
* @param partFile 分片文件参数
* @param uploadPartResult 文件上传结果信息
*/
private void setPartUploadStatus(SysPartFileParam partFile, FileUploadPartResult uploadPartResult) {
String redisKey = getPartFileKey(partFile.getUid());
if (!redisService.hasKey(redisKey)) {
throw new ServiceException(SysFileInfoExceptionEnum.FILE_CACHE_ERROR);
}
RLock lock = redisson.getLock(CommonConstant.getLockKey(redisKey));
try {
lock.lock();
SysPartFileCache fileCache = redisService.getCacheObject(redisKey);
Set<SysFilePart> filePartList = fileCache.getFilePartList();
if (Objects.isNull(filePartList)) {
filePartList = new HashSet<>();
fileCache.setFilePartList(filePartList);
}
SysFilePart sysFilePart = new SysFilePart();
sysFilePart.setPartNumber(partFile.getPartNumber());
sysFilePart.setPartState(SysPartFileEnum.SUCCESS.getCode());
sysFilePart.setPartMd5(partFile.getPartMd5());
sysFilePart.setPartSize(partFile.getFile().getSize());
sysFilePart.setFileUploadPartResult(uploadPartResult);
filePartList.add(sysFilePart);
fileCache.setFileState(SysPartFileEnum.UPLOADING.getCode());
// 所有文本块都已经上传完成
if (new HashSet<>(fileCache.getUploadedParts()).size() == fileCache.getTotalParts()) {
CompleteFileUploadPart completeFileUploadPart = SysPartFileConvert.INSTANCE.toCompleteFileUploadPart(fileCache);
fileOperator.completeMultipartUpload(completeFileUploadPart);
log.info("文件合并完成：{}，part: {}/{}", partFile.getFilename(), partFile.getPartNumber(), partFile.getTotalParts());
this.saveFileInfo(partFile, fileCache);
fileCache.setFileState(SysPartFileEnum.SUCCESS.getCode());
redisService.setCacheObject(redisKey, fileCache, 1L, TimeUnit.DAYS);
} else {
redisService.setCacheObject(redisKey, fileCache);
}
} catch (Exception e) {
log.error("设置文件分片上传状态异常，{},上传结果：{}", partFile, uploadPartResult, e);
throw new ServiceException(SysFileInfoExceptionEnum.PART_FILE_SET_STATE_ERROR);
}finally {
lock.unlock();
}
}
/**
* 保存文件信息到 数据库
*
* @param partFile 分片文件
* @param fileCache 文件缓存对象
*/
private void saveFileInfo(SysPartFileParam partFile, SysPartFileCache fileCache) {
SysFileInfo sysFileInfo = new SysFileInfo();
sysFileInfo.setId(Objects.isNull(fileCache.getFileId()) ? IdWorker.getId() : fileCache.getFileId());
sysFileInfo.setFileLocation(fileOperator.getFileLocation().getCode());
sysFileInfo.setFileBucket(fileCache.getFileInfo().getBucketName());
sysFileInfo.setFileOriginName(fileCache.getFilename());
sysFileInfo.setFileSuffix(FilenameUtils.getExtension(fileCache.getFileInfo().getObjectName()));
sysFileInfo.setFileSizeKb(SysFileUtils.getFileSizeKb(fileCache.getTotalSize()));
sysFileInfo.setFileSizeInfo(FileUtil.readableFileSize(fileCache.getTotalSize()));
sysFileInfo.setFileObjectName(fileCache.getFileInfo().getObjectName());
boolean save = sysFileInfoService.save(sysFileInfo);
log.info("保存文件信息完成：{}，结果：{}", partFile.getFilename(), save);
}
/**
* 获取文件上传分片信息
*
* @param partFile 分片文件参数
* @return 需要上传的分片文件信息
*/
private FileUploadPart getFileUploadPart(SysPartFileParam partFile) {
try {
SysPartFileCache fileCache = redisService.getCacheObject(getPartFileKey(partFile.getUid()));
if (Objects.isNull(fileCache)) {
fileCache = this.initSysPartFileCache(partFile);
}
return SysPartFileConvert.INSTANCE.toFileUploadPart(fileCache.getFileInfo(), partFile);
} catch (IOException e) {
log.error("获取文件分片对象异常：{}", e.getMessage(), e);
throw new ServiceException(SysFileInfoExceptionEnum.FILE_STREAM_ERROR);
}
}
/**
* 初始化文件缓存对象，进入该方法说明缓存为空
* @param partFile 分片文件
*/
private SysPartFileCache initSysPartFileCache(SysPartFileParam partFile) {
String key = getPartFileKey(partFile.getUid());
RLock lock = redisson.getLock(CommonConstant.getLockKey(key));
try {
lock.lock();
SysPartFileCache fileCache = redisService.getCacheObject(key);
if(Objects.isNull(fileCache)){
Long fileId = IdWorker.getId();
String objectName = SysFileUtils.getFileObjectName(partFile.getFilename(), fileId);
String uploadId = fileOperator.initiateMultipartUpload(DEFAULT_BUCKET, objectName);
fileCache = SysPartFileConvert.INSTANCE.toSysPartFileCache(partFile);
fileCache.setFileState(SysPartFileEnum.UPLOADING.getCode());
fileCache.setFileInfo(new FileInfo(DEFAULT_BUCKET, objectName, uploadId));
fileCache.setFileId(fileId);
redisService.setCacheObject(getPartFileKey(partFile.getUid()), fileCache);
}
return fileCache;
} catch (Exception e) {
log.error("文件缓存初始化异常：{}", partFile, e);
throw new ServiceException(SysFileInfoExceptionEnum.PART_FILE_INIT_CACHE_ERROR);
}finally {
lock.unlock();
}
}
/**
* 获取文件缓存key
*
* @param fileId 文件Id
* @return %s:%s:%s
*/
private String getPartFileKey(String fileId) {
return String.format("%s:%s:%s", PART_FILE_KEY, LoginContextHolder.me().getSysLoginUserId(), fileId);
}
}
文件分片上传定义公共服务类接口
package com.aimilin.common.base.file;
import com.aimilin.common.base.file.param.*;
/**
* 大文件分片操作服务类
*
* @version V1.0
* @date 2022/5/24 16:56
*/
public interface FilePartOperator extends FileOperator {
/**
* 初始化分片文件上传
*
* @param bucketName 文件桶
* @param key 文件key
* @return 本次文件上传唯一标识
*/
String initiateMultipartUpload(String bucketName, String key);
/**
* 上传分片文件
*
* @param fileUploadPart 分片文件参数
* @return 上传结果
*/
FileUploadPartResult uploadPart(FileUploadPart fileUploadPart);
/**
* 完成分片上传
*
* @param completeFileUploadPart 请求对象
* @return 结果信息
*/
CompleteFileUploadPartResult completeMultipartUpload(CompleteFileUploadPart completeFileUploadPart);
/**
* 取消文件分片上传
*
* @param bucketName 文件桶
* @param objectName 对象key
* @param uploadId 上传ID
* @return
*/
void abortMultipartUpload(String bucketName, String objectName, String uploadId);
}
/**
* 文件分片上传取消
*
* @version V1.0
* @date 2022/5/24 20:32
*/
public class AbortMultipartUploadResult {
}
/**
* 完成分片上传
*
* @version V1.0
* @date 2022/5/24 20:07
*/
@Getter
@Setter
@ToString
public class CompleteFileUploadPart implements Serializable {
private String bucketName;
private String objectName;
private String uploadId;
private List<FileUploadPartResult> partETags;
}
/**
* 分片上传结果
*
* @version V1.0
* @date 2022/5/24 20:08
*/
@Getter
@Setter
@ToString
public class CompleteFileUploadPartResult implements Serializable {
private String bucketName;
private String objectName;
private String location;
private String eTag;
}
/**
* 文件分片上传请求参数
*
* @version V1.0
* @date 2022/5/24 17:00
*/
@Getter
@Setter
@ToString
public class FileUploadPart implements Serializable {
/**
* 文件桶
*/
private String bucketName;
/**
* 文件key
*/
private String objectName;
/**
* 文件上传ID
*/
private String uploadId;
/**
* 分片大小，设置分片大小。除了最后一个分片没有大小限制，其他的分片最小为100 KB
*/
private Long partSize;
/**
* 设置分片号。每一个上传的分片都有一个分片号，取值范围是1~10000，如果超出此范围，OSS将返回InvalidArgument错误码。
*/
private Integer partNumber;
/**
* 分片Md5签名
*/
private String partMd5;
/**
* 分片文件内容
*/
@JsonIgnore
@JSONField(deserialize = false, serialize = false)
private InputStream partContent;
}
/**
* 文件分片上传结果
*
* @version V1.0
* @date 2022/5/24 17:01
*/
@Getter
@Setter
@ToString
public class FileUploadPartResult implements Serializable {
/**
* 分块编号
*/
private Integer partNumber;
/**
* 当前分片大小
*/
private Long partSize;
/**
* 上传结果tag
*/
private String partETag;
}
文件分片上传文件操作接口实现类
这里风两种实现，1：本地文件上传，2：oss对象存储方式分片上传
/**
* 本地文件上传操作
*
*/
@Slf4j
public class LocalFileOperator implements FilePartOperator {
@Override
public FileLocationEnum getFileLocation() {
return FileLocationEnum.LOCAL;
}
private final LocalFileProperties localFileProperties;
private String currentSavePath = "";
private Dict localClient;
public LocalFileOperator(LocalFileProperties localFileProperties) {
this.localFileProperties = localFileProperties;
initClient();
}
@Override
public void initClient() {
if (SystemUtil.getOsInfo().isWindows()) {
String savePathWindows = localFileProperties.getLocalFileSavePathWin();
if (!FileUtil.exist(savePathWindows)) {
FileUtil.mkdir(savePathWindows);
}
currentSavePath = savePathWindows;
} else {
String savePathLinux = localFileProperties.getLocalFileSavePathLinux();
if (!FileUtil.exist(savePathLinux)) {
FileUtil.mkdir(savePathLinux);
}
currentSavePath = savePathLinux;
}
localClient = Dict.create();
localClient.put("currentSavePath", currentSavePath);
localClient.put("localFileProperties", localFileProperties);
}
@Override
public void destroyClient() {
// empty
}
@Override
public Object getClient() {
// empty
return localClient;
}
@Override
public boolean doesBucketExist(String bucketName) {
String absolutePath = currentSavePath + File.separator + bucketName;
return FileUtil.exist(absolutePath);
}
@Override
public void setBucketAcl(String bucketName, BucketAuthEnum bucketAuthEnum) {
// empty
}
@Override
public boolean isExistingFile(String bucketName, String key) {
return FileUtil.exist(this.getAbsolutePath(bucketName, key));
}
@Override
public void storageFile(String bucketName, String key, byte[] bytes) {
// 判断bucket存在不存在
String bucketPath = currentSavePath + File.separator + bucketName;
if (!FileUtil.exist(bucketPath)) {
FileUtil.mkdir(bucketPath);
}
// 存储文件
FileUtil.writeBytes(bytes, this.getAbsolutePath(bucketName, key));
}
@Override
public void storageFile(String bucketName, String key, InputStream inputStream, long fileSize) {
// 判断bucket存在不存在
String bucketPath = currentSavePath + File.separator + bucketName;
if (!FileUtil.exist(bucketPath)) {
FileUtil.mkdir(bucketPath);
}
// 存储文件
FileUtil.writeFromStream(inputStream, this.getAbsolutePath(bucketName, key));
}
@Override
public byte[] getFileBytes(String bucketName, String key) {
// 判断文件存在不存在
String absoluteFile = this.getAbsolutePath(bucketName, key);
if (!FileUtil.exist(absoluteFile)) {
String message = StrUtil.format("文件不存在,bucket={},key={}", bucketName, key);
throw new FileServiceException(message);
} else {
return FileUtil.readBytes(absoluteFile);
}
}
@Override
public void setFileAcl(String bucketName, String key, BucketAuthEnum bucketAuthEnum) {
// empty
}
@Override
public void copyFile(String originBucketName, String originFileKey, String newBucketName, String newFileKey) {
// 判断文件存在不存在
String originFile = this.getAbsolutePath(originBucketName, originFileKey);
if (!FileUtil.exist(originFile)) {
String message = StrUtil.format("源文件不存在,bucket={},key={}", originBucketName, originFileKey);
throw new FileServiceException(message);
} else {
// 拷贝文件
String destFile = this.getAbsolutePath(newBucketName, newFileKey);
FileUtil.copy(originFile, destFile, true);
}
}
@Override
public String getFileAuthUrl(String bucketName, String key, Long timeoutMillis) {
// empty
return null;
}
@Override
public void deleteFile(String bucketName, String key) {
// 判断文件存在不存在
String file = this.getAbsolutePath(bucketName, key);
if (!FileUtil.exist(file)) {
return;
}
// 删除文件
FileUtil.del(file);
}
/**
* 初始化分片文件上传
*
* @param bucketName 文件桶
* @param key 文件key
* @return 本次文件上传唯一标识
*/
@Override
public String initiateMultipartUpload(String bucketName, String key) {
return FileNameUtil.getName(key);
}
/**
* 上传分片文件
*
* @param fileUploadPart 分片文件参数
* @return 上传结果
*/
@Override
public FileUploadPartResult uploadPart(FileUploadPart fileUploadPart) {
String partName = fileUploadPart.getObjectName() + "." + fileUploadPart.getPartNumber();
this.storageFile(fileUploadPart.getBucketName(), partName, fileUploadPart.getPartContent(), fileUploadPart.getPartSize());
FileUploadPartResult result = new FileUploadPartResult();
result.setPartNumber(fileUploadPart.getPartNumber());
result.setPartSize(fileUploadPart.getPartSize());
result.setPartETag(partName);
// TODO 正常文件上传完成之后需要验证文件的分片的MD5值是否与前端传入的值一样
return result;
}
/**
* 完成分片上传
*
* @param completeFileUploadPart 请求对象
* @return 结果信息
*/
@Override
public CompleteFileUploadPartResult completeMultipartUpload(CompleteFileUploadPart completeFileUploadPart) {
try {
List<FileUploadPartResult> partETags = completeFileUploadPart.getPartETags();
String path = this.getAbsolutePath(completeFileUploadPart.getBucketName(), completeFileUploadPart.getObjectName());
partETags.sort((o1, o2) -> {
String p1 = FileNameUtil.extName(o1.getPartETag());
String p2 = FileNameUtil.extName(o2.getPartETag());
return Integer.valueOf(p1).compareTo(Integer.valueOf(p2));
});
Files.createFile(Paths.get(path));
partETags.forEach(c -> {
try {
Path partPath = Paths.get(this.getAbsolutePath(completeFileUploadPart.getBucketName(), c.getPartETag()));
Files.write(Paths.get(path), Files.readAllBytes(partPath), StandardOpenOption.APPEND);
Files.delete(partPath);
} catch (IOException e) {
log.error("合并文件失败：{}", e.getMessage(), e);
throw new FileServiceException(e.getMessage());
}
});
// 文件合并完成之后需要校验文件的MD5值是否与前端传入的一致
return new CompleteFileUploadPartResult();
} catch (IOException e) {
log.error("合并文件失败：{}", e.getMessage(), e);
throw new FileServiceException(e.getMessage());
}
}
/**
* 取消文件分片上传
*
* @param bucketName 文件桶
* @param objectName 对象key
* @param uploadId 上传ID
* @return
*/
@Override
public void abortMultipartUpload(String bucketName, String objectName, String uploadId) {
try {
Path folder = Paths.get(this.getAbsolutePath(bucketName, objectName)).getParent();
String partName = objectName + ".";
Files.list(folder)
.filter(path -> StrUtil.contains(path.toString(), partName))
.forEach(path -> {
try {
Files.delete(path);
} catch (IOException e) {
log.warn("删除分片文件失败：{}", path);
}
});
} catch (IOException e) {
log.error("取消文件分片上传异常：{}", e.getMessage(), e);
throw new FileServiceException(e.getMessage());
}
}
/**
* 获取文件绝对路径
*
* @param bucketName 文件桶
* @param key 对象key
* @return
*/
private String getAbsolutePath(String bucketName, String key) {
return currentSavePath + File.separator + bucketName + File.separator + key;
}
}
OSS 阿里云对象存储 分片上传实现
/**
* 阿里云文件操作
*
*/
@Slf4j
public class AliyunFileOperator implements FilePartOperator {
@Override
public FileLocationEnum getFileLocation() {
return FileLocationEnum.ALIYUN;
}
/**
* 阿里云文件操作客户端
*/
private OSS ossClient;
/**
* 阿里云oss的配置
*/
private final AliyunOssProperties aliyunOssProperties;
public AliyunFileOperator(AliyunOssProperties aliyunOssProperties) {
this.aliyunOssProperties = aliyunOssProperties;
this.initClient();
}
@Override
public void initClient() {
String endpoint = aliyunOssProperties.getEndPoint();
String accessKeyId = aliyunOssProperties.getAccessKeyId();
String accessKeySecret = aliyunOssProperties.getAccessKeySecret();
// 创建OSSClient实例。
ossClient = new OSSClientBuilder().build(endpoint, accessKeyId, accessKeySecret);
}
@Override
public void destroyClient() {
ossClient.shutdown();
}
@Override
public Object getClient() {
return ossClient;
}
@Override
public boolean doesBucketExist(String bucketName) {
try {
return ossClient.doesBucketExist(bucketName);
} catch (OSSException e) {
throw new AliyunFileServiceException(e);
} catch (ClientException e) {
throw new AliyunFileServiceException(e);
}
}
@Override
public void setBucketAcl(String bucketName, BucketAuthEnum bucketAuthEnum) {
try {
if (bucketAuthEnum.equals(BucketAuthEnum.PRIVATE)) {
ossClient.setBucketAcl(bucketName, CannedAccessControlList.Private);
} else if (bucketAuthEnum.equals(BucketAuthEnum.PUBLIC_READ)) {
ossClient.setBucketAcl(bucketName, CannedAccessControlList.PublicRead);
} else if (bucketAuthEnum.equals(BucketAuthEnum.PUBLIC_READ_WRITE)) {
ossClient.setBucketAcl(bucketName, CannedAccessControlList.PublicReadWrite);
}
} catch (OSSException e) {
throw new AliyunFileServiceException(e);
} catch (ClientException e) {
throw new AliyunFileServiceException(e);
}
}
@Override
public boolean isExistingFile(String bucketName, String key) {
try {
return ossClient.doesObjectExist(bucketName, key);
} catch (OSSException e) {
throw new AliyunFileServiceException(e);
} catch (ClientException e) {
throw new AliyunFileServiceException(e);
}
}
@Override
public void storageFile(String bucketName, String key, byte[] bytes) {
try {
ossClient.putObject(bucketName, key, new ByteArrayInputStream(bytes));
} catch (OSSException e) {
throw new AliyunFileServiceException(e);
} catch (ClientException e) {
throw new AliyunFileServiceException(e);
}
}
@Override
public void storageFile(String bucketName, String key, InputStream inputStream, long fileSize) {
try {
String contentType = "application/octet-stream";
if (key.contains(".")) {
contentType = MimetypesFileTypeMap.getDefaultFileTypeMap().getContentType(key);
}
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentType(contentType);
metadata.setContentLength(fileSize);
PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, key, inputStream, metadata);
ossClient.putObject(putObjectRequest);
} catch (OSSException e) {
throw new AliyunFileServiceException(e);
} catch (ClientException e) {
throw new AliyunFileServiceException(e);
}
}
@Override
public byte[] getFileBytes(String bucketName, String key) {
InputStream objectContent = null;
try {
OSSObject ossObject = ossClient.getObject(bucketName, key);
objectContent = ossObject.getObjectContent();
return IoUtil.readBytes(objectContent);
} catch (OSSException e) {
throw new AliyunFileServiceException(e);
} catch (ClientException e) {
throw new AliyunFileServiceException(e);
} finally {
IoUtil.close(objectContent);
}
}
@Override
public void setFileAcl(String bucketName, String key, BucketAuthEnum bucketAuthEnum) {
try {
if (bucketAuthEnum.equals(BucketAuthEnum.PRIVATE)) {
ossClient.setObjectAcl(bucketName, key, CannedAccessControlList.Private);
} else if (bucketAuthEnum.equals(BucketAuthEnum.PUBLIC_READ)) {
ossClient.setObjectAcl(bucketName, key, CannedAccessControlList.PublicRead);
} else if (bucketAuthEnum.equals(BucketAuthEnum.PUBLIC_READ_WRITE)) {
ossClient.setObjectAcl(bucketName, key, CannedAccessControlList.PublicReadWrite);
}
} catch (OSSException e) {
throw new AliyunFileServiceException(e);
} catch (ClientException e) {
throw new AliyunFileServiceException(e);
}
}
@Override
public void copyFile(String originBucketName, String originFileKey, String newBucketName, String newFileKey) {
try {
ossClient.copyObject(originBucketName, originFileKey, newBucketName, newFileKey);
} catch (OSSException e) {
throw new AliyunFileServiceException(e);
} catch (ClientException e) {
throw new AliyunFileServiceException(e);
}
}
@Override
public String getFileAuthUrl(String bucketName, String key, Long timeoutMillis) {
try {
Date expiration = new Date(new Date().getTime() + timeoutMillis);
URL url = ossClient.generatePresignedUrl(bucketName, key, expiration);
return url.toString();
} catch (OSSException e) {
throw new AliyunFileServiceException(e);
} catch (ClientException e) {
throw new AliyunFileServiceException(e);
}
}
@Override
public void deleteFile(String bucketName, String key) {
ossClient.deleteObject(bucketName, key);
}
/**
* 初始化分片文件上传
*
* @param bucketName 文件桶
* @param key 文件key
* @return 本次文件上传唯一标识
*/
@Override
public String initiateMultipartUpload(String bucketName, String key) {
InitiateMultipartUploadRequest request = new InitiateMultipartUploadRequest(bucketName, key);
InitiateMultipartUploadResult result = ossClient.initiateMultipartUpload(request);
log.info("阿里云 初始化分片文件上传：{}", key);
return result.getUploadId();
}
/**
* 上传分片文件
*
* @param fileUploadPart 分片文件参数
* @return 上传结果
*/
@Override
public FileUploadPartResult uploadPart(FileUploadPart fileUploadPart) {
UploadPartRequest request = AliyunConvert.INSTANCE.convert(fileUploadPart);
UploadPartResult result = ossClient.uploadPart(request);
FileUploadPartResult convert = AliyunConvert.INSTANCE.convert(result);
convert.setPartSize(fileUploadPart.getPartSize());
log.info("阿里云 分片文件上传：{}，结果：{}", fileUploadPart, request);
return convert;
}
/**
* 完成分片上传
*
* @param completeFileUploadPart 请求对象
* @return 结果信息
*/
@Override
public CompleteFileUploadPartResult completeMultipartUpload(CompleteFileUploadPart completeFileUploadPart) {
List<PartETag> tags = new ArrayList<>();
for (FileUploadPartResult partETag : completeFileUploadPart.getPartETags()) {
tags.add(new PartETag(partETag.getPartNumber(), partETag.getPartETag()));
}
CompleteMultipartUploadRequest request = new CompleteMultipartUploadRequest(
completeFileUploadPart.getBucketName(),
completeFileUploadPart.getObjectName(),
completeFileUploadPart.getUploadId(),
tags
);
CompleteMultipartUploadResult result = ossClient.completeMultipartUpload(request);
log.info("京东云合并文件：{},结果：{}", completeFileUploadPart, result);
return AliyunConvert.INSTANCE.convert(result);
}
/**
* 取消文件分片上传
*
* @param bucketName 文件桶
* @param objectName 对象key
* @param uploadId 上传ID
* @return
*/
@Override
public void abortMultipartUpload(String bucketName, String objectName, String uploadId) {
AbortMultipartUploadRequest request = new AbortMultipartUploadRequest(bucketName, objectName, uploadId);
ossClient.abortMultipartUpload(request);
}
}
京东云对象存储实现
package com.aimilin.common.base.file.modular.jdcloud;
import cn.hutool.core.io.IoUtil;
import com.amazonaws.SdkClientException;
import com.amazonaws.services.s3.model.*;
import com.aimilin.common.base.file.FileOperator;
import com.aimilin.common.base.file.FilePartOperator;
import com.aimilin.common.base.file.common.enums.BucketAuthEnum;
import com.aimilin.common.base.file.common.enums.FileLocationEnum;
import com.aimilin.common.base.file.modular.jdcloud.exp.JdCloudFileServiceException;
import com.aimilin.common.base.file.modular.jdcloud.prop.JdCloudConvert;
import com.aimilin.common.base.file.modular.jdcloud.prop.JdCloudOssProperties;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.client.builder.AwsClientBuilder;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.ClientConfiguration;
import com.aimilin.common.base.file.param.*;
import lombok.extern.slf4j.Slf4j;
import javax.activation.MimetypesFileTypeMap;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.net.URL;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
/**
* 京东云对象存储
*
* @version V1.0
*/
@Slf4j
public class JdCloudFileOperator implements FilePartOperator {
@Override
public FileLocationEnum getFileLocation() {
return FileLocationEnum.JDCLOUD;
}
/**
* 京东云客户端
*/
private AmazonS3 ossClient;
/**
* 京东云oss的配置
*/
private final JdCloudOssProperties jdCloudOssProperties;
/**
*
* @param jdCloudOssProperties
*/
public JdCloudFileOperator(JdCloudOssProperties jdCloudOssProperties) {
this.jdCloudOssProperties = jdCloudOssProperties;
this.initClient();
}
@Override
public void initClient() {
ClientConfiguration config = new ClientConfiguration();
AwsClientBuilder.EndpointConfiguration endpointConfig =
new AwsClientBuilder.EndpointConfiguration(jdCloudOssProperties.getEndPoint(), jdCloudOssProperties.getSigningRegion());
AWSCredentials awsCredentials = new BasicAWSCredentials(jdCloudOssProperties.getAccessKeyID(),jdCloudOssProperties.getAccessKeySecret());
AWSCredentialsProvider awsCredentialsProvider = new AWSStaticCredentialsProvider(awsCredentials);
ossClient = AmazonS3Client.builder()
.withEndpointConfiguration(endpointConfig)
.withClientConfiguration(config)
.withCredentials(awsCredentialsProvider)
.disableChunkedEncoding()
.build();
}
@Override
public void destroyClient() {
ossClient.shutdown();
}
@Override
public Object getClient() {
return ossClient;
}
@Override
public boolean doesBucketExist(String bucketName) {
return ossClient.doesBucketExistV2(bucketName);
}
@Override
public void setBucketAcl(String bucketName, BucketAuthEnum bucketAuthEnum) {
try {
if (bucketAuthEnum.equals(BucketAuthEnum.PRIVATE)) {
ossClient.setBucketAcl(bucketName, CannedAccessControlList.Private);
} else if (bucketAuthEnum.equals(BucketAuthEnum.PUBLIC_READ)) {
ossClient.setBucketAcl(bucketName, CannedAccessControlList.PublicRead);
} else if (bucketAuthEnum.equals(BucketAuthEnum.PUBLIC_READ_WRITE)) {
ossClient.setBucketAcl(bucketName, CannedAccessControlList.PublicReadWrite);
}
} catch (Exception e) {
log.error("JdCloud-oss-设置预定义策略异常",e);
throw new JdCloudFileServiceException(e);
}
}
@Override
public boolean isExistingFile(String bucketName, String key) {
try {
return ossClient.doesObjectExist(bucketName, key);
} catch (Exception e) {
log.error("JdCloud-oss-判断是否存在文件异常",e);
throw new JdCloudFileServiceException(e);
}
}
@Override
public void storageFile(String bucketName, String key, byte[] bytes) {
try {
InputStream is = new ByteArrayInputStream(bytes);
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentType("text/plain");
metadata.setContentLength((long)bytes.length);
ossClient.putObject(bucketName, key, is, metadata);
} catch (Exception e) {
log.error("JdCloud-oss-存储文件异常",e);
throw new JdCloudFileServiceException(e);
}
}
@Override
public void storageFile(String bucketName, String key, InputStream inputStream, long fileSize) {
try {
String contentType = "application/octet-stream";
if (key.contains(".")) {
contentType = MimetypesFileTypeMap.getDefaultFileTypeMap().getContentType(key);
}
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentType(contentType);
metadata.setContentLength(fileSize);
ossClient.putObject(bucketName, key, inputStream, metadata);
} catch (Exception e) {
log.error("JdCloud-oss-存储文件异常",e);
throw new JdCloudFileServiceException(e);
}
}
@Override
public byte[] getFileBytes(String bucketName, String key) {
InputStream objectContent = null;
try {
S3Object s3Object = ossClient.getObject(bucketName, key);
objectContent = s3Object.getObjectContent();
return IoUtil.readBytes(objectContent);
} catch (Exception e) {
log.error("JdCloud-oss-获取某个bucket下的文件字节异常",e);
throw new JdCloudFileServiceException(e);
}finally {
IoUtil.close(objectContent);
}
}
@Override
public void setFileAcl(String bucketName, String key, BucketAuthEnum bucketAuthEnum) {
try {
if (bucketAuthEnum.equals(BucketAuthEnum.PRIVATE)) {
ossClient.setObjectAcl(bucketName, key, CannedAccessControlList.Private);
} else if (bucketAuthEnum.equals(BucketAuthEnum.PUBLIC_READ)) {
ossClient.setObjectAcl(bucketName, key, CannedAccessControlList.PublicRead);
} else if (bucketAuthEnum.equals(BucketAuthEnum.PUBLIC_READ_WRITE)) {
ossClient.setObjectAcl(bucketName, key, CannedAccessControlList.PublicReadWrite);
}
} catch (Exception e) {
log.error("JdCloud-oss-文件访问权限管理异常",e);
throw new JdCloudFileServiceException(e);
}
}
@Override
public void copyFile(String originBucketName, String originFileKey, String newBucketName, String newFileKey) {
try {
ossClient.copyObject(originBucketName, originFileKey, newBucketName, newFileKey);
} catch (Exception e) {
log.error("JdCloud-oss-拷贝文件异常",e);
throw new JdCloudFileServiceException(e);
}
}
@Override
public String getFileAuthUrl(String bucketName, String key, Long timeoutMillis) {
try {
Date expiration = new Date(System.currentTimeMillis() + timeoutMillis);
URL url = ossClient.generatePresignedUrl(bucketName, key, expiration);
return url.toString();
} catch (Exception e) {
log.error("JdCloud-oss-获取文件的下载地址异常",e);
throw new JdCloudFileServiceException(e);
}
}
@Override
public void deleteFile(String bucketName, String key) {
try {
ossClient.deleteObject(bucketName, key);
} catch (Exception e) {
log.error("JdCloud-oss-删除文件异常", e);
throw new JdCloudFileServiceException(e);
}
}
/**
* 初始化分片文件上传
*
* @param bucketName 文件桶
* @param key 文件key
* @return 本次文件上传唯一标识
*/
@Override
public String initiateMultipartUpload(String bucketName, String key) {
InitiateMultipartUploadRequest request = new InitiateMultipartUploadRequest(bucketName, key);
InitiateMultipartUploadResult initiateMultipartUploadResult = ossClient.initiateMultipartUpload(request);
log.info("京东云 初始化分片文件上传：{}", key);
return initiateMultipartUploadResult.getUploadId();
}
/**
* 上传分片文件
*
* @param fileUploadPart 分片文件参数
* @return 上传结果
*/
@Override
public FileUploadPartResult uploadPart(FileUploadPart fileUploadPart) {
UploadPartRequest request = JdCloudConvert.INSTANCE.convert(fileUploadPart);
UploadPartResult uploadPartResult = ossClient.uploadPart(request);
FileUploadPartResult result = JdCloudConvert.INSTANCE.convert(uploadPartResult.getPartETag());
result.setPartSize(fileUploadPart.getPartSize());
log.info("京东云 分片文件上传：{}，结果：{}", fileUploadPart, request);
return result;
}
/**
* 完成分片上传
*
* @param completeFileUploadPart 请求对象
* @return 结果信息
*/
@Override
public CompleteFileUploadPartResult completeMultipartUpload(CompleteFileUploadPart completeFileUploadPart) {
CompleteMultipartUploadRequest request = JdCloudConvert.INSTANCE.convert(completeFileUploadPart);
CompleteMultipartUploadResult result = ossClient.completeMultipartUpload(request);
log.info("京东云合并文件：{},结果：{}", completeFileUploadPart, result);
return JdCloudConvert.INSTANCE.convert(result);
}
/**
* 取消文件分片上传
*
* @param bucketName 文件桶
* @param objectName 对象key
* @param uploadId 上传ID
* @return
*/
@Override
public void abortMultipartUpload(String bucketName, String objectName, String uploadId) {
ossClient.abortMultipartUpload(new AbortMultipartUploadRequest(bucketName, objectName, uploadId));
}
}
腾讯云对象存储分片上传
package com.aimilin.common.base.file.modular.tencent;
import cn.hutool.core.io.IoUtil;
import com.aimilin.common.base.file.FilePartOperator;
import com.aimilin.common.base.file.common.enums.FileLocationEnum;
import com.aimilin.common.base.file.modular.aliyun.prop.AliyunConvert;
import com.aimilin.common.base.file.modular.tencent.prop.TenConvert;
import com.aimilin.common.base.file.param.CompleteFileUploadPart;
import com.aimilin.common.base.file.param.CompleteFileUploadPartResult;
import com.aimilin.common.base.file.param.FileUploadPart;
import com.aimilin.common.base.file.param.FileUploadPartResult;
import com.qcloud.cos.COSClient;
import com.qcloud.cos.ClientConfig;
import com.qcloud.cos.auth.BasicCOSCredentials;
import com.qcloud.cos.auth.COSCredentials;
import com.qcloud.cos.exception.CosClientException;
import com.qcloud.cos.exception.CosServiceException;
import com.qcloud.cos.http.HttpMethodName;
import com.qcloud.cos.model.*;
import com.qcloud.cos.region.Region;
import com.qcloud.cos.transfer.TransferManager;
import com.qcloud.cos.transfer.TransferManagerConfiguration;
import com.aimilin.common.base.file.FileOperator;
import com.aimilin.common.base.file.common.enums.BucketAuthEnum;
import com.aimilin.common.base.file.modular.tencent.exp.TencentFileServiceException;
import com.aimilin.common.base.file.modular.tencent.prop.TenCosProperties;
import lombok.extern.slf4j.Slf4j;
import javax.activation.MimetypesFileTypeMap;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.net.URL;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
/**
* 腾讯云内网文件操作
*
*/
@Slf4j
public class TenFileOperator implements FilePartOperator {
@Override
public FileLocationEnum getFileLocation() {
return FileLocationEnum.TENCENT;
}
private final TenCosProperties tenCosProperties;
private COSClient cosClient;
private TransferManager transferManager;
public TenFileOperator(TenCosProperties tenCosProperties) {
this.tenCosProperties = tenCosProperties;
initClient();
}
@Override
public void initClient() {
// 1.初始化用户身份信息
String secretId = tenCosProperties.getSecretId();
String secretKey = tenCosProperties.getSecretKey();
COSCredentials cred = new BasicCOSCredentials(secretId, secretKey);
// 2.设置 bucket 的区域, COS 地域的简称请参照 https://cloud.tencent.com/document/product/436/6224
Region region = new Region(tenCosProperties.getRegionId());
ClientConfig clientConfig = new ClientConfig(region);
// 3.生成 cos 客户端。
cosClient = new COSClient(cred, clientConfig);
// 4.线程池大小，建议在客户端与 COS 网络充足（例如使用腾讯云的 CVM，同地域上传 COS）的情况下，设置成16或32即可，可较充分的利用网络资源
// 对于使用公网传输且网络带宽质量不高的情况，建议减小该值，避免因网速过慢，造成请求超时。
ExecutorService threadPool = Executors.newFixedThreadPool(32);
// 5.传入一个 threadpool, 若不传入线程池，默认 TransferManager 中会生成一个单线程的线程池。
transferManager = new TransferManager(cosClient, threadPool);
// 6.设置高级接口的分块上传阈值和分块大小为10MB
TransferManagerConfiguration transferManagerConfiguration = new TransferManagerConfiguration();
transferManagerConfiguration.setMultipartUploadThreshold(10 * 1024 * 1024);
transferManagerConfiguration.setMinimumUploadPartSize(10 * 1024 * 1024);
transferManager.setConfiguration(transferManagerConfiguration);
}
@Override
public void destroyClient() {
cosClient.shutdown();
}
@Override
public Object getClient() {
return cosClient;
}
@Override
public boolean doesBucketExist(String bucketName) {
try {
return cosClient.doesBucketExist(bucketName);
} catch (CosServiceException e) {
throw new TencentFileServiceException(e);
} catch (CosClientException e) {
throw new TencentFileServiceException(e);
}
}
@Override
public void setBucketAcl(String bucketName, BucketAuthEnum bucketAuthEnum) {
try {
if (bucketAuthEnum.equals(BucketAuthEnum.PRIVATE)) {
cosClient.setBucketAcl(bucketName, CannedAccessControlList.Private);
} else if (bucketAuthEnum.equals(BucketAuthEnum.PUBLIC_READ)) {
cosClient.setBucketAcl(bucketName, CannedAccessControlList.PublicRead);
} else if (bucketAuthEnum.equals(BucketAuthEnum.PUBLIC_READ_WRITE)) {
cosClient.setBucketAcl(bucketName, CannedAccessControlList.PublicReadWrite);
}
} catch (CosServiceException e) {
throw new TencentFileServiceException(e);
} catch (CosClientException e) {
throw new TencentFileServiceException(e);
}
}
@Override
public boolean isExistingFile(String bucketName, String key) {
try {
cosClient.getObjectMetadata(bucketName, key);
return true;
} catch (CosServiceException e) {
return false;
}
}
@Override
public void storageFile(String bucketName, String key, byte[] bytes) {
// 根据文件名获取contentType
String contentType = "application/octet-stream";
if (key.contains(".")) {
contentType = MimetypesFileTypeMap.getDefaultFileTypeMap().getContentType(key);
}
// 上传文件
ByteArrayInputStream byteArrayInputStream = null;
try {
byteArrayInputStream = new ByteArrayInputStream(bytes);
ObjectMetadata objectMetadata = new ObjectMetadata();
objectMetadata.setContentType(contentType);
cosClient.putObject(bucketName, key, new ByteArrayInputStream(bytes), objectMetadata);
} catch (CosServiceException e) {
throw new TencentFileServiceException(e);
} catch (CosClientException e) {
throw new TencentFileServiceException(e);
} finally {
IoUtil.close(byteArrayInputStream);
}
}
@Override
public void storageFile(String bucketName, String key, InputStream inputStream, long fileSize) {
// 根据文件名获取contentType
String contentType = "application/octet-stream";
if (key.contains(".")) {
contentType = MimetypesFileTypeMap.getDefaultFileTypeMap().getContentType(key);
}
// 上传文件
try {
ObjectMetadata objectMetadata = new ObjectMetadata();
objectMetadata.setContentType(contentType);
objectMetadata.setContentLength(fileSize);
cosClient.putObject(bucketName, key, inputStream, objectMetadata);
} catch (CosServiceException e) {
throw new TencentFileServiceException(e);
} catch (CosClientException e) {
throw new TencentFileServiceException(e);
} finally {
IoUtil.close(inputStream);
}
}
@Override
public byte[] getFileBytes(String bucketName, String key) {
COSObjectInputStream cosObjectInput = null;
try {
GetObjectRequest getObjectRequest = new GetObjectRequest(bucketName, key);
COSObject cosObject = cosClient.getObject(getObjectRequest);
cosObjectInput = cosObject.getObjectContent();
return IoUtil.readBytes(cosObjectInput);
} catch (CosServiceException e) {
throw new TencentFileServiceException(e);
} catch (CosClientException e) {
throw new TencentFileServiceException(e);
} finally {
IoUtil.close(cosObjectInput);
}
}
@Override
public void setFileAcl(String bucketName, String key, BucketAuthEnum bucketAuthEnum) {
if (bucketAuthEnum.equals(BucketAuthEnum.PRIVATE)) {
cosClient.setObjectAcl(bucketName, key, CannedAccessControlList.Private);
} else if (bucketAuthEnum.equals(BucketAuthEnum.PUBLIC_READ)) {
cosClient.setObjectAcl(bucketName, key, CannedAccessControlList.PublicRead);
} else if (bucketAuthEnum.equals(BucketAuthEnum.PUBLIC_READ_WRITE)) {
cosClient.setObjectAcl(bucketName, key, CannedAccessControlList.PublicReadWrite);
}
}
@Override
public void copyFile(String originBucketName, String originFileKey, String newBucketName, String newFileKey) {
// 初始化拷贝参数
Region srcBucketRegion = new Region(tenCosProperties.getRegionId());
CopyObjectRequest copyObjectRequest = new CopyObjectRequest(
srcBucketRegion, originBucketName, originFileKey, newBucketName, newFileKey);
// 拷贝对象
try {
transferManager.copy(copyObjectRequest, cosClient, null);
} catch (CosServiceException e) {
throw new TencentFileServiceException(e);
} catch (CosClientException e) {
throw new TencentFileServiceException(e);
}
}
@Override
public String getFileAuthUrl(String bucketName, String key, Long timeoutMillis) {
GeneratePresignedUrlRequest presignedUrlRequest = new GeneratePresignedUrlRequest(bucketName, key, HttpMethodName.GET);
Date expirationDate = new Date(System.currentTimeMillis() + timeoutMillis);
presignedUrlRequest.setExpiration(expirationDate);
URL url = null;
try {
url = cosClient.generatePresignedUrl(presignedUrlRequest);
} catch (CosServiceException e) {
throw new TencentFileServiceException(e);
} catch (CosClientException e) {
throw new TencentFileServiceException(e);
}
return url.toString();
}
@Override
public void deleteFile(String bucketName, String key) {
cosClient.deleteObject(bucketName, key);
}
/**
* 初始化分片文件上传
*
* @param bucketName 文件桶
* @param key 文件key
* @return 本次文件上传唯一标识
*/
@Override
public String initiateMultipartUpload(String bucketName, String key) {
InitiateMultipartUploadRequest request = new InitiateMultipartUploadRequest(bucketName, key);
InitiateMultipartUploadResult result = cosClient.initiateMultipartUpload(request);
log.info("腾讯云 初始化分片文件上传：{}", key);
return result.getUploadId();
}
/**
* 上传分片文件
*
* @param fileUploadPart 分片文件参数
* @return 上传结果
*/
@Override
public FileUploadPartResult uploadPart(FileUploadPart fileUploadPart) {
UploadPartRequest request = TenConvert.INSTANCE.convert(fileUploadPart);
UploadPartResult result = cosClient.uploadPart(request);
FileUploadPartResult convert = TenConvert.INSTANCE.convert(result);
convert.setPartSize(fileUploadPart.getPartSize());
log.info("腾讯云 分片文件上传：{}，结果：{}", fileUploadPart, request);
return convert;
}
/**
* 完成分片上传
*
* @param completeFileUploadPart 请求对象
* @return 结果信息
*/
@Override
public CompleteFileUploadPartResult completeMultipartUpload(CompleteFileUploadPart completeFileUploadPart) {
List<PartETag> tags = new ArrayList<>();
for (FileUploadPartResult partETag : completeFileUploadPart.getPartETags()) {
tags.add(new PartETag(partETag.getPartNumber(), partETag.getPartETag()));
}
CompleteMultipartUploadRequest request = new CompleteMultipartUploadRequest(
completeFileUploadPart.getBucketName(),
completeFileUploadPart.getObjectName(),
completeFileUploadPart.getUploadId(),
tags);
CompleteMultipartUploadResult result = cosClient.completeMultipartUpload(request);
log.info("京东云合并文件：{},结果：{}", completeFileUploadPart, result);
return TenConvert.INSTANCE.convert(result);
}
/**
* 取消文件分片上传
*
* @param bucketName 文件桶
* @param objectName 对象key
* @param uploadId 上传ID
* @return
*/
@Override
public void abortMultipartUpload(String bucketName, String objectName, String uploadId) {
AbortMultipartUploadRequest request = new AbortMultipartUploadRequest(bucketName, objectName, uploadId);
cosClient.abortMultipartUpload(request);
}
}
分片上传前端代码实现：
<template>
<div class="upload__wrap" :class="`upload__wrap--${size}`">
<div class="files" v-for="img in existsImgs" :key="img.id">
<template v-if="pictureType.includes(handleType(img.fileSuffix))">
<!-- 图片类型 -->
<img style="object-fit: cover;" width="104" height="104" :src="handleImg(img.id, 208, 208)" />
<div class="btn__wraps">
<div class="btn__innerwraps">
<a-icon class="icon__btn" type="eye" @click="$refs.previewForm.preview({ id: img.id })" />
<a-popconfirm placement="topRight" title="确认删除？" @confirm="() => deleteImg(img.id)">
<a-icon class="icon__btn" type="delete" />
</a-popconfirm>
</div>
</div>
</template>
<template v-else-if="threedType.includes(handleType(img.fileSuffix))">
<img
style="object-fit: cover;cursor: pointer;"
width="104"
height="104"
src="https://aimilin.com/stata/test.png"
/>
<div class="btn__wraps">
<div class="btn__innerwraps">
<a-icon class="icon__btn" type="eye" @click="show3dModal(img)" />
<a-popconfirm placement="topRight" title="确认删除？" @confirm="() => deleteImg(img.id)">
<a-icon class="icon__btn" type="delete" />
</a-popconfirm>
</div>
</div>
</template>
<template v-else>
当前类型文件暂不支持预览
</template>
</div>
<div class="tempimg__placeholder" v-for="temp in tempImgArr" :key="temp.uid">上传中…</div>
<a-upload
name="upload"
:list-type="listType"
:file-list="fileList"
:accept="format"
:multiple="multiple"
:before-upload="beforeUpload"
:customRequest="customRequest"
>
<div v-if="existsImgs.length + tempImgArr.length < maxPicsLength">
<a-icon type="plus" />
<div class="ant-upload-text">
上传
</div>
</div>
</a-upload>
<preview-form ref="previewForm"></preview-form>
<preview3d-model :is3dModelShow="is3dModelShow" :carousel-lists="preview3dModel" title="3D模型预览" @closeModal="closeModal"></preview3d-model>
<!-- :carousel-lists="" -->
</div>
</template>
<script>
// import { sysFileInfoPage, sysFileInfoDelete, sysFileInfoPartUpload, sysFileInfoDownload } from '@/api/modular/system/fileManage'
import { sysFileInfoPartUpload } from '@/api/modular/system/fileManage'
import previewForm from '@/views/system/file/previewForm.vue'
import Preview3dModel from '@/views/system/file/preview3dmodel.vue'
import { handleImg } from '@/utils/util'
import SparkMD5 from 'spark-md5'
import { SUCCESS, SERVICE_ERROR, UPLOADING } from '@/assets/js/responseCode'
const SIZEUNIT = 1 * 1024 * 1024
export default {
components: {
previewForm,
Preview3dModel
},
props: {
isCloseUpload: {
type: Boolean,
default: false
},
size: {
type: String,
default: 'default'
},
format: {
type: String,
default: 'image/gif, image/jpeg, image/png, image/jpg'
},
listType: {
type: String,
default: 'picture-card'
},
maxPicsLength: {
type: Number,
default: 9
},
uploadText: {
type: String,
default: '上传'
},
existsImgs: {
type: Array,
default () {
return []
}
},
maxSize: {
type: Number,
default: 20
},
multiple: {
type: Boolean,
default: false
}
},
data() {
return {
pictureType: ['.gif', '.jpeg', '.png', '.jpg'],
threedType: ['.json', '.obj', '.dae', '.ply', '.gltf', '.stl', '.fbx'],
previewVisible: false,
previewImage: '',
fileList: [],
// loading: false,
is3dModelShow: false,
preview3dModel: [],
tempImgArr: [],
isStopUpload: false
}
},
create() {
this.timer = null
console.log('this', this)
},
watch: {
isCloseUpload: {
handler (newval) {
if (newval) {
this.$set(this, 'tempImgArr', [])
this.$emit('imgUploadingStatus', 0)
}
},
immediate: true
}
},
methods: {
handleImg,
show3dModal (obj) {
this.preview3dModel = [obj]
this.is3dModelShow = true
},
closeModal () {
this.is3dModelShow = false
},
handleType (filetType) {
return filetType.indexOf('.') > -1 ? filetType : '.' + filetType
},
beforeUpload(file, fileList) {
console.log('this', this)
return new Promise((resolve, reject) => {
let type = file.type
if (!type) {
type = '.' + file.name.split('.').pop()
}
const isFormatFiles = this.format.replace(/\s*/g, '').split(',').includes(type)
if (!isFormatFiles) {
this.$message.error(`只支持以下${this.format}格式！`)
return reject(new Error(`只支持以下${this.format}格式！`))
}
const maxSizeLimit = this.threedType.includes(type) ? 100 : 20
const isLtMaxSize = file.size / SIZEUNIT < maxSizeLimit
if (!isLtMaxSize) {
this.$message.error(`图片须小于${maxSizeLimit}MB!`)
return reject(new Error(`图片须小于${maxSizeLimit}MB!`))
}
// 是否上传图片超过最大限度
if (this.existsImgs.length + this.tempImgArr.length >= this.maxPicsLength) {
if (this.timer) {
clearTimeout(this.timer)
}
this.timer = setTimeout(() => {
this.$message.error(`最多只能上传${this.maxPicsLength}张！`)
}, 300)
return reject(new Error(`最多只能上传${this.maxPicsLength}张！`))
}
this.isStopUpload = false
// this.loading = true
this.$set(this, 'tempImgArr', [...this.tempImgArr, file.uid])
this.$emit('imgUploadingStatus', [...this.tempImgArr, file.uid].length)
this.$emit('resetUploadStatus')
resolve(true)
})
// return isFormatFiles && isLt2M
},
preview (id) {
this.$refs.previewForm.preview({ id })
},
deleteImg (id) {
this.$emit('deletePic', id)
},
/**
* 上传文件
*/
customRequest (data) {
const fileType = '.' + data.file.name.split('.').pop()
const fileReader = new FileReader()
const blobSlice = File.prototype.slice || File.prototype.mozSlice || File.prototype.webkitSlice
let currentChunk = 0
const chunkSize = 4 * 1024 * 1024
const chunks = Math.ceil(data.file.size / chunkSize)
const spark = new SparkMD5.ArrayBuffer()
const partChunksArr = []
const fileData = {}
loadNext()
fileReader.onload = e => {
spark.append(e.target.result)
const sparkChunk = new SparkMD5.ArrayBuffer()
sparkChunk.append(e.target.result)
const partMd5 = sparkChunk.end()
partChunksArr.push({
file: fileData[currentChunk],
partNumber: currentChunk + 1,
partMd5,
partSize: chunkSize,
totalSize: data.file.size
})
currentChunk++
if (currentChunk < chunks) {
loadNext()
} else {
const md5 = spark.end()
this.finalUploadFn(partChunksArr, fileType, data, md5)
}
}
fileReader.onerror = function () {
this.$message.error(`文件${data.file.name}读取出错，请检查该文件`)
// data.cancel()
}
function loadNext() {
const start = currentChunk * chunkSize
const end = ((start + chunkSize) >= data.file.size) ? data.file.size : start + chunkSize
const currentChunkData = blobSlice.call(data.file, start, end)
fileReader.readAsArrayBuffer(currentChunkData)
fileData[currentChunk] = currentChunkData
}
},
finalUploadFn (formData, fileType, data, wholeFileMd5) {
formData.forEach(item => {
const newFormData = new FormData()
// newFormData.set('file', data.file)
newFormData.set('uid', data.file.uid)
newFormData.set('filename', data.file.name)
Object.keys(item).forEach(key => {
newFormData.set(
key, item[key]
)
newFormData.set('fileMd5', wholeFileMd5)
})
if (this.isStopUpload) {
return
}
sysFileInfoPartUpload(newFormData).then((res) => {
// this.loading = false
if (res.code === SUCCESS && res.data?.fileState === SUCCESS) {
this.$emit('getNewPics', {
id: res.data.fileId,
fileSuffix: fileType
})
const newTempImgArr = this.tempImgArr.filter(item => item !== res.data?.uid)
this.$set(this, 'tempImgArr', newTempImgArr)
this.$emit('imgUploadingStatus', newTempImgArr.length)
// this.$refs.table.refresh()
} else if (res.code === SUCCESS && res.data?.fileState === UPLOADING) {
} else if (res.code === SUCCESS && res.data?.fileState === SERVICE_ERROR) {
if (!this.failupload) {
this.failupload = {}
this.failupload[data.file.uid] = data.file.uid
sysFileInfoPartUpload(newFormData)
} else {
if (!this.failupload[data.file.uid]) {
sysFileInfoPartUpload(newFormData)
this.failupload[data.file.uid] = data.file.uid
}
}
} else if (res.code !== SUCCESS) {
// 上传失败，从占位图中移除一个
const newTempImgArr = this.tempImgArr
newTempImgArr.pop()
this.$set(this, 'tempImgArr', newTempImgArr)
this.$emit('imgUploadingStatus', newTempImgArr.length)
if (this.timer) {
clearTimeout(this.timer)
}
this.timer = setTimeout(() => {
this.$message.error('上传失败！' + res.message)
}, 300)
}
}).catch(e => {
const newTempImgArr = this.tempImgArr
newTempImgArr.pop()
this.$set(this, 'tempImgArr', newTempImgArr)
this.$emit('imgUploadingStatus', newTempImgArr.length)
console.log('error', e)
// this.loading = false
// this.tempImgArr.length && this.$message.error('上传失败，请重新上传')
}).finally((p) => {
console.log('sysFileInfoPartUpload', p)
// this.loading = false
})
})
},
clearTimer() {
clearTimeout(this.timer)
this.$set(this, 'tempImgArr', [])
this.$emit('imgUploadingStatus', 0)
this.isStopUpload = true
}
},
beforeDestoryed() {
this.clearTimer()
}
}
</script>
<style>
/* you can make up upload button and sample style by using stylesheets */
.ant-upload-select-picture-card i {
font-size: 32px;
color: #999;
}
.ant-upload-select-picture-card .ant-upload-text {
margin-top: 8px;
color: #666;
}
</style>
<style lang="less" scoped>
.upload__wrap{
display: -webkit-inline-box;
display: -moz-inline-box;
display: inline-box;
flex-wrap: wrap;
.files{
position: relative;
width:104px;
height: 104px;
margin-right: 10px;
margin-bottom: 10px;
.btn__wraps{
position: absolute;
left: 0;
top: 0;
width: 100%;
height: 100%;
background: rgba(0,0,0,0);
display: flex;
align-items: center;
justify-content: center;
transition: all 0.3s linear;
z-index: -1;
.btn__innerwraps{
display: flex;
.icon__btn{
margin-right: 10px;
font-size: 16px;
color: rgba(255,255,255, 0);
cursor: pointer;
&:last-child{
margin-right: 0;
}
}
}
}
&:hover{
.btn__wraps{
background: rgba(0,0,0,0.5);
transition: all 0.3s linear;
z-index: 1;
.btn__innerwraps{
.icon__btn{
color: rgba(255,255,255, 0.8);
}
}
}
}
}
.tempimg__placeholder{
width: 104px;
height: 104px;
display: flex;
justify-content: center;
align-items: center;
border: 1px solid #d9d9d9;
margin-right: 10px;
margin-bottom: 10px;
}
}
</style>
参考文章：http://blog.ncmem.com/wordpress/2023/12/14/java-%e5%a4%a7%e6%96%87%e4%bb%b6%e5%88%86%e7%89%87%e4%b8%8a%e4%bc%a0/
欢迎入群一起讨论
