阿里云大文件分片上传
1:新建阿里云工具类
package com.bamboo.water_chivalry.project.file.controller; import com.aliyun.oss.OSSClient; import com.aliyun.oss.model.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.web.multipart.MultipartFile; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.List; /** * @PROJECT_NAME: water_chivalry * @AUTHOR: Hanson-Hsc * @DATE: 2020-07-15 11:16 * @DESCRIPTION: * @VERSION: */ public class AliyunOSSUpload implements Runnable{ private MultipartFile localFile; private long startPos; private long partSize; private int partNumber; private String uploadId; private static String key; private static String bucketName; // 新建一个List保存每个分块上传后的ETag和PartNumber protected static List<PartETag> partETags = Collections.synchronizedList(new ArrayList<PartETag>()); private static Logger logger = LoggerFactory.getLogger(FileUploader.class); private static OSSClient client = null; /** * 创建构造方法 * * @param localFile * 要上传的文件 * @param startPos * 每个文件块的开始 * @param partSize * @param partNumber * @param uploadId * 作为块的标识 * @param key * 上传到OSS后的文件名 */ public AliyunOSSUpload(MultipartFile localFile, long startPos, long partSize, int partNumber, String uploadId, String key , String bucketName) { this.localFile = localFile; this.startPos = startPos; this.partSize = partSize; this.partNumber = partNumber; this.uploadId = uploadId; AliyunOSSUpload.key = key; AliyunOSSUpload.bucketName = bucketName; } /** * 分块上传核心方法(将文件分成按照每个5M分成N个块，并加入到一个list集合中) */ @Override public void run() { InputStream instream = null; try { // 获取文件流 instream = localFile.getInputStream(); // 跳到每个分块的开头 instream.skip(this.startPos); // 创建UploadPartRequest，上传分块 UploadPartRequest uploadPartRequest = new UploadPartRequest(); uploadPartRequest.setBucketName(bucketName); uploadPartRequest.setKey(key); uploadPartRequest.setUploadId(this.uploadId); uploadPartRequest.setInputStream(instream); uploadPartRequest.setPartSize(this.partSize); uploadPartRequest.setPartNumber(this.partNumber); UploadPartResult uploadPartResult = FileUploader.client.uploadPart(uploadPartRequest); logger.info("Part#" + this.partNumber + " done\n"); synchronized (partETags) { // 将返回的PartETag保存到List中。 partETags.add(uploadPartResult.getPartETag()); } } catch (Exception e) { e.printStackTrace(); } finally { if (instream != null) { try { // 关闭文件流 instream.close(); } catch (IOException e) { e.printStackTrace(); } } } } /** * 初始化分块上传事件并生成uploadID，用来作为区分分块上传事件的唯一标识 * * @return */ protected static String claimUploadId(String bucketName, String key) { InitiateMultipartUploadRequest request = new InitiateMultipartUploadRequest(bucketName, key); InitiateMultipartUploadResult result = FileUploader.client.initiateMultipartUpload(request); logger.info(result.getUploadId()); return result.getUploadId(); } /** * 将文件分块进行升序排序并执行文件上传。 * * @param uploadId */ protected static void completeMultipartUpload(String uploadId) { // 将文件分块按照升序排序 Collections.sort(partETags, new Comparator<PartETag>() { @Override public int compare(PartETag p1, PartETag p2) { return p1.getPartNumber() - p2.getPartNumber(); } }); CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest(bucketName, key, uploadId, partETags); // 完成分块上传 FileUploader.client.completeMultipartUpload(completeMultipartUploadRequest); } /** * 列出文件所有分块的清单 * * @param uploadId */ protected static void listAllParts(String uploadId) { ListPartsRequest listPartsRequest = new ListPartsRequest(bucketName, key, uploadId); // 获取上传的所有分块信息 PartListing partListing = FileUploader.client.listParts(listPartsRequest); // 获取分块的大小 int partCount = partListing.getParts().size(); // 遍历所有分块 for (int i = 0; i < partCount; i++) { PartSummary partSummary = partListing.getParts().get(i); logger.info("分块编号 " + partSummary.getPartNumber() + ", ETag=" + partSummary.getETag()); } } }
2:新建文件上传接口,此处只需要controller
package com.bamboo.water_chivalry.project.file.controller; import com.alibaba.fastjson.JSONObject; import com.aliyun.oss.OSSClient; import com.bamboo.water_chivalry.common.enums.ResultEnum; import com.bamboo.water_chivalry.common.exception.GlobalException; import com.bamboo.water_chivalry.common.utils.JudgeVIFormat; import com.bamboo.water_chivalry.common.utils.OSSUtil; import com.bamboo.water_chivalry.common.utils.ResultVoUtil; import com.bamboo.water_chivalry.common.vo.other.ResultVo; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import org.apache.shiro.authz.annotation.Logical; import org.apache.shiro.authz.annotation.RequiresRoles; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.multipart.MultipartFile; import java.util.ArrayList; import java.util.List; import java.util.UUID; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import static com.bamboo.water_chivalry.common.config.FileConfig.*; import static com.bamboo.water_chivalry.common.constant.FileConstant.FILE_UPLOAD_SIZE; import static com.bamboo.water_chivalry.common.enums.ResultEnum.IMAGE_FILES_CANNOT_BE_UPLOADED; import static com.bamboo.water_chivalry.common.enums.ResultEnum.VIDEO_UPLOAD_ERROR; /** * @PROJECT_NAME: water_chivalry * @AUTHOR: Hanson-Hsc * @DATE: 2020-07-15 11:18 * @DESCRIPTION: * @VERSION: */ @RestController @RequestMapping("/file") @Api(tags = "大文件上传接口(多线程分片上传)") public class FileUploader { protected static OSSClient client = null; private static Logger logger = LoggerFactory.getLogger(FileUploader.class); @PostMapping("/thread") @ApiOperation(value = "视频分片上传")public static ResultVo fileUpload(@RequestParam("file") MultipartFile file) { // 创建一个可重用固定线程数的线程池。若同一时间线程数大于10，则多余线程会放入队列中依次执行 ExecutorService executorService = Executors.newFixedThreadPool(20); // 获取上传文件的名称，作为在OSS上的文件名 String key = file.getOriginalFilename(); String newFileName = UUID.randomUUID() + key.substring(key.lastIndexOf(".")); // 创建OSSClient实例 client = new OSSClient("oss-cn-shenzhen.aliyuncs.com", "你的accessKeyId", "你的access密钥");
try {
String uploadId = AliyunOSSUpload.claimUploadId(VIDEO_BUCKET_NAME, newFileName);
// 设置每块为 5M(除最后一个分块以外，其他的分块大小都要大于5MB)
final long partSize = 5 * 1024 * 1024L;
//final long partSize = 1024 * 1024L;
// 计算分块数目
long fileLength = file.getSize();
int partCount = (int) (fileLength / partSize);
if (fileLength % partSize != 0) {
partCount++;
}
// 分块 号码的范围是1~10000。如果超出这个范围，OSS将返回InvalidArgument的错误码。
if (partCount > BLOCK_SCOPE) {
throw new RuntimeException("文件过大(分块大小不能超过10000)");
} else {
logger.info("一共分了 " + partCount + " 块");
}
/**
* 将分好的文件块加入到list集合中
*/
for (int i = 0; i < partCount; i++) {
long startPos = i * partSize;
long curPartSize = (i + 1 == partCount) ? (fileLength - startPos) : partSize;
// 线程执行。将分好的文件块加入到list集合中
executorService.execute(new AliyunOSSUpload(file, startPos, curPartSize, i + 1, uploadId, newFileName, VIDEO_BUCKET_NAME));
}
/**
* 等待所有分片完毕
*/
// 关闭线程池（线程池不马上关闭），执行以前提交的任务，但不接受新任务。
executorService.shutdown();
// 如果关闭后所有任务都已完成，则返回 true。
while (!executorService.isTerminated()) {
try {
// 用于等待子线程结束，再继续执行下面的代码
executorService.awaitTermination(5, TimeUnit.SECONDS);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
/**
* partETags(上传块的ETag与块编号（PartNumber）的组合) 如果校验与之前计算的分块大小不同，则抛出异常
*/
System.out.println(AliyunOSSUpload.partETags.size() + " ----- " + partCount);
if (AliyunOSSUpload.partETags.size() != partCount) {
throw new IllegalStateException("OSS分块大小与文件所计算的分块大小不一致");
} else {
logger.info("将要上传的文件名 " + key + "\n");
}
/*
* 列出文件所有的分块清单并打印到日志中，该方法仅仅作为输出使用
*/
AliyunOSSUpload.listAllParts(uploadId);
/*
* 完成分块上传
*/
AliyunOSSUpload.completeMultipartUpload(uploadId);
JSONObject result = new JSONObject();
result.put("url", HTTPS + VIDEO_BUCKET_NAME + "." + END_POINT + "/" + client.getObject(VIDEO_BUCKET_NAME, newFileName).getKey());
// 返回上传文件的URL地址
return ResultVoUtil.success(result);
} catch (Exception e) {
logger.error(VIDEO_UPLOAD_ERROR.getMsg(), e);
return ResultVoUtil.error(ResultEnum.VIDEO_UPLOAD_ERROR);
} finally {
AliyunOSSUpload.partETags.clear();
if (client != null) {
client.shutdown();
}
}
}
}
