大文件上传分片技术
https://www.cnblogs.com/xiahj/p/8529545.html
npm install vue-simple-uploader --save
npm install --save spark-md5
参考
vue https://www.cnblogs.com/xiahj/p/vue-simple-uploader.html
vue前端代码
<template>
<div>
<uploader :key="uploader_key" :options="options" class="uploader-example"
@file-success="onFileSuccess" @file-progress="onFileProgress"
@file-added="onFileAdded" :autoStart="false"
:fileStatusText="{success: '上传成功', error: '上传失败', uploading: '正在上传', paused: '暂停上传', waiting: '等待上传'}">
<uploader-unsupport></uploader-unsupport>
<uploader-drop>
<uploader-btn :directory="true" :single="true">选择文件夹</uploader-btn>
<uploader-btn>选择文件</uploader-btn>
</uploader-drop>
<uploader-list></uploader-list>
</uploader>
<button @click="download">下载</button>
</div>
</template>
<script>
import {getFilePrimaryId,fileChunkMerge,getFileDetail,downLoadFile} from "@/api/upload";
import { getToken } from '@/utils/auth';
import FileSave from "file-saver";
export default {
data() {
return {
filename:'',
pieces:[],
fileBlog:null,
uploader_key: new Date().getTime(),//这个用来刷新组件--解决不刷新页面连续上传的缓存上传数据（注：每次上传时，强制这个值进行更改---根据自己的实际情况重新赋值）
options: {
target: `${window.CLIENT_COLLECTION_URL}/commonapi/fileChunk`,//SpringBoot后台接收文件夹数据的接口
testChunks: false,//是否分片-不分片
singleFile: true,//单文件上传
chunkSize: 30 * 1024 * 1024,
maxChunkRetries: 5,
simultaneousUploads: 1,
fileParameterName: 'file',
panelShow: false,
attrs: {
accept: "*"
},
query:{
uniqueId:'',
},
// 服务器分片校验函数，秒传及断点续传基础
checkChunkUploadedByResponse: function (chunk, message) {
//debugger
//console.log(chunk)
// let objMessage = JSON.parse(message);
// if (objMessage.skipUpload) {
// return true;
// }
// return null
},
headers: {
'X-Token':getToken(),
'Authorization':'eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJ7XCJwb3NpdGlvbnNcIjpbe1wiY29kZVwiOlwiMVwiLFwibGF5ZXJcIjowLFwicHJvdmlkZXJJZFwiOlwiY3JjY1wiLFwiaXNMZWFmXCI6MCxcImlkXCI6XCIxXCJ9XSxcInByb3ZpZGVySWRcIjpcImhyXCIsXCJpZFwiOlwiMzY5ODE4XCIsXCJuYW1lXCI6XCLnjovlu7rpgqZcIixcInJvbGVzXCI6W1wiTUFUX0FVRElUXCIsXCJNQVRfQ09ERVwiLFwiTUFUX1NPUlRcIixcIk1BVF9QRVJNSVNTSU9OXCIsXCJWSUVXRVJcIl19IiwiZXhwIjoxNjU5MTQ2NDQ0fQ.hZn61IIcBJ2rGZMHhA5FDO6PaR6enatWpGd0hdBBLTt9ApLTX105lC3lUez7UfORiWxBt3TVBhXd_bxC1NTvneuaBiXseGmQeRFuxTh2cF2cTsB_QLG1DxtpEqY_VIpHLcgmt4ON01iFLWKOcj_ZnBp6JFl6YJcXRHFDPLvud_er398aK1Jxw0z4sP8jkzQGLrdQSP50xQXL9ApJ6FjEX8-EBo9IyX2gdWxCu9ljMdNt0mDuLTMFeekA1IZHkm0ZkZadjCbUo3b0tBHpweyWyBO9d7nYFe8yPfwE0_ibnDzqRn5WUKyOrnEmniyI54oDbEqPP247Y4tGzn8Wt3r3-A'
},
},
};
},
mounted() {
},
methods: {
async onFileAdded(file) {
let res = await getFilePrimaryId();
this.options.query.uniqueId = res.data;
console.log(res.data)
},
async onFileSuccess(rootFile, file, response, chunk) {
//这里可以根据response（接口）返回的数据处理自己的实际问题（如：从response拿到后台返回的想要的数据进行组装并显示）
//注，这里从文件夹每上传成功一个文件会调用一次这个方法
await fileChunkMerge(this.options.query.uniqueId,rootFile.name,rootFile.size,JSON.parse(response).data.contenttype).then((res) => {
if (res && res.code == 200) {
console.log(res.data)
}
})
},
async download(){
await getFileDetail('1452528712738729985').then((res) => {
if (res && res.code == 200) {
var file = res.data
this.pieces = file.pieces
this.filename = file.name +'.'+file.suffix
}
})
var filepieces = this.pieces
this.fileBlog = null
for(var j = 0,len=filepieces.length; j < len; j++){
var piece = filepieces[j]
await downLoadFile(piece.id).then((res) => {
if(this.fileBlog == null){
this.fileBlog = new Blob([res],{ type: "binary/octet-stream;" })
}else{
this.fileBlog = new Blob([this.fileBlog,res],{ type: "binary/octet-stream;" })
}
});
}
debugger
FileSave.saveAs(this.fileBlog, this.filename);
},
onFileProgress(rootFile, file, chunk) {
console.log(`上传中 ${file.name}，chunk：${chunk.startByte / 1024 / 1024} ~ ${chunk.endByte / 1024 / 1024}`)
},
},
watch: {},
components: {}
}
</script>
<style>
.uploader-example {
width: 90%;
padding: 15px;
margin: 40px auto 0;
font-size: 12px;
box-shadow: 0 0 10px rgba(0, 0, 0, .4);
}
.uploader-example .uploader-btn {
margin-right: 4px;
}
.uploader-example .uploader-list {
max-height: 440px;
overflow: auto;
overflow-x: hidden;
overflow-y: auto;
}
</style>
java上传接收的实体类
import lombok.Data;
import org.springframework.web.multipart.MultipartFile;
import java.io.Serializable;
@Data
public class Chunk implements Serializable {
/**
* 当前文件块，从1开始
*/
private Integer chunkNumber;
/**
* 分块大小
*/
private Long chunkSize;
/**
* 当前分块大小
*/
private Long currentChunkSize;
/**
* 总大小
*/
private Long totalSize;
/**
* 文件标识
*/
private String identifier;
/**
* 文件名
*/
private String filename;
/**
* 相对路径
*/
private String relativePath;
/**
* 总块数
*/
private Integer totalChunks;
/**
* 二进制文件
*/
private MultipartFile file;
private String uniqueId;
}
java 后台接口
@GetMapping("/generatePrimayId")
@ApiOperation("分片上传主文件Id")
public Object generatePrimayId() throws Exception{
return new Result<Object>().ok(StringIdUtils.getId());
}
@Resource
private UploadService uploadService;
@PostMapping("/fileChunk")
@ApiOperation("上传一片文件")
public Object uploadBizFile(@ModelAttribute Chunk chunk,HttpServletResponse response) throws Exception{
MultipartFile file = chunk.getFile();
String path = fastdfsUtil.uploadFile(file.getBytes(),chunk.getIdentifier());
BAttachment attachment = BAttachment.Builder(DateUtils.format(new Date(),DateUtils.DATE_YEAR_PATTERN),"2",path,chunk.getIdentifier(),""+chunk.getChunkNumber());
attachment.setSaze(chunk.getChunkSize());
attachment.setContenttype(file.getContentType());
attachment.setFileId(chunk.getUniqueId());
attachment = attachmentService.insesrtAttachment(attachment);
return new Result<Object>().ok(attachment);
}
@PostMapping("/fileChunk/merge/{fileId}")
@ApiOperation("上传文件合并")
public Object mergeUploadBizFile(@PathVariable String fileId,@RequestParam Map<String,String> param) throws Exception{
String filename = param.get("name");
BAttachment attachment = BAttachment.Builder(DateUtils.format(new Date(),DateUtils.DATE_YEAR_PATTERN),"2","MERGE-PIECE",filename.substring(0,filename.lastIndexOf(".")),filename.substring(filename.lastIndexOf(".")+1));
attachment.setSaze(Long.parseLong(param.get("size")));
attachment.setContenttype(param.get("contenttype"));
attachment.setId(fileId);
attachment = attachmentService.insesrtAttachment(attachment);
return new Result<Object>().ok(attachment);
}
@GetMapping("/getBizFileInfo/{fileId}")
@ApiOperation("获取文件信息")
@ApiImplicitParams({
@ApiImplicitParam(name = "fileId", value = "文件ID", paramType = "query", required = true, dataType = "String"),
})
public Object getBizFileInfo(@PathVariable String fileId) throws Exception{
BAttachment attachment = attachmentService.getAttachment(fileId);
if(attachment.getPath().equals("MERGE-PIECE")){
attachment.setPieces(attachmentService.getPiecesAttachment(fileId));
}
return new Result<Object>().ok(attachment);
}
java后端代码传文件到本地
import org.apache.commons.io.FilenameUtils;
import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.multipart.MultipartHttpServletRequest;
import org.springframework.web.multipart.commons.CommonsMultipartResolver;
import javax.servlet.http.HttpServletRequest;
import java.io.*;
import java.util.Iterator;
import java.util.UUID;
/**
* 断点续传
* @author jwj
*/
public class Uploader {
/**
* 临时文件夹
*/
private String temporaryFolder;
/**
* 最大文件大小
*/
private Integer maxFileSize = 524288000;
// private String fileParameterName;
public Uploader(String temporaryFolder, String fileParameterName) {
this.temporaryFolder = temporaryFolder;
File file = new File(temporaryFolder);
if (!file.exists()) {
file.mkdirs();
}
// if (fileParameterName == null) {
// this.fileParameterName = "file";
// } else {
// this.fileParameterName = fileParameterName;
// }
}
public String cleanIdentifier(String identifier) {
return identifier.replaceAll("[^0-9A-Za-z_-]", "");
}
public String getChunkFilename(int chunkNumber, String identifier) {
identifier = cleanIdentifier(identifier);
return new File(temporaryFolder, "jwj-" + identifier + '-' + chunkNumber).getAbsolutePath();
}
public String validateRequest(int chunkNumber, int chunkSize, int totalSize, String identifier, String filename,
Integer fileSize) {
identifier = cleanIdentifier(identifier);
if (chunkNumber == 0 || chunkSize == 0 || totalSize == 0 || identifier.length() == 0
|| filename.length() == 0) {
return "non_uploader_request";
}
int numberOfChunks = (int) Math.max(Math.floor(totalSize / (chunkSize * 1.0)), 1);
if (chunkNumber > numberOfChunks) {
return "invalid_uploader_request1";
}
if (this.maxFileSize != null && totalSize > this.maxFileSize) {
return "invalid_uploader_request2";
}
if (fileSize != null) {
if (chunkNumber < numberOfChunks && fileSize != chunkSize) {
return "invalid_uploader_request3";
}
if (numberOfChunks > 1 && chunkNumber == numberOfChunks
&& fileSize != ((totalSize % chunkSize) + chunkSize)) {
return "invalid_uploader_request4";
}
if (numberOfChunks == 1 && fileSize != totalSize) {
return "invalid_uploader_request5";
}
}
return "valid";
}
public int getParamInt(HttpServletRequest req, String key, int def) {
String value = req.getParameter(key);
try {
return Integer.parseInt(value);
} catch (Exception e) {
}
return def;
}
public String getParamString(HttpServletRequest req, String key, String def) {
String value = req.getParameter(key);
try {
return value == null ? def : value;
} catch (Exception e) {
}
return def;
}
public void get(HttpServletRequest req, UploadListener listener) {
int chunkNumber = this.getParamInt(req, "chunkNumber", 0);
int chunkSize = this.getParamInt(req, "chunkSize", 0);
int totalSize = this.getParamInt(req, "totalSize", 0);
String identifier = this.getParamString(req, "identifier", "");
String filename = this.getParamString(req, "filename", "");
if (validateRequest(chunkNumber, chunkSize, totalSize, identifier, filename, null).equals("valid")) {
String chunkFilename = getChunkFilename(chunkNumber, identifier);
if (new File(chunkFilename).exists()) {
listener.callback("found", chunkFilename, filename, identifier, null);
} else {
listener.callback("not_found", null, null, null, null);
}
} else {
listener.callback("not_found", null, null, null, null);
}
}
public void post(HttpServletRequest req, UploadListener listener) throws IllegalStateException, IOException {
int chunkNumber = this.getParamInt(req, "chunkNumber", 0);
int chunkSize = this.getParamInt(req, "chunkSize", 0);
int totalSize = this.getParamInt(req, "totalSize", 0);
String identifier = this.getParamString(req, "identifier", "");
String filename = this.getParamString(req, "filename", "");
CommonsMultipartResolver multipartResolver = new CommonsMultipartResolver(req.getSession().getServletContext());
if (multipartResolver.isMultipart(req)) {
// 将request变成多部分request
MultipartHttpServletRequest multiRequest = (MultipartHttpServletRequest) req;
// 获取multiRequest 中所有的文件名
Iterator<String> iter = multiRequest.getFileNames();
while (iter.hasNext()) {
String name = iter.next().toString();
// if (!this.fileParameterName.equals(name)) {
// continue;
// }
MultipartFile file = multiRequest.getFile(name);
if (file != null && file.getSize() > 0) {
String original_filename = file.getOriginalFilename();
// String original_filename =
// files[this.fileParameterName]['originalFilename'];
String validation = validateRequest(chunkNumber, chunkSize, totalSize, identifier, filename,
(int) file.getSize());
if ("valid".equals(validation)) {
String chunkFilename = getChunkFilename(chunkNumber, identifier);
File f = new File(chunkFilename);
if (!f.exists()) {
file.transferTo(f);
}
int currentTestChunk = 1;
int numberOfChunks = (int) Math.max(Math.floor(totalSize / (chunkSize * 1.0)), 1);
currentTestChunk = this.testChunkExists(currentTestChunk, chunkNumber, numberOfChunks,
chunkFilename, original_filename, identifier, listener,"file");
} else {
listener.callback(validation, filename, original_filename, identifier,"file");
}
} else {
listener.callback("invalid_uploader_request", null, null, null, null);
}
}
}
}
private void pipeChunk(int number, String identifier, UploadOptions options, OutputStream writableStream)
throws IOException {
String chunkFilename = getChunkFilename(number, identifier);
if (new File(chunkFilename).exists()) {
FileInputStream inputStream = new FileInputStream(new File(chunkFilename));
int maxlen = 1024;
int len = 0;
try {
byte[] buff = new byte[maxlen];
while ((len = inputStream.read(buff, 0, maxlen)) > 0) {
writableStream.write(buff, 0, len);
}
} finally {
inputStream.close();
}
pipeChunk(number + 1, identifier, options, writableStream);
} else {
if (options.end)
writableStream.close();
if (options.listener != null)
options.listener.onDone();
}
}
public void write(String identifier, OutputStream writableStream, UploadOptions options) throws IOException {
if (options == null) {
options = new UploadOptions();
}
if (options.end == null) {
options.end = true;
}
pipeChunk(1, identifier, options, writableStream);
}
/**
*
* @param currentTestChunk
* @param chunkNumber 当前上传块
* @param numberOfChunks 总块数
* @param filename 文件名称
* @param original_filename 源文件名称
* @param identifier 文件
* @param listener 监听
* @param fileType
* @return
*/
private int testChunkExists(int currentTestChunk, int chunkNumber, int numberOfChunks, String filename,
String original_filename, String identifier, UploadListener listener, String fileType) {
String cfile = getChunkFilename(currentTestChunk, identifier);
if (new File(cfile).exists()) {
currentTestChunk++;
if (currentTestChunk >= chunkNumber) {
if (chunkNumber == numberOfChunks) {
try {
System.out.println("done");
// 文件合并
UploadOptions options = new UploadOptions();
File f = new File(this.temporaryFolder,
original_filename.substring(0,original_filename.lastIndexOf("."))+"-"+ UUID.randomUUID()+"." + FilenameUtils.getExtension(original_filename));
options.listener = new UploadDoneListener() {
@Override
public void onError(Exception err) {
listener.callback("invalid_uploader_request", f.getAbsolutePath(), original_filename,
identifier, fileType);
clean(identifier, null);
}
@Override
public void onDone() {
listener.callback("done", f.getAbsolutePath(), original_filename, identifier, fileType);
clean(identifier, null);
}
};
this.write(identifier, new FileOutputStream(f), options);
} catch (FileNotFoundException e) {
e.printStackTrace();
listener.callback("invalid_uploader_request", filename, original_filename, identifier,
fileType);
} catch (IOException e) {
e.printStackTrace();
listener.callback("invalid_uploader_request", filename, original_filename, identifier,
fileType);
}
} else {
listener.callback("partly_done", filename, original_filename, identifier, fileType);
}
} else {
return testChunkExists(currentTestChunk, chunkNumber, numberOfChunks, filename, original_filename,
identifier, listener, fileType);
}
} else {
listener.callback("partly_done", filename, original_filename, identifier, fileType);
}
return currentTestChunk;
}
public void clean(String identifier, UploadOptions options) {
if (options == null) {
options = new UploadOptions();
}
pipeChunkRm(1, identifier, options);
}
private void pipeChunkRm(int number, String identifier, UploadOptions options) {
String chunkFilename = getChunkFilename(number, identifier);
File file = new File(chunkFilename);
if (file.exists()) {
try {
file.delete();
} catch (Exception e) {
if (options.listener != null) {
options.listener.onError(e);
}
}
pipeChunkRm(number + 1, identifier, options);
} else {
if (options.listener != null)
options.listener.onDone();
}
}
public static interface UploadListener {
public void callback(String status, String filename, String original_filename, String identifier,
String fileType);
}
public static interface UploadDoneListener {
public void onDone();
public void onError(Exception err);
}
public static class UploadOptions {
public Boolean end;
public UploadDoneListener listener;
}
}
#################################实例类#######################################
import com.crcc.subaccess.common.util.Uploader;
import org.springframework.stereotype.Service;
@Service
public class UploadService extends Uploader {
public UploadService(){
super("C:/tmp/","file");
}
}
##################################上传代码##########################################
uploadService.post(request, new Uploader.UploadListener() {
@Override
public void callback(String status, String filename, String original_filename, String identifier, String fileType) {
if(status != null){
System.out.println(filename);
}
}
});
样式参数函数
