springboot+minio实现分片上传(超大文件快速上传)
设计由来
在实际的项目开发中常遇到超大附件上传的情况，有时候客户会上传GB大小的文件，如果按照普通的
MultipartFile方式来接收上传的文件，那么无疑会把服务器给干崩溃，更别说并发操作了。
于是笔者决定要写一个超大附件上传的方法，于是有此。
功能实现图
功能介绍
-
上传请求异步操作，前端使用Worker线程处理，避免主线程阻塞
-
使用vue+springboot+minio实现方式
-
前端对大文件进行分片+后台进行合并
-
由于md5计算耗时太大故隐去改功能，md5可以实现妙传功能（校验文件是否存在）
-
支持多文件上传+文件夹上传（递归文件夹中的所有文件）
核心代码
<template>
<div class="container">
<h2>Minio 上传示例</h2>
<el-upload class="upload-demo"
ref="upload"
action="https://jsonplaceholder.typicode.com/posts/"
:on-remove="handleRemove"
:on-change="handleFileChange"
:file-list="uploadFileList"
:show-file-list="false"
:auto-upload="false">
<el-button slot="trigger" type="primary" plain>选择文件</el-button>
<el-button style="margin-left: 10px;" type="success" @click="handleUpload" plain>上传</el-button>
<el-button type="danger" @click="clearFileHandler" plain>清空</el-button>
</el-upload>
</div>
</div>
</template>
<script>import SparkMD5 from 'spark-md5'import axios from 'axios'const FILE_UPLOAD_ID_KEY = 'file_upload_id'const chunkSize = 10 * 1024 * 1024let currentFileIndex = 0const FileStatus = {
wait: '等待上传',
getMd5: '校验MD5',
uploading: '正在上传',
success: '上传成功',
error: '上传错误'}
export default {
data () { return {
changeDisabled: false,
uploadDisabled: false, // 上传并发数
simultaneousUploads: 3,
partCount:0,
uploadIdInfo: null,
uploadFileList: [],
retryList: []
}
},
methods: {
handleUpload() { const self = this
const files = this.uploadFileList if (files.length === 0) { this.$message.error('请先选择文件') return
} // 当前操作文件
const currentFile = files[currentFileIndex]
currentFile.status = FileStatus.getMd5 // 1. 计算MD5
this.getFileMd5(currentFile.raw, async (md5) => { // 2. 检查是否已上传 // const checkResult = await self.checkFileUploadedByMd5(md5) // // 已上传 // if (checkResult.data.status === 1) { // self.$message.success(`上传成功，文件地址：${checkResult.data.url}`) // console.log('文件访问地址：' + checkResult.data.url) // currentFile.status = FileStatus.success // currentFile.uploadProgress = 100 // return // } else if (checkResult.data.status === 2) { // "上传中" 状态 // // 获取已上传分片列表 // let chunkUploadedList = checkResult.data.chunkUploadedList // currentFile.chunkUploadedList = chunkUploadedList
// } else { // 未上传 // console.log('未上传') // }
console.log('文件MD5：' + md5) // 3. 正在创建分片
let fileChunks = self.createFileChunk(currentFile.raw, chunkSize)
let param = {
fileName: currentFile.name,
fileSize: currentFile.size,
chunkSize: chunkSize,
fileMd5: md5,
contentType: 'application/octet-stream',
partCount:this.partCount
} // 4. 获取上传url
let uploadIdInfoResult = await self.getFileUploadUrls(param)
self.uploadIdInfo = uploadIdInfoResult.data.uploadId
self.saveFileUploadId(uploadIdInfoResult.data.uploadId)
let uploadUrls = uploadIdInfoResult.data.uploadUrls if (fileChunks.length !== uploadUrls.length) {
self.$message.error('文件分片上传地址获取错误') return
}
self.$set(currentFile, 'chunkList', [])
fileChunks.map((chunkItem, index) => {
currentFile.chunkList.push({
chunkNumber: index + 1,
chunk: chunkItem,
uploadUrl: uploadUrls[index],
progress: 0,
status: '—'
})
})
let tempFileChunks = []
currentFile.chunkList.forEach((item) => {
tempFileChunks.push(item)
})
currentFile.status = FileStatus.uploading // 处理分片列表，删除已上传的分片
tempFileChunks = self.processUploadChunkList(tempFileChunks) // 5. 上传 await self.uploadChunkBase(tempFileChunks)
console.log('上传完成')
debugger // 6. 合并文件
const mergeResult = await self.mergeFile({
uploadId: self.uploadIdInfo,
fileName: currentFile.name,
md5: md5
}) if (!mergeResult.success) {
currentFile.status = FileStatus.error
self.$message.error(mergeResult.error)
} else {
currentFile.status = FileStatus.success
console.log('文件访问地址：' + mergeResult.data.url)
self.$message.success(`上传成功，文件地址：${mergeResult.data.url}`)
}
})
},
clearFileHandler() { this.uploadFileList = [] this.uploadIdInfo = null
},
handleFileChange(file, fileList) { this.uploadFileList = fileList this.uploadFileList.forEach((item) => { // 初始化自定义属性
this.initFileProperties(item)
})
},
initFileProperties(file) {
file.chunkList = []
file.status = FileStatus.wait
file.progressStatus = 'warning'
file.uploadProgress = 0
},
handleRemove(file, fileList) { this.uploadFileList = fileList
}, /**
* 分片读取文件 MD5 */
getFileMd5(file, callback) { const blobSlice = File.prototype.slice || File.prototype.mozSlice || File.prototype.webkitSlice const fileReader = new FileReader() // 计算分片数
const totalChunks = Math.ceil(file.size / chunkSize)
console.log('总分片数：' + totalChunks) this.partCount=totalChunks
let currentChunk = 0 const spark = new SparkMD5.ArrayBuffer()
loadNext()
fileReader.onload = function (e) { try {
spark.append(e.target.result)
} catch (error) {
console.log('获取Md5错误：' + currentChunk)
} if (currentChunk < totalChunks) {
currentChunk++
loadNext()
} else {
callback(spark.end())
}
}
fileReader.onerror = function () {
console.warn('读取Md5失败，文件读取错误')
}
function loadNext () { const start = currentChunk * chunkSize const end = ((start + chunkSize) >= file.size) ? file.size : start + chunkSize // 注意这里的 fileRaw fileReader.readAsArrayBuffer(blobSlice.call(file, start, end))
}
}, /**
* 文件分片 */
createFileChunk(file, size = chunkSize) { const fileChunkList = []
let count = 0 while(count < file.size) {
fileChunkList.push({
file: file.slice(count, count + size),
})
count += size
} return fileChunkList
}, /**
* 处理即将上传的分片列表，判断是否有已上传的分片，有则从列表中删除 */
processChunkList(chunkList) { const currentFile = this.uploadFileList[currentFileIndex]
let chunkUploadedList = currentFile.chunkUploadedList if (chunkUploadedList === undefined || chunkUploadedList === null || chunkUploadedList.length === 0) { return chunkList
} //
for (let i = chunkList.length - 1; i >= 0; i--) { const chunkItem = chunkList[i] for (let j = 0; j < chunkUploadedList.length; j++) { if (chunkItem.chunkNumber === chunkUploadedList[j]) {
chunkList.splice(i, 1) break
}
}
} return chunkList
},
uploadBase(chunkList) { const self = this
let successCount = 0
let totalChunks = chunkList.length return new Promise((resolve, reject) => { const handler = () => { if (chunkList.length) { const chunkItem = chunkList.shift() // 直接上传二进制，不需要构造 FormData，否则上传后文件损坏 axios.put(chunkItem.uploadUrl, chunkItem.chunk.file, { // 上传进度处理 onUploadProgress: self.checkChunkUploadProgress(chunkItem),
headers: { 'Content-Type': 'application/octet-stream'
}
}).then(response => { if (response.status === 200) {
console.log('分片：' + chunkItem.chunkNumber + ' 上传成功')
successCount++ // 继续上传下一个分片 handler()
} else {
console.log('上传失败：' + response.status + '，' + response.statusText)
}
}).catch(error => { // 更新状态
console.log('分片：' + chunkItem.chunkNumber + ' 上传失败，' + error) // 重新添加到队列中 chunkList.push(chunkItem)
handler()
})
} if (successCount >= totalChunks) {
resolve()
}
} // 并发
for (let i = 0; i < this.simultaneousUploads; i++) {
handler()
}
})
},
getFileUploadUrls(fileParam) {
debugger
let url = `http://127.0.0.1:8006/multipart/init`
return axios.post(url, fileParam)
},
saveFileUploadId(data) {
localStorage.setItem(FILE_UPLOAD_ID_KEY, data)
},
checkFileUploadedByMd5(md5) {
console.log(md5); // let url = `http://127.0.0.1:8006/upload/check?md5=${md5}` // return new Promise((resolve, reject) => { // axios.get(url).then((response) => { // resolve(response.data) // }).catch(error => { // reject(error) // }) // }) }, /**
* 合并文件 */
mergeFile(file) { const self = this
let url = `http://127.0.0.1:8006/multipart/complete`
return new Promise((resolve, reject) => {
axios.post(url,{ "uploadId":file.uploadId, "fileName":file.fileName, "md5":file.md5
}).then(response => {
let data = response.data if (!data.success) {
resolve(data)
} else {
file.status = FileStatus.success
resolve(data)
}
}).catch(error => {
self.$message.error('合并文件失败：' + error)
file.status = FileStatus.error
reject()
})
})
}, /**
* 检查分片上传进度 */
checkChunkUploadProgress(item) { return p => {
item.progress = parseInt(String((p.loaded / p.total) * 100)) this.updateChunkUploadStatus(item)
}
},
updateChunkUploadStatus(item) {
let status = FileStatus.uploading
let progressStatus = 'normal' if (item.progress >= 100) {
status = FileStatus.success
progressStatus = 'success'
}
let chunkIndex = item.chunkNumber - 1
let currentChunk = this.uploadFileList[currentFileIndex].chunkList[chunkIndex] // 修改状态
currentChunk.status = status
currentChunk.progressStatus = progressStatus // 更新状态
this.$set(this.uploadFileList[currentFileIndex].chunkList, chunkIndex, currentChunk) // 获取文件上传进度
this.getCurrentFileProgress()
},
getCurrentFileProgress() { const currentFile = this.uploadFileList[currentFileIndex] if (!currentFile || !currentFile.chunkList) { return
} const chunkList = currentFile.chunkList const uploadedSize = chunkList.map((item) => item.chunk.file.size * item.progress).reduce((acc, cur) => acc + cur) // 计算方式：已上传大小 / 文件总大小
let progress = parseInt((uploadedSize / currentFile.size).toFixed(2))
currentFile.uploadProgress = progress this.$set(this.uploadFileList, currentFileIndex, currentFile)
}
},
filters: {
transformByte(size) { if (!size) { return '0B'
} const unitSize = 1024 if (size < unitSize) { return size + ' B'
} // KB
if (size < Math.pow(unitSize, 2)) { return (size / unitSize).toFixed(2) + ' K';
} // MB
if (size < Math.pow(unitSize, 3)) { return (size / Math.pow(unitSize, 2)).toFixed(2) + ' MB'
} // GB
if (size < Math.pow(unitSize, 4)) { return (size / Math.pow(unitSize, 3)).toFixed(2) + ' GB';
} // TB
return (size / Math.pow(unitSize, 4)).toFixed(2) + ' TB';
}
}
}</script>
说明：由于篇幅有限仅提供核心内容部分， 最终效果：
示例下载：
