前言
大文件分片上传和断点续传是为了解决在网络传输过程中可能遇到的问题，以提高文件传输的效率和稳定性。
- 首先，大文件分片上传是将大文件分割成较小的片段进行上传。这样做的好处是可以减少单个文件的传输时间，因为较小的文件片段更容易快速上传到目标服务器。同时，如果在传输过程中出现错误或中断，只需要重新上传出现问题的文件片段，而不需要重新上传整个文件，从而减少了传输的时间和带宽消耗。
- 其次，断点续传是指在文件传输过程中，如果传输被中断或者发生错误，可以从上一次中断的地方继续传输，而不是从头开始。这对于大文件的传输尤为重要，因为传输一个大文件可能需要较长的时间，而中断可能是由网络问题、电源故障、软件崩溃或其他因素引起的。断点续传功能允许用户在中断后恢复传输，而无需重新开始，节省了时间和资源。
大文件分片上传和断点续传在以下情况下尤为重要：
- 低带宽网络环境：在网络速度较慢或不稳定的情况下，将大文件分割为较小的片段进行上传可以降低传输的时间和失败的风险。
- 大文件传输：对于大文件，一次性完整上传可能需要很长时间，而且中途出现问题时需要重新传输整个文件，因此将文件分割并实现断点续传功能可以提高效率和可靠性。
- 网络中断或传输错误：网络中断、电源故障或软件崩溃等因素可能导致文件传输中断，断点续传功能可以从中断处恢复，避免重新传输整个文件。
- 多用户并发上传：在有多个用户同时上传文件的情况下，分片上传和断点续传可以减少对服务器资源的占用，提高并发传输的效率。
前端
采用百度的webuploader，在file.html中引用webuploader.js、jquery.js
代码如下：
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>大文件上传下载</title>
<link rel="stylesheet" type="text/css" href="webuploader.css">
<script src="jquery.js"></script>
<script src="webuploader.js"></script>
<style>
#upload-container {
width: 100px;
height: 50px;
background: #94d3e7;
padding-bottom: 10px;
}
</style>
</head>
<body>
<div id="upload-container"><span>文件拖拽上传</span></div>
<button id="picker" style="margin-top: 20px">分片上传</button>
<div id="upload-list"></div>
<hr/>
<a href="/file/download" >普通下载</a>
<hr/>
<a href="/file/downloads" target="_blank">分片下载</a>
</body>
<script>
$('#upload-container').click(function (event) {
$("#picker").find('input').click();
});
// 初始化上传组件
const uploader = WebUploader.create({
auto: true,
swf: 'Uploader.swf', // swf文件路径
server: '/file/upload', // 上传接口
dnd: '#upload-container',
pick: '#picker', // 内部根据当前运行创建
multiple: true, // 选择多个
chunked: true, // 开启分片
threads: 8, // 并发数，默认 3
chunkRetry: 8, // 如果遇到网络错误,重新上传次数
method: 'POST',
fileSizeLimit: 1024 * 1024 * 1024 * 10, // 文件总大小为10G
fileSingleSizeLimit: 1024 * 1024 * 1024 * 1, // 单个文件大小最大为1G
fileVal: 'upload'
});
// 入队之前触发事件
uploader.on("beforeFileQueued", function (file) {
// 获取文件后缀
console.log(file.name);
});
// 当有文件被添加进队列的时候
uploader.on('fileQueued', function (file) {
$('#upload-list').append( '<div id="' + file.id + '" class="item">' +
'<h4 class="info">' + file.name + '</h4>' +
'<p class="state">等待上传...</p>' +
'</div>' );
});
// 文件上传过程中创建进度条实时显示。
uploader.on('uploadProgress', function (file, percentage) {
var $li = $('#' + file.id),
$percent = $li.find('.progress .progress-bar');
// 避免重复创建
if (!$percent.length) {
$percent = $('<div class="progress progress-striped active">' +
'<div class="progress-bar" role="progressbar" style="width: 0%">' +
'</div>' +
'</div>').appendTo($li).find('.progress-bar');
}
$li.find('p.state').text('上传中');
$percent.css('width', percentage * 100 + '%');
});
uploader.on( 'uploadSuccess', function( file ) {
$( '#'+file.id ).find('p.state').text('已上传');
});
uploader.on( 'uploadError', function( file ) {
$( '#'+file.id ).find('p.state').text('上传出错');
});
uploader.on( 'uploadComplete', function( file ) {
$( '#'+file.id ).find('.progress').fadeOut();
});
</script>
</html>
后端
1.Pom文件添加依赖
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.7.4</version>
<relativePath/>
</parent>
<groupId>com.liyh</groupId>
<artifactId>springboot-file</artifactId>
<version>0.0.1</version>
<name>springboot-file</name>
<description>Demo project for Spring Boot</description>
<properties>
<java.version>1.8</java.version>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>commons-fileupload</groupId>
<artifactId>commons-fileupload</artifactId>
<version>1.3.1</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.4</version>
</dependency>
<!-- 做断点下载使用 -->
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.18.22</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.12.0</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>
2.yml配置文件
# 配置服务端口
server:
port: 8018
spring:
servlet:
multipart:
# Spring Boot中有默认的文件上传组件，在使用ServletFileUpload时需要关闭Spring Boot的默认配置
enabled: false
# 设置单个文件大小
max-file-size: 1GB
# 设置单次请求文件的总大小
max-request-size: 10GB
3..编写测试接口controller
package com.liyh.controller;
import com.liyh.service.FileService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
/**
* 文件上传测试接口
*
* @author liyh
*/
@RestController
@RequestMapping("/file")
public class FileController {
@Autowired
private FileService fileService;
/**
* 单个文件上传，支持断点续传
*/
@PostMapping("/upload")
public void upload(HttpServletRequest request, HttpServletResponse response) {
try {
fileService.upload(request, response);
} catch (Exception e) {
e.printStackTrace();
}
}
/**
* 普通文件下载
*/
@GetMapping("/download")
public void download(HttpServletRequest request, HttpServletResponse response) throws IOException {
fileService.download(request, response);
}
/**
* 分片文件下载
*/
@GetMapping("/downloads")
public String downloads() throws IOException {
fileService.downloads();
return "下载成功";
}
}
4.编写service
package com.liyh.service;
import com.liyh.entity.DownloadFileInfo;
import com.liyh.entity.FileInfo;
import com.liyh.entity.UploadFileInfo;
import org.apache.commons.fileupload.FileItem;
import org.apache.commons.fileupload.disk.DiskFileItemFactory;
import org.apache.commons.fileupload.servlet.ServletFileUpload;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.HttpClients;
import org.springframework.stereotype.Service;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.*;
import java.net.URLDecoder;
import java.net.URLEncoder;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@Service
public class FileService {
/**
* 编码
*/
private static final String UTF_8 = "UTF-8";
/**
* 文件上传路径(当前项目路径下，也可配置固定路径)
*/
private String uploadPath = System.getProperty("user.dir") + "/springboot-file/upload/";
/**
* 下载指定文件
*/
private String downloadFile = "D:\\Download\\git.exe";
/**
* 文件下载地址(当前项目路径下，也可配置固定路径)
*/
private String downloadPath = System.getProperty("user.dir") + "/springboot-file/download/";
/**
* 分片下载每一片大小为50M
*/
private static final Long PER_SLICE = 1024 * 1024 * 50L;
/**
* 定义分片下载线程池
*/
private ExecutorService executorService = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
/**
* final string
*/
private static final String RANGE = "Range";
/**
* 上传文件
*/
public void upload(HttpServletRequest request, HttpServletResponse response) throws Exception {
// 获取ServletFileUpload
ServletFileUpload servletFileUpload = getServletFileUpload();
List<FileItem> items = servletFileUpload.parseRequest(request);
// 获取文件信息
UploadFileInfo uploadFileInfo = getFileInfo(items);
// 写入临时文件
writeTempFile(items, uploadFileInfo);
// 判断是否合并
mergeFile(uploadFileInfo);
// 返回结果
response.setCharacterEncoding(UTF_8);
response.getWriter().write("上传成功");
}
/**
* 获取ServletFileUpload
*/
private ServletFileUpload getServletFileUpload() {
// 设置缓冲区大小，先读到内存里在从内存写
DiskFileItemFactory factory = new DiskFileItemFactory();
factory.setSizeThreshold(1024);
File file = new File(uploadPath);
// 如果文件夹不存在则创建
if (!file.exists() && !file.isDirectory()) {
file.mkdirs();
}
factory.setRepository(file);
// 解析
ServletFileUpload upload = new ServletFileUpload(factory);
// 设置单个大小与最大大小
upload.setFileSizeMax(1 * 1024 * 1024 * 1024L);
upload.setSizeMax(10 * 1024 * 1024 * 1024L);
return upload;
}
/**
* 获取文件信息
*
* @param items
* @return
* @throws UnsupportedEncodingException
*/
private UploadFileInfo getFileInfo(List<FileItem> items) throws UnsupportedEncodingException {
UploadFileInfo uploadFileInfo = new UploadFileInfo();
for (FileItem item : items) {
if (item.isFormField()) {
// 获取分片数据
if ("chunk".equals(item.getFieldName())) {
uploadFileInfo.setCurrentChunk(Integer.parseInt(item.getString(UTF_8)));
}
if ("chunks".equals(item.getFieldName())) {
uploadFileInfo.setChunks(Integer.parseInt(item.getString(UTF_8)));
}
if ("name".equals(item.getFieldName())) {
uploadFileInfo.setFileName(item.getString(UTF_8));
}
}
}
return uploadFileInfo;
}
/**
* 写入临时文件
*
* @param items
* @param uploadFileInfo
* @throws Exception
*/
private void writeTempFile(List<FileItem> items, UploadFileInfo uploadFileInfo) throws Exception {
// 获取文件基本信息后
for (FileItem item : items) {
if (!item.isFormField()) {
// 有分片需要临时目录
String tempFileName = uploadFileInfo.getFileName();
if (StringUtils.isNotBlank(tempFileName)) {
if (uploadFileInfo.getCurrentChunk() != null) {
tempFileName = uploadFileInfo.getCurrentChunk() + "_" + uploadFileInfo.getFileName();
}
// 判断文件是否存在
File tempFile = new File(uploadPath, tempFileName);
// 断点续传，判断文件是否存在，若存在则不传
if (!tempFile.exists()) {
item.write(tempFile);
}
}
}
}
}
/**
* 判断是否合并
*
* @param uploadFileInfo
* @throws IOException
* @throws InterruptedException
*/
private void mergeFile(UploadFileInfo uploadFileInfo) throws IOException, InterruptedException {
Integer currentChunk = uploadFileInfo.getCurrentChunk();
Integer chunks = uploadFileInfo.getChunks();
String fileName = uploadFileInfo.getFileName();
// 如果当前分片等于总分片那么合并文件
if (currentChunk != null && chunks != null && currentChunk.equals(chunks - 1)) {
File tempFile = new File(uploadPath, fileName);
try (BufferedOutputStream os = new BufferedOutputStream(new FileOutputStream(tempFile))) {
// 根据之前命名规则找到所有分片
for (int i = 0; i < chunks; i++) {
File file = new File(uploadPath, i + "_" + fileName);
// 并发情况，需要判断所有，因为可能最后一个分片传完，之前有的还没传完
while (!file.exists()) {
// 不存在休眠100毫秒后在重新判断
Thread.sleep(100);
}
// 分片存在，读入数组中
byte[] bytes = FileUtils.readFileToByteArray(file);
os.write(bytes);
os.flush();
file.delete();
}
os.flush();
}
}
}
/**
* 文件下载
*
* @param request
* @param response
* @throws IOException
*/
public void download(HttpServletRequest request, HttpServletResponse response) throws IOException {
// 获取文件
File file = new File(downloadFile);
// 获取下载文件信息
DownloadFileInfo downloadFileInfo = getDownloadFileInfo(file.length(), request, response);
// 设置响应头
setResponse(response, file.getName(), downloadFileInfo);
// 下载文件
try (InputStream is = new BufferedInputStream(new FileInputStream(file));
OutputStream os = new BufferedOutputStream(response.getOutputStream())) {
// 跳过已经读取文件
is.skip(downloadFileInfo.getPos());
byte[] buffer = new byte[1024];
long sum = 0;
// 读取
while (sum < downloadFileInfo.getRangeLength()) {
int length = is.read(buffer, 0, (downloadFileInfo.getRangeLength() - sum) <= buffer.length ? (int) (downloadFileInfo.getRangeLength() - sum) : buffer.length);
sum = sum + length;
os.write(buffer, 0, length);
}
}
}
/**
* 有两个map，我要去判断里面相同键的值一致不一致，除了双重for循环，有没有别的好办法
*/
private DownloadFileInfo getDownloadFileInfo(long fSize, HttpServletRequest request, HttpServletResponse response) {
long pos = 0;
long last = fSize - 1;
// 判断前端是否需要分片下载
if (request.getHeader(RANGE) != null) {
response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT);
String numRange = request.getHeader(RANGE).replace("bytes=", "");
String[] strRange = numRange.split("-");
if (strRange.length == 2) {
pos = Long.parseLong(strRange[0].trim());
last = Long.parseLong(strRange[1].trim());
// 若结束字节超出文件大小，取文件大小
if (last > fSize - 1) {
last = fSize - 1;
}
} else {
// 若只给一个长度，开始位置一直到结束
pos = Long.parseLong(numRange.replace("-", "").trim());
}
}
long rangeLength = last - pos + 1;
String contentRange = "bytes " + pos + "-" + last + "/" + fSize;
return new DownloadFileInfo(fSize, pos, last, rangeLength, contentRange);
}
/**
* 分片下载
*
* @throws IOException
*/
public void downloads() throws IOException {
File file = new File(downloadPath);
// 如果文件夹不存在则创建
if (!file.exists() && !file.isDirectory()) {
file.mkdirs();
}
// 探测下载，获取文件相关信息
FileInfo fileInfoDto = sliceDownload(1, 10, -1, null);
// 如果不为空，执行分片下载
if (fileInfoDto != null) {
// 计算有多少分片
long pages = fileInfoDto.getFileSize() / PER_SLICE;
// 适配最后一个分片
for (long i = 0; i <= pages; i++) {
long start = i * PER_SLICE;
long end = (i + 1) * PER_SLICE - 1;
executorService.execute(new SliceDownloadRunnable(start, end, i, fileInfoDto.getFileName()));
}
}
}
/**
* 分片下载
*
* @param start 分片起始位置
* @param end 分片结束位置
* @param page 第几个分片, page=-1时是探测下载
*/
private FileInfo sliceDownload(long start, long end, long page, String fName) throws IOException {
// 断点下载
File file = new File(downloadPath, page + "-" + fName);
// 如果当前文件已经存在，并且不是探测任务，并且文件的长度等于分片的大小，那么不用下载当前文件
if (file.exists() && page != -1 && file.length() == PER_SLICE) {
return null;
}
// 创建HttpClient
HttpClient client = HttpClients.createDefault();
HttpGet httpGet = new HttpGet("http://localhost:8018/file/download");
httpGet.setHeader(RANGE, "bytes=" + start + "-" + end);
HttpResponse httpResponse = client.execute(httpGet);
String fSize = httpResponse.getFirstHeader("fSize").getValue();
fName = URLDecoder.decode(httpResponse.getFirstHeader("fName").getValue(), UTF_8);
HttpEntity entity = httpResponse.getEntity();
// 下载
try (InputStream is = entity.getContent();
FileOutputStream fos = new FileOutputStream(file)) {
byte[] buffer = new byte[1024];
int ch;
while ((ch = is.read(buffer)) != -1) {
fos.write(buffer, 0, ch);
}
fos.flush();
}
// 判断是否是最后一个分片，如果是那么合并
if (end - Long.parseLong(fSize) > 0) {
mergeFile(fName, page);
}
return new FileInfo(Long.parseLong(fSize), fName);
}
private void mergeFile(String fName, long page) throws IOException {
File file = new File(downloadPath, fName);
try (BufferedOutputStream os = new BufferedOutputStream(new FileOutputStream(file))) {
for (int i = 0; i <= page; i++) {
File tempFile = new File(downloadPath, i + "-" + fName);
// 文件不存在或文件没写完
while (!tempFile.exists() || (i != page && tempFile.length() < PER_SLICE)) {
Thread.sleep(100);
}
byte[] bytes = FileUtils.readFileToByteArray(tempFile);
os.write(bytes);
os.flush();
tempFile.delete();
}
// 删除文件
File f = new File(downloadPath, "-1" + "-null");
if (f.exists()) {
f.delete();
}
} catch (InterruptedException e) {
e.printStackTrace();
}
}
private class SliceDownloadRunnable implements Runnable {
private final long start;
private final long end;
private final long page;
private final String fName;
private SliceDownloadRunnable(long start, long end, long page, String fName) {
this.start = start;
this.end = end;
this.page = page;
this.fName = fName;
}
@Override
public void run() {
try {
sliceDownload(start, end, page, fName);
} catch (IOException e) {
e.printStackTrace();
}
}
}
/**
* 设置响应头
*/
private void setResponse(HttpServletResponse response, String fileName, DownloadFileInfo downloadFileInfo) throws UnsupportedEncodingException {
response.setCharacterEncoding(UTF_8);
response.setContentType("application/x-download");
response.addHeader("Content-Disposition", "attachment;filename=" + URLEncoder.encode(fileName, UTF_8));
// 支持分片下载
response.setHeader("Accept-Range", "bytes");
response.setHeader("fSize", String.valueOf(downloadFileInfo.getFSize()));
response.setHeader("fName", URLEncoder.encode(fileName, UTF_8));
// range响应头
response.setHeader("Content-Range", downloadFileInfo.getContentRange());
response.setHeader("Content-Length", String.valueOf(downloadFileInfo.getRangeLength()));
}
}
5..编写上传文件实体类、文件信息实体类和下载文件实体类
package com.liyh.entity;
import lombok.Data;
@Data
public class UploadFileInfo {
/**
* 文件名称
*/
private String fileName;
/**
* 上传文件会有多个分片，记录当前为那个分片
*/
private Integer currentChunk;
/**
* 总分片数
*/
private Integer chunks;
}
package com.liyh.entity;
import lombok.AllArgsConstructor;
import lombok.NoArgsConstructor;
import lombok.Data;
@Data
@AllArgsConstructor
@NoArgsConstructor
public class FileInfo {
private long fileSize;
private String fileName;
}
package com.liyh.entity;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@AllArgsConstructor
@NoArgsConstructor
@Data
public class DownloadFileInfo {
/**
* 文件总大小
*/
private long fSize;
/**
* 断点起始位置
*/
private long pos;
/**
* 断点结束位置
*/
private long last;
/**
* rang响应
*/
private long rangeLength;
/**
* range响应
*/
private String contentRange;
}
运行效果
