Java Implementation of breakpoint fragment continuation (backend part)
originally
Company doing recently a large file upload needs, due to the use of frameworks have limitations, so the maximum file cannot be more than 100 m, so if you use before way directly to upload the files directly, will direct error, so I think can shard large files, files uploaded to the backend individually, then after the file upload, Files are merged in the background and then uploaded to the specified server.
The solution
Using large file fragmentation breakpoint continuingly, namely front-end to divide a large file, divided into multiple small files, then send, in turn, the background, receipt of a request to each request sent by the content of the files generated a small file, and then stored in a temporary directory, the current end after uploading the file, the back-end to merge small files generated, Then upload to the specified service (file storage server such as FTP or OSS).
implementation
1, Write a controller interface (method only)
/ * * * *@paramRequest name File name, MD5 of a large file, current Number of fragments, * Total Number of fragments, file is the data of a small file, and this is the bloB type field * transmitted from the front end@return
* @throws IOException
*/
@RequestMapping(value = "/uploadFile", method = {RequestMethod.POST})
public R uploadFile(HttpServletRequest request) throws IOException {
R r = new R();
String fileName = request.getParameter("name");
String md5 = request.getParameter("md5");
String basePath = request.getSession().getServletContext().getRealPath("/") + "temp" + File.separatorChar + md5;
File file= new File(basePath);
if(! file.exists()) { file.mkdirs(); } System.out.println("File directory is:"+ basePath);
File[] files = file.listFiles();
Integer current = Integer.parseInt(request.getParameter("current"));
// Total number of films
Integer total = Integer.parseInt(request.getParameter("total"));
int combineFlag = 0;
// Check whether the file already has related slices. If so, skip the existing slices and start uploading new slices
if(files.length >= current) {
// If the total number of slices in the file is the same as the data passed from the front-end to slice, the file is directly merged
if(file.length() >= total) {
String fileUrl = null;
try {
// Merge files
fileUrl = combineFiles(fileName, basePath);
if(StringUtils.isNotBlank(fileUrl)) {
r.put("filePath", fileUrl);
combineFlag = 1; }}catch (Exception e) {
e.printStackTrace();
} finally {
if(combineFlag == 0) {
// Merge failed
r.put("errMessage"."Failed to merge file");
r.put("code".500);
} else {
r.put("code".0);
}
r.put("next", current + 1); }}else {
r.put("code".0);
r.put("next", files.length + 1); }}else {
// Size of the original file
Long size = Long.parseLong(request.getParameter("size"));
// The path to the current file
String fileNamePath = basePath + File.separatorChar + current;
OutputStream out = new FileOutputStream(fileNamePath);
// Get the relevant data of the file and write it to the file
MultipartFile data = ((MultipartHttpServletRequest) request).getFile("data");
byte[] bytes = data.getBytes();
out.write(bytes);
out.flush();
out.close();
// After uploading the last image, merge the images
if(current.equals(total)) {
// Merge operations
try {
String fileUrl = combineFiles(fileName, basePath);
if(StringUtils.isNotBlank(fileUrl)) {
combineFlag = 1;
r.put("filePath", fileUrl); }}catch (Exception e) {
e.printStackTrace();
logger.error(e.toString());
} finally {
if(combineFlag == 0) {
// Merge failed
r.put("errMessage"."Failed to merge file");
r.put("code".500);
} else {
r.put("code".0);
}
r.put("next", current + 1); }}else {
r.put("code".0);
r.put("next", current + 1); }}return r;
}
private String combineFiles(String fileName, String basePath) throws Exception {
int returnFlag = FileUtil.CombineFile(fileName, basePath);
String fileUrl = null;
if(returnFlag == 1) {
// Upload the file to OSS or FTP and return a file path
// TODO's current code needs you to update, which is the logic of uploading to a file server
fileUrl = ""
// Delete local files
File deleteFile = new File(basePath);
FileUtil.delTempChild(deleteFile);
}
return fileUrl;
}
Copy the codeFile utility class fileUtil
public class FileUtil {
/** * merge files, *@paramTar The file name of the synthesis target *@paramBaseFilePath Which folder you want to merge the files in, it must be the files to merge *@return
* @throws Exception
*/
public static int CombineFile(String tar, String baseFilePath) throws Exception {
File dirFile = new File(baseFilePath);
FileInputStream fis;
// Read 2M data at a time and save the read data into the byte array
byte buffer[] = new byte[1024 * 1024 * 2];
int len;
// Check whether file is a directory
if(dirFile.isDirectory()) {
String[] fileNames = dirFile.list();
FileOutputStream fos = new FileOutputStream(baseFilePath + File.separatorChar + tar);
// Implement custom directory sorting
Arrays.sort(fileNames, new StringComparator());
for (int i = 0; i<fileNames.length ; i++){ fis =new FileInputStream(baseFilePath + File.separatorChar + fileNames[i]);
len = 0;
while((len = fis.read(buffer)) ! = -1) {
// Buffer is written from the specified byte array. Buffer: the initial offset in the data. Len: the number of words written.
fos.write(buffer, 0, len);
}
fos.flush();
fis.close();
}
fos.close();
}
return 1;
}
/** * Delete a file or directory *@paramFile File path or folder path */
public static void delTempChild(File file){
if (file.isDirectory()) {
// Get all subfolders under the folder
String[] children = file.list();
// Recursively delete subdirectories in a directory
for (int i=0; i<children.length; i++) {
delTempChild(newFile(file, children[i])); }}// The directory is emptyfile.delete(); }}Copy the codeMore classes
import java.util.Comparator;
/ * * *@author yanming.gu
* @ClassName: StringComparator
* @Description
* @since1.0 * /
public class StringComparator implements Comparator<String> {
@Override
public int compare(String s1, String s2) {
if (returnDouble(s1) < returnDouble(s2)){
return -1;
} else if (returnDouble(s1) > returnDouble(s2)) {
return 1;
} else {
return 0; }}public static double returnDouble(String str){
StringBuffer sb = new StringBuffer();
for(int i=0; i<str.length(); i++){if(Character.isDigit(str.charAt(i))) {
sb.append(str.charAt(i));
} else if(str.charAt(i)=='. '&&i<str.length()-1&&Character.isDigit(str.charAt(i+1))) {
sb.append(str.charAt(i));
} else {
break; }}if(sb.toString().isEmpty()){
return 0;
} else {
returnDouble.parseDouble(sb.toString()); }}}Copy the codeThe above is the code to implement the breakpoint continuation, if there is any deficiency or feel the lack of the above, you can directly comment oh, and I will make up on time, or to solve the problem for you.