How to implement a slice upload service

A requirement encountered in a recent project requires that if the upload fails due to network outage or other circumstances, the next time you start uploading the same file, you can start uploading from the breakpoint. Baidu, found that to achieve this function, need backend cooperation, so I use KOA to achieve a simple slice upload service, used for the development of the front end of the trial. Now write down the implementation process for a reminder.

Train of thought

To implement breakpoint continuation, do the following:

1. Gets the unique identity of the file
2. Gets the length of the file
3. Record the length that has been uploaded
4. Record the data
5. Slice up the file and upload it
6. Merge slicing files
7. Verify file integrity

These need to be done together with the back end and the front end.

implementation

Based on the above points, let’s look at how to implement a slice upload interface.

Record file metadata

We need to provide an interface for the front end to call, upload the metadata of the file, and generate an upload task according to the metadata. If the task is interrupted abnormally later, we can also obtain the progress of the current task according to the metadata. Metadata includes file name, unique file identifier, file length, and slice size. The unique identifier of the file is calculated by the hash algorithm. Here we choose the hash algorithm md5, which is a very common hash encryption algorithm, characterized by fast and stable.

The front-end code

/** * input file onChange callback */
async function onFileChange(e) {
  const files = e.target.files;
  const file = files[0];
  const fileMetadata = await getFileMetadata(file); // Get file metadata
  const task = await getTaskInfo(fileMetadata); // Upload metadata to get task information
  const chunks = await getFileChunks(file, task.chunkSize); // Slice the file
  readyUploadFiles[task.hash] = { task, chunks }; // Save task and slice information locally
  updateTable();
}

/** * Get file meta information *@param {File}} file
 */
async function getFileMetadata(file) {
  const hash = await getFileMd5(file); // Get file hash; The spark-MD5 library is used
  const fileName = file.name;
  const fileType = file.type;
  const fileSize = file.size;
  return { hash, fileName, fileType, fileSize };
}

/** * Get upload task information *@param {{hash: string, fileName: string, fileType: string, fileSize: number}} metadata
 */
async function getTaskInfo(metadata) {
  return fetch("http://127.0.0.1:38080/api/task", {
    method: "POST".body: JSON.stringify(metadata),
    headers: { "Content-Type": "application/json" },
  }).then((res) = > res.json());
}
Copy the code

Back-end interface code

import Koa from "koa";
import KoaRouter from "@koa/router";
const router = new KoaRouter({ prefix: "/api" });
const upload_map = {};
router.post("/task".(ctx) = > {
  const metadata = ctx.request.body;
  // Create a temporary folder to store the chunks file for subsequent data consolidation
  makeTempDirByFileHash(metadata.hash);
  let task = upload_map[metadata.hash];
  if(! task) {// Save the task information for subsequent breakpoint continuations
    task = { chunkSize: 500.currentChunk: 0.done: false. metadata }; upload_map[metadata.hash] = task; } ctx.body = task; });const app = new Koa();
app.use(router.routes());
Copy the code

Uploading file slices

Once you get the upload task, slice the file according to the chunkSize in the task and upload it.

The front-end code

By recursively calling the function, chunks are uploaded in turn.

/** * Slice the file according to chunkSize *@param {File} file
 * @param {number} chunkSize* /
async function getFileChunks(file, chunkSize) {
  const result = [];
  const chunks = Math.ceil(file.size / chunkSize);

  for (let index = 0; index < chunks; index++) {
    const start = index * chunkSize,
      end = start + chunkSize >= file.size ? file.size : start + chunkSize;
    result.push(file.slice(start, end));
  }
  return result;
}

/** * Start uploading slices *@param {*} task
 * @param {*} chunks* /
async function beginUploadChunks(task, chunks) {
  if (task.done) {
    return;
  }
  const start = task.currentChunk * task.chunkSize;
  const end =
    start + task.chunkSize >= task.fileSize
      ? task.fileSize
      : start + task.chunkSize;
  try {
    const nextTask = await uploadChunks(
      task.hash,
      chunks[task.currentChunk],
      start,
      end
    );
    readyUploadFiles[task.hash].task = nextTask;
    updateTable();
    await beginUploadChunks(nextTask, chunks);
  } catch (error) {
    console.error(error); }}/** * Upload chunk data *@param {string} hash
 * @param {Blob} chunk
 * @param {number} start
 * @param {number} end* /
async function uploadChunks(hash, chunk, start, end) {
  const data = new FormData();
  data.append("hash", hash);
  data.append("chunk", chunk);
  data.append("start", start);
  data.append("end", end);
  const res = await fetch("http://127.0.0.1:38080/api/upload_chunk", {
    method: "POST".body: data,
  }).then((res) = > res.json());
  if (res.error) {
    throw new Error(res.error);
  } else {
    returnres; }}Copy the code

The back-end code

The back end uses the KOA-body library to parse multipart/form-data data

import KoaBody from "koa-body";
app.use(KoaBody({ multipart: true }));
// Receive the chunk uploaded
router.post("/upload_chunk".async (ctx) => {
  const upload = ctx.request.body;
  const files = ctx.request.files;
  if(! files) {return;
  }
  const { hash, start, end } = upload;
  const { chunk } = files;
  // Koa-body will automatically write the file in the form-data to the hard disk. We need to get the path to the file and write it to the temporary folder we created
  let filePath;
  if (chunk instanceof Array) {
    filePath = chunk[0].path;
  } else {
    filePath = chunk.path;
  }

  const task = upload_map[hash];
  if(task && ! task.done) {// Save chunk to a temporary folder
    const chunkPath = getTempDirByHash(hash) + ` /${start}-${end}`;
    const fileRead = fs.createReadStream(filePath);
    const chunkWrite = fs.createWriteStream(chunkPath);
    fileRead.pipe(chunkWrite);
    // Wait for the write to complete
    await new Promise((resolve) = > fileRead.on("end", resolve));
    // Delete the temporary file koa-body saved for us
    await fs.promises.unlink(filePath);
    // Index of the next chunk
    task.currentChunk++;
    if (task.currentChunk >= Math.ceil(task.fileSize / task.chunkSize)) {
      // Chunk uploads all task status to complete
      (task.done as any) = true;
      (task.currentChunk as any) = null;
    }
    ctx.body = task;
  } else {
    ctx.status = 400;
    ctx.body = { error: "Task not created"}; }});Copy the code

File merge and verification

After all slices have been uploaded, you can merge the slices and verify the integrity of the file

The front-end code

async function concatChunks(hash) {
  return fetch("http://127.0.0.1:38080/api/concat_chunk", {
    method: "POST".body: JSON.stringify({ hash }),
    headers: { "Content-Type": "application/json" },
  }).then((res) = > res.json());
}
Copy the code

The back-end code

In the final merge step, we check the integrity of the file through the data

router.post("/concat_chunk".async (ctx) => {
  const hash = ctx.request.body.hash;
  const task = upload_map[hash];
  if(! task) { ctx.body = {error: "Mission not found" };
    ctx.status = 400;
    return;
  }

  if(! task.done) { ctx.body = {error: "Not all files uploaded" };
    ctx.status = 400;
    return;
  }

  // Check whether the number of chunks is consistent
  const chunkDir = getTempDirByHash(hash);
  const chunkCount = Math.ceil(task.fileSize / task.chunkSize);
  const chunkPaths = await fs.promises.readdir(chunkDir);
  if(chunkCount ! == chunkPaths.length) { ctx.body = {error: "Inconsistent file slice verification" };
    ctx.status = 400;
    return;
  }
  const chunkFullPaths = chunkPaths
    .sort((a, b) = > {
      const a1 = a.split("-") [0];
      const b1 = b.split("-") [0];
      return Number(a1) - Number(b1);
    })
    .map((chunkPath) = > path.join(chunkDir, chunkPath));
  const filePath = path.resolve(
    path.join(__dirname, ".. /upload".`/file/${task.fileName}`));// Merge files
  await concatChunks(filePath, chunkFullPaths);
  const stat = await fs.promises.stat(filePath);
  // Verify file size
  if(stat.size ! == task.fileSize) { ctx.body = {error: "Inconsistent file size verification" };
    ctx.status = 400;
    return;
  }

  // Finally verify the hash
  const fileHash = await getFileMd5(filePath);
  if(fileHash ! == task.hash) { ctx.body = {error: "File hash check inconsistent" };
    ctx.status = 400;
    return;
  }

  // The task and temporary folder are deleted
  upload_map[task.hash] = undefined;

  ctx.body = { ok: true };
});
Copy the code

conclusion

Finally, release the complete code

If it is helpful to you, I hope to point a star~