Section uploading

The front end

• Slice the file using the blob.prototype. slice method
• Use the spark-MD5 library to compute the hash of files. (The spark-MD5 library cannot directly compute the entire file, but needs to compute fragments. For details, see the example spark-md5-example on its official website.
• Create a Web worker to compute hash and reduce page lag.
• Upload each fragment file to the server with hash
• When all files are uploaded, a merge request is sent with the file name and fragment size

The service side

Create a folder based on the received hash and store the fragmented files in the folder. After receiving the merge request, read the fragmented files and merge them. Delete the folder that stores the fragmented files and their contents

A pass way of thinking

The client

• When uploading a file, it computes the hash and sends the hash and file name to the server
• The server returns the status of whether the file exists
• If yes, the client displays a message indicating that the file is uploaded successfully. Otherwise, the file is uploaded

The service side

• The server sets a hash= file mapping table
• Search the table based on the hash sent by the client
• Returns an existing state

Web worker on pit

The self.importScript function can only import files with absolute paths, but the documentation says it can use relative paths, but has not been tested many times. Therefore, save the spark.md5.js file in the public folder.

rendering

The client

upload.js

import React, { useState, useEffect, useMemo } from "react";
import request from ".. /utils/request";
import styled from "styled-components";
import hashWorker from ".. /utils/hash-worker";
import WorkerBuilder from ".. /utils/worker-build";

const chunkSize = 1 * 1024 * 1024;
const UpLoadFile = function () {
  const [sourceFile, setSourceFile] = useState(null);
  const [chunksData, setChunksData] = useState([])
  const [myWorker, setMyWorker] = useState(null)
  const [fileHash, setFileHash] = useState("")
  const [hashPercentage, setHashPercentage] = useState(0)

  const handleFileChange = (e) = > {
    const { files } = e.target;
    if (files.length === 0) return;
    // Save the source file
    setSourceFile(files[0]);
    // File fragmentation
    splitFile(files[0])}// Send the merge request
  const mergeRequest = () = > {
    request({
      url: "http://localhost:3001/merge".method: "post".headers: {
        "content-type": "application/json"
      },
      data: JSON.stringify({
        filename: sourceFile.name,
        // For the server to merge files
        size: chunkSize
      })
    })
  }
  // Upload fragments
  const uploadChunks = async (chunksData) => {
    const formDataList = chunksData.map(({ chunk, hash }) = > {
      const formData = new FormData()
      formData.append("chunk", chunk);
      formData.append("hash", hash);
      formData.append("filename", sourceFile.name);
      return { formData };
    })

    const requestList = formDataList.map(({ formData }, index) = > {
      return request({
        url: "http://localhost:3001/upload".data: formData,
        onprogress: e= > {
          let list = [...chunksData];
          list[index].progress = parseInt(String((e.loaded / e.total) * 100));
          setChunksData(list)
        }
      })
    })
    // Upload the file
    await Promise.all(requestList);
    // Delay sending merge request
    setTimeout(() = > {
      mergeRequest();
    }, 500);
  }
  // Computes the file hash
  const calculateHash = (chunkList) = > {
    return new Promise(resolve= > {
      const w = new WorkerBuilder(hashWorker)
      w.postMessage({ chunkList: chunkList })
      w.onmessage = e= > {
        const { percentage, hash } = e.data;
        setHashPercentage(percentage);
        if (hash) {
          // When the hash is complete, execute resolve
          resolve(hash)
        }
      }
      setMyWorker(w)
    })
  }
  // Upload the file
  const handleUpload = async (e) => {
    if(! sourceFile) { alert("Please select the file first")
      return;
    }
    // Split the file
    const chunklist = splitFile(sourceFile);
    / / calculate the hash
    const hash = await calculateHash(chunklist)
    console.log("hash=======", hash)
    setFileHash(hash)
    const { shouldUpload } = await verfileIsExist(sourceFile.name, hash);
    if(! shouldUpload) { alert("File already exists, no need to upload again");
      return;
    }
    const chunksData = chunklist.map(({ chunk }, index) = > ({
      chunk: chunk,
      hash: hash + "-" + index,
      progress: 0
    }))
    // Save fragmented data
    setChunksData(chunksData)
    // Start uploading fragments
    uploadChunks(chunksData)
  }

  // Split the file
  const splitFile = (file, size = chunkSize) = > {
    const fileChunkList = [];
    let curChunkIndex = 0;
    while (curChunkIndex <= file.size) {
      const chunk = file.slice(curChunkIndex, curChunkIndex + size);
      fileChunkList.push({ chunk: chunk, })
      curChunkIndex += size;
    }
    return fileChunkList;
  }
  // second pass: verifies whether the file exists on the server
  const verfileIsExist = async (fileName, fileHash) => {
    const { data } = await request({
      url: "http://localhost:3001/verFileIsExist".headers: {
        "content-type": "application/json"
      },
      data: JSON.stringify({
        fileName: fileName,
        fileHash: fileHash
      })
    })
    return JSON.parse(data);
  }

  return (
    <div>
      <input type="file" onChange={handleFileChange} /><br />
      <button onClick={handleUpload}>upload</button>
      <ProgressBox chunkList={chunksData} />
    </div>)}const BlockWraper = styled.div`
  width: ${({ size }) => size + "px"};
  height: ${({ size }) => size + "px"};
  text-align: center;
  font-size: 12px;
  line-height: ${({ size }) => size + "px"}; 
  border: 1px solid #ccc;
  position: relative;
  float: left;
  &:before {
    content: "${({ chunkIndex }) => chunkIndex}";
    position: absolute;
    width: 100%;
    height: 10px;
    left: 0;
    top: 0;
    font-size: 12px;
    text-align: left;
    line-height: initial;
    color: #000
  }
  &:after {
    content: "";
    position: absolute;
    width: 100%;
    height: ${({ progress }) => progress + "%"};
    background-color: pink;
    left: 0;
    top: 0;
    z-index: -1;
  }
`
const ChunksProgress = styled.div` *zoom: 1; &:after { content: ""; display: block; clear: both; } `
const Label = styled.h3` `
const ProgressWraper = styled.div` `
const Block = ({ progress, size, chunkIndex }) = > {
  return (<BlockWraper size={size} chunkIndex={chunkIndex} progress={progress}>
    {progress}%
  </BlockWraper>)}const ProgressBox = ({ chunkList = [], size = 40 }) = > {
  const sumProgress = useMemo(() = > {
    if (chunkList.length === 0) return 0
    return chunkList.reduce((pre, cur, sum) = > pre + cur.progress / 100.0) * 100 / (chunkList.length)
  }, [chunkList])

  return (
    <ProgressWraper>
      <Label>The file is divided into {chunklist. length} segments, and the upload progress of each segment is as follows:</Label>
      <ChunksProgress>
        {chunkList.map(({ progress }, index) => (
          <Block key={index} size={size} chunkIndex={index} progress={progress} />
        ))}
      </ChunksProgress>
      <Label>Total progress: {sumProgress. ToFixed (2)} %</Label>
    </ProgressWraper >
  )
}

export default UpLoadFile;
Copy the code

hash-worker.js

const hashWorker = () = > {
  self.importScripts("http://localhost:3000/spark-md5.min.js")
  self.onmessage = (e) = > {
    const { chunkList } = e.data;
    const spark = new self.SparkMD5.ArrayBuffer();
    let percentage = 0;
    let count = 0;
    const loadNext = index= > {
      const reader = new FileReader();
      reader.readAsArrayBuffer(chunkList[index].chunk);
      reader.onload = event= > {
        count++;
        spark.append(event.target.result);
        if (count === chunkList.length) {
          self.postMessage({
            percentage: 100.hash: spark.end()
          })
          self.close();
        } else {
          percentage += (100 / chunkList.length)
          self.postMessage({
            percentage
          })
          loadNext(count)
        }
      }
    }
    loadNext(count)
  }
}

export default hashWorker
Copy the code

worker-build.js

export default class WorkerBuilder extends Worker {
  constructor(worker) {
    const code = worker.toString();
    const blob = new Blob([` (${code}`) ()]);
    return newWorker(URL.createObjectURL(blob)); }}Copy the code

request.js

const request = ({
  url,
  method = "post",
  data,
  headers = {},
  onprogress
}) = > {
  return new Promise(resolve= > {
    const xhr = new XMLHttpRequest();
    xhr.open(method, url);
    Object.keys(headers).forEach(key= >
      xhr.setRequestHeader(key, headers[key])
    );
    xhr.upload.onprogress = onprogress
    xhr.send(data);
    xhr.onload = e= > {
      resolve({
        data: e.target.response
      });
    };
  });
}

export default request;
Copy the code

The service side

import express from 'express'
import path from "path";
import fse from "fs-extra";
import multiparty from "multiparty";
import bodyParser from "body-parser"
let app = express()
const __dirname = path.resolve(path.dirname(' '));
const upload_files_dir = path.resolve(__dirname, "./filelist")

const jsonParser = bodyParser.json({ extended: false });

app.use(function (req, res, next) {
  res.setHeader("Access-Control-Allow-Origin"."*");
  res.setHeader("Access-Control-Allow-Headers"."*");
  next()
})

app.post('/verFileIsExist', jsonParser, async (req, res) => {
  const { fileName, fileHash } = req.body;
  const filePath = path.resolve(upload_files_dir, fileName);
  if (fse.existsSync(filePath)) {
    res.send({
      code: 200.shouldUpload: false})}else {
    res.send({
      code: 200.shouldUpload: true
    })
  }
})

app.post('/upload'.async (req, res) => {
  const multipart = new multiparty.Form();
  multipart.parse(req, async (err, fields, files) => {
    if (err) return;
    const [chunk] = files.chunk;
    const [hash] = fields.hash;
    const [filename] = fields.filename;
    // Add "_dir" to prevent file name and folder name conflicts during merge
    const chunkDir = path.resolve(upload_files_dir, filename + "_dir");
    if(! fse.existsSync(chunkDir)) {await fse.mkdirs(chunkDir);
    }
    await fse.move(chunk.path, `${chunkDir}/${hash}`);
  })
  res.status(200).send("received file chunk")})const pipeStream = (path, writeStream) = >
  new Promise(resolve= > {
    const readStream = fse.createReadStream(path);
    readStream.on("end".() = > {
      fse.unlinkSync(path);
      resolve();
    });
    readStream.pipe(writeStream);
  });

// merge slices
const mergeFileChunk = async (filePath, filename, size) => {
  const chunkDir = path.resolve(upload_files_dir, filename + "_dir");
  const chunkPaths = await fse.readdir(chunkDir);
  // Sort by slice subscript
  // Otherwise, the order of obtaining the directory directly may be out of order
  chunkPaths.sort((a, b) = > a.split("-") [1] - b.split("-") [1]);
  console.log("Specify a location to create a writable stream", filePath);
  await Promise.all(
    chunkPaths.map((chunkPath, index) = >
      pipeStream(
        path.resolve(chunkDir, chunkPath),
        // Create a writable stream at the specified location
        fse.createWriteStream(filePath, {
          start: index * size,
          end: (index + 1) * size
        })
      )
    )
  );
  fse.rmdirSync(chunkDir); // Delete the directory where the slices are saved after merging
};

app.post('/merge', jsonParser, async (req, res) => {
  const { filename, size } = req.body;
  const filePath = path.resolve(upload_files_dir, filename);
  await mergeFileChunk(filePath, filename, size);
  res.send({
    code: 200.message: "success"
  });
})

app.listen(3001.() = > {
  console.log('listen:3001')})Copy the code