Skip to content

Commit d5b78d3

Browse files
pingan8787pingan8787
committed
大文件并发下载
1 parent e9bff85 commit d5b78d3

9 files changed

Lines changed: 1193 additions & 0 deletions

File tree

‎Cute-Gist/Web/.DS_Store‎

6 KB
Binary file not shown.

‎Cute-Gist/Web/larger-file-upload/.DS_Store‎

6 KB
Binary file not shown.

‎Cute-Gist/Web/larger-file-upload/README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
学习文章：[JavaScript 中如何实现大文件并发上传？](https://mp.weixin.qq.com/s/-iSpCMaLruerHv7717P0Wg)

‎Cute-Gist/Web/larger-file-upload/client/index.html‎

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
<!DOCTYPE html>
2+
<html lang="zh-CN">
3+
<head>
4+
<meta charset="UTF-8" />
5+
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
6+
<meta http-equiv="X-UA-Compatible" content="ie=edge" />
7+
<title>大文件并发上传示例（阿宝哥）</title>
8+
<script src="https://cdn.bootcdn.net/ajax/libs/axios/0.21.1/axios.min.js"></script>
9+
<script src="https://cdn.bootcdn.net/ajax/libs/spark-md5/3.0.0/spark-md5.min.js"></script>
10+
</head>
11+
<body>
12+
<input type="file" id="uploadFile" />
13+
<button id="submit" onclick="uploadFile()">上传文件</button>
14+
<script>
15+
const uploadFileEle = document.querySelector("#uploadFile");
16+
17+
const request = axios.create({
18+
baseURL: "http://localhost:3000/upload",
19+
timeout: 10000,
20+
});
21+
22+
function calcFileMD5(file) {
23+
return new Promise((resolve, reject) => {
24+
let chunkSize = 2097152, // 2M
25+
chunks = Math.ceil(file.size / chunkSize),
26+
currentChunk = 0,
27+
spark = new SparkMD5.ArrayBuffer(),
28+
fileReader = new FileReader();
29+
30+
fileReader.onload = (e) => {
31+
spark.append(e.target.result);
32+
currentChunk++;
33+
if (currentChunk < chunks) {
34+
loadNext();
35+
} else {
36+
resolve(spark.end());
37+
}
38+
};
39+
40+
fileReader.onerror = (e) => {
41+
reject(fileReader.error);
42+
reader.abort();
43+
};
44+
45+
function loadNext() {
46+
let start = currentChunk * chunkSize,
47+
end =
48+
start + chunkSize >= file.size ? file.size : start + chunkSize;
49+
fileReader.readAsArrayBuffer(file.slice(start, end));
50+
}
51+
loadNext();
52+
});
53+
}
54+
55+
function checkFileExist(url, name, md5) {
56+
return request
57+
.get(url, {
58+
params: {
59+
name,
60+
md5,
61+
},
62+
})
63+
.then((response) => response.data);
64+
}
65+
66+
async function asyncPool(poolLimit, array, iteratorFn) {
67+
const ret = [];
68+
const executing = [];
69+
for (const item of array) {
70+
const p = Promise.resolve().then(() => iteratorFn(item, array));
71+
ret.push(p);
72+
73+
if (poolLimit <= array.length) {
74+
const e = p.then(() => executing.splice(executing.indexOf(e), 1));
75+
executing.push(e);
76+
if (executing.length >= poolLimit) {
77+
await Promise.race(executing);
78+
}
79+
}
80+
}
81+
return Promise.all(ret);
82+
}
83+
84+
async function uploadFile() {
85+
if (!uploadFileEle.files.length) return;
86+
const file = uploadFileEle.files[0]; // 获取待上传的文件
87+
const fileMd5 = await calcFileMD5(file); // 计算文件的MD5
88+
const fileStatus = await checkFileExist(
89+
// 判断文件是否已存在
90+
"/exists",
91+
file.name,
92+
fileMd5
93+
);
94+
if (fileStatus.data && fileStatus.data.isExists) {
95+
alert("文件已上传[秒传]");
96+
return;
97+
} else {
98+
await upload({
99+
url: "/single",
100+
file,
101+
fileMd5,
102+
fileSize: file.size,
103+
chunkSize: 1 * 1024 * 1024,
104+
chunkIds: fileStatus.data.chunkIds,
105+
poolLimit: 3,
106+
});
107+
}
108+
await concatFiles("/concatFiles", file.name, fileMd5);
109+
}
110+
111+
function upload({
112+
url,
113+
file,
114+
fileMd5,
115+
fileSize,
116+
chunkSize,
117+
chunkIds,
118+
poolLimit = 1,
119+
}) {
120+
const chunks =
121+
typeof chunkSize === "number" ? Math.ceil(fileSize / chunkSize) : 1;
122+
return asyncPool(poolLimit, [...new Array(chunks).keys()], (i) => {
123+
if (chunkIds.indexOf(i + "") !== -1) {
124+
// 已上传的分块直接跳过
125+
return Promise.resolve();
126+
}
127+
let start = i * chunkSize;
128+
let end = i + 1 == chunks ? fileSize : (i + 1) * chunkSize;
129+
const chunk = file.slice(start, end);
130+
return uploadChunk({
131+
url,
132+
chunk,
133+
chunkIndex: i,
134+
fileMd5,
135+
fileName: file.name,
136+
});
137+
});
138+
}
139+
140+
function uploadChunk({ url, chunk, chunkIndex, fileMd5, fileName }) {
141+
let formData = new FormData();
142+
formData.set("file", chunk, fileMd5 + "-" + chunkIndex);
143+
formData.set("name", fileName);
144+
formData.set("timestamp", Date.now());
145+
return request.post(url, formData);
146+
}
147+
148+
function concatFiles(url, name, md5) {
149+
return request.get(url, {
150+
params: {
151+
name,
152+
md5,
153+
},
154+
});
155+
}
156+
</script>
157+
</body>
158+
</html>

‎Cute-Gist/Web/larger-file-upload/server/.DS_Store‎

6 KB
Binary file not shown.

‎Cute-Gist/Web/larger-file-upload/server/index.js‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
const fs = require("fs");
2+
const path = require("path");
3+
const util = require("util");
4+
const Koa = require("koa");
5+
const cors = require("@koa/cors");
6+
const multer = require("@koa/multer");
7+
const Router = require("@koa/router");
8+
const serve = require("koa-static");
9+
const fse = require("fs-extra");
10+
const { dirExists } = require("./util");
11+
const readdir = util.promisify(fs.readdir);
12+
const unlink = util.promisify(fs.unlink);
13+
14+
const app = new Koa();
15+
const router = new Router();
16+
const TMP_DIR = path.join(__dirname, "tmp"); // 临时目录
17+
const UPLOAD_PATH = "./public/upload";
18+
const UPLOAD_DIR = path.join(__dirname, UPLOAD_PATH);
19+
const IGNORES = [".DS_Store"]; // 忽略的文件列表
20+
21+
const storage = multer.diskStorage({
22+
destination: async function (req, file, cb) {
23+
let fileMd5 = file.originalname.split("-")[0];
24+
const fileDir = path.join(TMP_DIR, fileMd5);
25+
await fse.ensureDir(fileDir);
26+
cb(null, fileDir);
27+
},
28+
filename: function (req, file, cb) {
29+
let chunkIndex = file.originalname.split("-")[1];
30+
cb(null, `${chunkIndex}`);
31+
},
32+
});
33+
34+
dirExists(UPLOAD_PATH);
35+
36+
const multerUpload = multer({ storage });
37+
38+
router.get("/", async (ctx) => {
39+
ctx.body = "大文件并发上传示例（阿宝哥）";
40+
});
41+
42+
router.get("/upload/exists", async (ctx) => {
43+
const { name: fileName, md5: fileMd5 } = ctx.query;
44+
const filePath = path.join(UPLOAD_DIR, fileName);
45+
const isExists = await fse.pathExists(filePath);
46+
if (isExists) {
47+
ctx.body = {
48+
status: "success",
49+
data: {
50+
isExists: true,
51+
url: `http://localhost:3000/${fileName}`,
52+
},
53+
};
54+
} else {
55+
let chunkIds = [];
56+
const chunksPath = path.join(TMP_DIR, fileMd5);
57+
const hasChunksPath = await fse.pathExists(chunksPath);
58+
if (hasChunksPath) {
59+
let files = await readdir(chunksPath);
60+
chunkIds = files.filter((file) => {
61+
return IGNORES.indexOf(file) === -1;
62+
});
63+
}
64+
ctx.body = {
65+
status: "success",
66+
data: {
67+
isExists: false,
68+
chunkIds,
69+
},
70+
};
71+
}
72+
});
73+
74+
router.post(
75+
"/upload/single",
76+
multerUpload.single("file"),
77+
async (ctx, next) => {
78+
ctx.body = {
79+
code: 1,
80+
data: ctx.file,
81+
};
82+
}
83+
);
84+
85+
router.get("/upload/concatFiles", async (ctx) => {
86+
const { name: fileName, md5: fileMd5 } = ctx.query;
87+
await concatFiles(
88+
path.join(TMP_DIR, fileMd5),
89+
path.join(UPLOAD_DIR, fileName)
90+
);
91+
ctx.body = {
92+
status: "success",
93+
data: {
94+
url: `http://localhost:3000/${fileName}`,
95+
},
96+
};
97+
});
98+
99+
async function concatFiles(sourceDir, targetPath) {
100+
const readFile = (file, ws) =>
101+
new Promise((resolve, reject) => {
102+
fs.createReadStream(file)
103+
.on("data", (data) => ws.write(data))
104+
.on("end", resolve)
105+
.on("error", reject);
106+
});
107+
const files = await readdir(sourceDir);
108+
const sortedFiles = files
109+
.filter((file) => {
110+
return IGNORES.indexOf(file) === -1;
111+
})
112+
.sort((a, b) => a - b);
113+
const writeStream = fs.createWriteStream(targetPath);
114+
for (const file of sortedFiles) {
115+
let filePath = path.join(sourceDir, file);
116+
await readFile(filePath, writeStream);
117+
await unlink(filePath); // 删除已合并的分块
118+
}
119+
writeStream.end();
120+
}
121+
122+
// 注册中间件
123+
app.use(cors());
124+
app.use(serve(UPLOAD_DIR));
125+
app.use(router.routes()).use(router.allowedMethods());
126+
127+
app.listen(3000, () => {
128+
console.log("app starting at port 3000");
129+
});

0 commit comments

Comments
 (0)