Skip to content

Commit eead634

Browse files
lycheeEnglycheeEng
committed
add new files
1 parent cd78141 commit eead634

4 files changed

Lines changed: 323 additions & 0 deletions

File tree

‎5-network/09-resume-upload/article.md‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
# Resumable file upload
2+
3+
With `fetch` method it's fairly easy to upload a file.
4+
5+
How to resume the upload after lost connection? There's no built-in option for that, but we have the pieces to implement it.
6+
7+
Resumable uploads should come with upload progress indication, as we expect big files (if we may need to resume). So, as `fetch` doesn't allow to track upload progress, we'll use [XMLHttpRequest](info:xmlhttprequest).
8+
9+
## Not-so-useful progress event
10+
11+
To resume upload, we need to know how much was uploaded till the connection was lost.
12+
13+
There's `xhr.upload.onprogress` to track upload progress.
14+
15+
Unfortunately, it's useless here, as it triggers when the data is *sent*, but was it received by the server? The browser doesn't know.
16+
17+
Maybe it was buffered by a local network proxy, or maybe the remote server process just died and couldn't process them, or it was just lost in the middle when the connection broke, and didn't reach the receiver.
18+
19+
So, this event is only useful to show a nice progress bar.
20+
21+
To resume upload, we need to know exactly the number of bytes received by the server. And only the server can tell that.
22+
23+
## Algorithm
24+
25+
1. First, we create a file id, to uniquely identify the file we're uploading, e.g.
26+
```js
27+
let fileId = file.name + '-' + file.size + '-' + +file.lastModifiedDate;
28+
```
29+
That's needed for resume upload, to tell the server what we're resuming.
30+
31+
2. Send a request to the server, asking how many bytes it already has, like this:
32+
```js
33+
let response = await fetch('status', {
34+
headers: {
35+
'X-File-Id': fileId
36+
}
37+
});
38+
39+
// The server has that many bytes
40+
let startByte = +await response.text();
41+
```
42+
43+
This assumes that the server tracks file uploads by `X-File-Id` header. Should be implemented at server-side.
44+
45+
3. Then, we can use `Blob` method `slice` to send the file from `startByte`:
46+
```js
47+
xhr.open("POST", "upload", true);
48+
49+
// send file id, so that the server knows which file to resume
50+
xhr.setRequestHeader('X-File-Id', fileId);
51+
// send the byte we're resuming from, so the server knows we're resuming
52+
xhr.setRequestHeader('X-Start-Byte', startByte);
53+
54+
xhr.upload.onprogress = (e) => {
55+
console.log(`Uploaded ${startByte + e.loaded} of ${startByte + e.total}`);
56+
};
57+
58+
// file can be from input.files[0] or another source
59+
xhr.send(file.slice(startByte));
60+
```
61+
62+
Here we send the server both file id as `X-File-Id`, so it knows which file we're uploading, and the starting byte as `X-Start-Byte`, so it knows we're not uploading it initially, but resuming.
63+
64+
The server should check its records, and if there was an upload of that file, and the current uploaded size is exactly `X-Start-Byte`, then append the data to it.
65+
66+
67+
Here's the demo with both client and server code, written on Node.js.
68+
69+
It works only partially on this site, as Node.js is behind another server named Nginx, that buffers uploads, passing them to Node.js when fully complete.
70+
71+
But you can download it and run locally for the full demonstration:
72+
73+
[codetabs src="upload-resume" height=200]
74+
75+
As you can see, modern networking methods are close to file managers in their capabilities -- control over headers, progress indicator, sending file parts, etc.

‎5-network/09-resume-upload/upload-resume.view/index.html‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
<!DOCTYPE HTML>
2+
3+
<script src="uploader.js"></script>
4+
5+
<form name="upload" method="POST" enctype="multipart/form-data" action="/upload">
6+
<input type="file" name="myfile">
7+
<input type="submit" name="submit" value="Upload (Resumes automatically)">
8+
</form>
9+
10+
<button onclick="uploader.stop()">Stop upload</button>
11+
12+
13+
<div id="log">Progress indication</div>
14+
15+
<script>
16+
function log(html) {
17+
document.getElementById('log').innerHTML = html;
18+
console.log(html);
19+
}
20+
21+
function onProgress(loaded, total) {
22+
log("progress " + loaded + ' / ' + total);
23+
}
24+
25+
let uploader;
26+
27+
document.forms.upload.onsubmit = async function(e) {
28+
e.preventDefault();
29+
30+
let file = this.elements.myfile.files[0];
31+
if (!file) return;
32+
33+
uploader = new Uploader({file, onProgress});
34+
35+
try {
36+
let uploaded = await uploader.upload();
37+
38+
if (uploaded) {
39+
log('success');
40+
} else {
41+
log('stopped');
42+
}
43+
44+
} catch(err) {
45+
console.error(err);
46+
log('error');
47+
}
48+
};
49+
50+
</script>

‎5-network/09-resume-upload/upload-resume.view/server.js‎

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
let http = require('http');
2+
let static = require('node-static');
3+
let fileServer = new static.Server('.');
4+
let path = require('path');
5+
let fs = require('fs');
6+
let debug = require('debug')('example:resume-upload');
7+
8+
let uploads = Object.create(null);
9+
10+
function onUpload(req, res) {
11+
12+
let fileId = req.headers['x-file-id'];
13+
let startByte = req.headers['x-start-byte'];
14+
15+
if (!fileId) {
16+
res.writeHead(400, "No file id");
17+
res.end();
18+
}
19+
20+
// we'll files "nowhere"
21+
let filePath = '/dev/null';
22+
// could use a real path instead, e.g.
23+
// let filePath = path.join('/tmp', fileId);
24+
25+
debug("onUpload fileId: ", fileId);
26+
27+
// initialize a new upload
28+
if (!uploads[fileId]) uploads[fileId] = {};
29+
let upload = uploads[fileId];
30+
31+
debug("bytesReceived:" + upload.bytesReceived + " startByte:" + startByte)
32+
33+
let fileStream;
34+
35+
// if startByte is 0 or not set, create a new file, otherwise check the size and append to existing one
36+
if (!startByte) {
37+
upload.bytesReceived = 0;
38+
fileStream = fs.createWriteStream(filePath, {
39+
flags: 'w'
40+
});
41+
debug("New file created: " + filePath);
42+
} else {
43+
// we can check on-disk file size as well to be sure
44+
if (upload.bytesReceived != startByte) {
45+
res.writeHead(400, "Wrong start byte");
46+
res.end(upload.bytesReceived);
47+
return;
48+
}
49+
// append to existing file
50+
fileStream = fs.createWriteStream(filePath, {
51+
flags: 'a'
52+
});
53+
debug("File reopened: " + filePath);
54+
}
55+
56+
57+
req.on('data', function(data) {
58+
debug("bytes received", upload.bytesReceived);
59+
upload.bytesReceived += data.length;
60+
});
61+
62+
// send request body to file
63+
req.pipe(fileStream);
64+
65+
// when the request is finished, and all its data is written
66+
fileStream.on('close', function() {
67+
if (upload.bytesReceived == req.headers['x-file-size']) {
68+
debug("Upload finished");
69+
delete uploads[fileId];
70+
71+
// can do something else with the uploaded file here
72+
73+
res.end("Success " + upload.bytesReceived);
74+
} else {
75+
// connection lost, we leave the unfinished file around
76+
debug("File unfinished, stopped at " + upload.bytesReceived);
77+
res.end();
78+
}
79+
});
80+
81+
// in case of I/O error - finish the request
82+
fileStream.on('error', function(err) {
83+
debug("fileStream error");
84+
res.writeHead(500, "File error");
85+
res.end();
86+
});
87+
88+
}
89+
90+
function onStatus(req, res) {
91+
let fileId = req.headers['x-file-id'];
92+
let upload = uploads[fileId];
93+
debug("onStatus fileId:", fileId, " upload:", upload);
94+
if (!upload) {
95+
res.end("0")
96+
} else {
97+
res.end(String(upload.bytesReceived));
98+
}
99+
}
100+
101+
102+
function accept(req, res) {
103+
if (req.url == '/status') {
104+
onStatus(req, res);
105+
} else if (req.url == '/upload' && req.method == 'POST') {
106+
onUpload(req, res);
107+
} else {
108+
fileServer.serve(req, res);
109+
}
110+
111+
}
112+
113+
114+
115+
116+
// -----------------------------------
117+
118+
if (!module.parent) {
119+
http.createServer(accept).listen(8080);
120+
console.log('Server listening at port 8080');
121+
} else {
122+
exports.accept = accept;
123+
}

‎5-network/09-resume-upload/upload-resume.view/uploader.js‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
class Uploader {
2+
3+
constructor({file, onProgress}) {
4+
this.file = file;
5+
this.onProgress = onProgress;
6+
7+
// create fileId that uniquely identifies the file
8+
// we could also add user session identifier (if had one), to make it even more unique
9+
this.fileId = file.name + '-' + file.size + '-' + +file.lastModifiedDate;
10+
}
11+
12+
async getUploadedBytes() {
13+
let response = await fetch('status', {
14+
headers: {
15+
'X-File-Id': this.fileId
16+
}
17+
});
18+
19+
if (response.status != 200) {
20+
throw new Error("Can't get uploaded bytes: " + response.statusText);
21+
}
22+
23+
let text = await response.text();
24+
25+
return +text;
26+
}
27+
28+
async upload() {
29+
this.startByte = await this.getUploadedBytes();
30+
31+
let xhr = this.xhr = new XMLHttpRequest();
32+
xhr.open("POST", "upload", true);
33+
34+
// send file id, so that the server knows which file to resume
35+
xhr.setRequestHeader('X-File-Id', this.fileId);
36+
// send the byte we're resuming from, so the server knows we're resuming
37+
xhr.setRequestHeader('X-Start-Byte', this.startByte);
38+
39+
xhr.upload.onprogress = (e) => {
40+
this.onProgress(this.startByte + e.loaded, this.startByte + e.total);
41+
};
42+
43+
console.log("send the file, starting from", this.startByte);
44+
xhr.send(this.file.slice(this.startByte));
45+
46+
// return
47+
// true if upload was successful,
48+
// false if aborted
49+
// throw in case of an error
50+
return await new Promise((resolve, reject) => {
51+
52+
xhr.onload = xhr.onerror = () => {
53+
console.log("upload end status:" + xhr.status + " text:" + xhr.statusText);
54+
55+
if (xhr.status == 200) {
56+
resolve(true);
57+
} else {
58+
reject(new Error("Upload failed: " + xhr.statusText));
59+
}
60+
};
61+
62+
// onabort triggers only when xhr.abort() is called
63+
xhr.onabort = () => resolve(false);
64+
65+
});
66+
67+
}
68+
69+
stop() {
70+
if (this.xhr) {
71+
this.xhr.abort();
72+
}
73+
}
74+
75+
}

0 commit comments

Comments
 (0)