Skip to content

Commit e766819

Browse files
author
Karl Rieb
committed
Increase default socket read timeout and update upload-file example to use chunked uploads.
Fixes T5863.
1 parent 7e361b8 commit e766819

File tree

4 files changed

+267
-111
lines changed

4 files changed

+267
-111
lines changed

‎ChangeLog.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
- Update upload-file example to include chunked upload example.
2+
- Increase default socket read timeout to 2 minutes.
13
- Parse Retry-After header for 503 retry exceptions in API v1.
24
- Add example from online tutorial.
35

‎examples/upload-file/src/com/dropbox/core/examples/upload_file/Main.java‎

Lines changed: 0 additions & 110 deletions
This file was deleted.

‎examples/upload-file/src/main/java/com/dropbox/core/examples/upload_file/Main.java‎

Lines changed: 262 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,262 @@
1+
package com.dropbox.core.examples.upload_file;
2+
3+
import com.dropbox.core.DbxAuthInfo;
4+
import com.dropbox.core.DbxException;
5+
import com.dropbox.core.DbxRequestConfig;
6+
import com.dropbox.core.DbxWebAuth;
7+
import com.dropbox.core.NetworkIOException;
8+
import com.dropbox.core.RetryException;
9+
import com.dropbox.core.json.JsonReader;
10+
import com.dropbox.core.v2.DbxClientV2;
11+
import com.dropbox.core.v2.DbxPathV2;
12+
import com.dropbox.core.v2.files.CommitInfo;
13+
import com.dropbox.core.v2.files.FileMetadata;
14+
import com.dropbox.core.v2.files.UploadErrorException;
15+
import com.dropbox.core.v2.files.UploadSessionCursor;
16+
import com.dropbox.core.v2.files.UploadSessionFinishErrorException;
17+
import com.dropbox.core.v2.files.UploadSessionLookupErrorException;
18+
import com.dropbox.core.v2.files.UploadSessionOffsetError;
19+
import com.dropbox.core.v2.files.WriteMode;
20+
21+
import java.io.File;
22+
import java.io.FileInputStream;
23+
import java.io.IOException;
24+
import java.io.InputStream;
25+
import java.util.logging.Level;
26+
import java.util.logging.Logger;
27+
import java.util.Date;
28+
import java.util.Locale;
29+
30+
/**
31+
* An example command-line application that runs through the web-based OAuth
32+
* flow (using {@link DbxWebAuth}).
33+
*/
34+
public class Main {
35+
private static final long CHUNKED_UPLOAD_CHUNK_SIZE = 4L << 20; // 4MiB
36+
private static final int CHUNKED_UPLOAD_MAX_ATTEMPTS = 20;
37+
38+
private static void uploadFile(DbxClientV2 dbxClient, File localFile, String dropboxPath) {
39+
try (InputStream in = new FileInputStream(localFile)) {
40+
FileMetadata metadata = dbxClient.files().uploadBuilder(dropboxPath)
41+
.withMode(WriteMode.ADD)
42+
.withClientModified(new Date(localFile.lastModified()))
43+
.uploadAndFinish(in);
44+
45+
System.out.println(metadata.toStringMultiline());
46+
} catch (UploadErrorException ex) {
47+
System.err.println("Error uploading to Dropbox: " + ex.getMessage());
48+
System.exit(1);
49+
} catch (DbxException ex) {
50+
System.err.println("Error uploading to Dropbox: " + ex.getMessage());
51+
System.exit(1);
52+
} catch (IOException ex) {
53+
System.err.println("Error reading from file \"" + localFile + "\": " + ex.getMessage());
54+
System.exit(1);
55+
}
56+
}
57+
58+
private static void chunkedUploadFile(DbxClientV2 dbxClient, File localFile, String dropboxPath) {
59+
long size = localFile.length();
60+
61+
// assert our file is at least the chunk upload size
62+
if (size < CHUNKED_UPLOAD_CHUNK_SIZE) {
63+
System.err.println("File too small, use upload() instead.");
64+
System.exit(1);
65+
return;
66+
}
67+
68+
long uploaded = 0L;
69+
DbxException thrown = null;
70+
71+
// Chunked uploads have 3 phases, each of which can accept uploaded bytes:
72+
//
73+
// (1) Start: initiate the upload and get an upload session ID
74+
// (2) Append: upload chunks of the file to append to our session
75+
// (3) Finish: commit the upload and close the session
76+
//
77+
// We track how many bytes we uploaded to determine which phase we should be in.
78+
String sessionId = null;
79+
for (int i = 0; i < CHUNKED_UPLOAD_MAX_ATTEMPTS; ++i) {
80+
if (i > 0) {
81+
System.out.printf("Retrying chunked upload (%d / %d attempts)\n", i + 1, CHUNKED_UPLOAD_MAX_ATTEMPTS);
82+
}
83+
84+
try (InputStream in = new FileInputStream(localFile)) {
85+
// if this is a retry, make sure seek to the correct offset
86+
in.skip(uploaded);
87+
88+
// (1) Start
89+
if (sessionId == null) {
90+
sessionId = dbxClient.files().uploadSessionStart()
91+
.uploadAndFinish(in, CHUNKED_UPLOAD_CHUNK_SIZE)
92+
.getSessionId();
93+
uploaded += CHUNKED_UPLOAD_CHUNK_SIZE;
94+
printProgress(uploaded, size);
95+
}
96+
97+
// (2) Append
98+
while ((size - uploaded) > CHUNKED_UPLOAD_CHUNK_SIZE) {
99+
dbxClient.files().uploadSessionAppend(sessionId, uploaded)
100+
.uploadAndFinish(in, CHUNKED_UPLOAD_CHUNK_SIZE);
101+
uploaded += CHUNKED_UPLOAD_CHUNK_SIZE;
102+
printProgress(uploaded, size);
103+
}
104+
105+
// (3) Finish
106+
long remaining = size - uploaded;
107+
UploadSessionCursor cursor = new UploadSessionCursor(sessionId, uploaded);
108+
CommitInfo commitInfo = CommitInfo.newBuilder(dropboxPath)
109+
.withMode(WriteMode.ADD)
110+
.withClientModified(new Date(localFile.lastModified()))
111+
.build();
112+
FileMetadata metadata = dbxClient.files().uploadSessionFinish(cursor, commitInfo)
113+
.uploadAndFinish(in, remaining);
114+
115+
System.out.println(metadata.toStringMultiline());
116+
return;
117+
} catch (RetryException ex) {
118+
thrown = ex;
119+
// RetryExceptions are never automatically retried by the client for uploads. Must
120+
// catch this exception even if DbxRequestConfig.getMaxRetries() > 0.
121+
sleepQuietly(ex.getBackoffMillis());
122+
continue;
123+
} catch (NetworkIOException ex) {
124+
thrown = ex;
125+
// network issue with Dropbox (maybe a timeout?) try again
126+
continue;
127+
} catch (UploadSessionLookupErrorException ex) {
128+
thrown = ex;
129+
// server offset into the stream doesn't match our offset (uploaded). Seek to
130+
// the expected offset according to the server and try again.
131+
if (ex.errorValue.isIncorrectOffset()) {
132+
System.out.printf("%s: current offset: %d, expected offset: %d\n", sessionId, uploaded, ex.errorValue
133+
.getIncorrectOffsetValue()
134+
.getCorrectOffset());
135+
uploaded = ex.errorValue
136+
.getIncorrectOffsetValue()
137+
.getCorrectOffset();
138+
continue;
139+
}
140+
System.err.println("Error uploading to Dropbox: " + ex.getMessage());
141+
System.exit(1);
142+
return;
143+
} catch (UploadSessionFinishErrorException ex) {
144+
thrown = ex;
145+
// server offset into the stream doesn't match our offset (uploaded). Seek to
146+
// the expected offset according to the server and try again.
147+
if (ex.errorValue.isLookupFailed() && ex.errorValue.getLookupFailedValue().isIncorrectOffset()) {
148+
uploaded = ex.errorValue
149+
.getLookupFailedValue()
150+
.getIncorrectOffsetValue()
151+
.getCorrectOffset();
152+
continue;
153+
}
154+
System.err.println("Error uploading to Dropbox: " + ex.getMessage());
155+
System.exit(1);
156+
return;
157+
} catch (DbxException ex) {
158+
System.err.println("Error uploading to Dropbox: " + ex.getMessage());
159+
System.exit(1);
160+
return;
161+
} catch (IOException ex) {
162+
System.err.println("Error reading from file \"" + localFile + "\": " + ex.getMessage());
163+
System.exit(1);
164+
return;
165+
}
166+
}
167+
168+
// if we made it here, then we must have run out of attempts
169+
System.err.println("Maxed out upload attempts to Dropbox. Most recent error: " + thrown.getMessage());
170+
System.exit(1);
171+
}
172+
173+
private static void printProgress(long uploaded, long size) {
174+
System.out.printf("Uploaded %12d / %12d bytes (%5.2f%%)\n", uploaded, size, 100 * (uploaded / (double) size));
175+
}
176+
177+
private static void sleepQuietly(long millis) {
178+
try {
179+
Thread.sleep(millis);
180+
} catch (InterruptedException ex) {
181+
// just exit
182+
System.err.println("Error uploading to Dropbox: interrupted during backoff.");
183+
System.exit(1);
184+
}
185+
}
186+
187+
public static void main(String[] args) throws IOException {
188+
// Only display important log messages.
189+
Logger.getLogger("").setLevel(Level.WARNING);
190+
191+
if (args.length == 0) {
192+
System.out.println("");
193+
System.out.println("Usage: COMMAND <auth-file> <local-path> <dropbox-path>");
194+
System.out.println("");
195+
System.out.println(" <auth-file>: An \"auth file\" that contains the information necessary to make");
196+
System.out.println(" an authorized Dropbox API request. Generate this file using the \"authorize\"");
197+
System.out.println(" example program.");
198+
System.out.println("");
199+
System.out.println(" <local-path>: The path to a local file whose contents you want to upload.");
200+
System.out.println("");
201+
System.out.println(" <dropbox-path>: The path on Dropbox to save the file to.");
202+
System.out.println("");
203+
System.exit(1);
204+
}
205+
206+
if (args.length != 3) {
207+
System.err.println("Expecting exactly 3 arguments, got " + args.length + ".");
208+
System.err.println("Run with no arguments for help.");
209+
System.exit(1);
210+
}
211+
212+
String argAuthFile = args[0];
213+
String localPath = args[1];
214+
String dropboxPath = args[2];
215+
216+
// Read auth info file.
217+
DbxAuthInfo authInfo;
218+
try {
219+
authInfo = DbxAuthInfo.Reader.readFromFile(argAuthFile);
220+
} catch (JsonReader.FileLoadException ex) {
221+
System.err.println("Error loading <auth-file>: " + ex.getMessage());
222+
System.exit(1);
223+
return;
224+
}
225+
226+
String pathError = DbxPathV2.findError(dropboxPath);
227+
if (pathError != null) {
228+
System.err.println("Invalid <dropbox-path>: " + pathError);
229+
System.exit(1);
230+
return;
231+
}
232+
233+
File localFile = new File(localPath);
234+
if (!localFile.exists()) {
235+
System.err.println("Invalid <local-path>: file does not exist.");
236+
System.exit(1);
237+
return;
238+
}
239+
240+
if (!localFile.isFile()) {
241+
System.err.println("Invalid <local-path>: not a file.");
242+
System.exit(1);
243+
return;
244+
}
245+
246+
247+
// Create a DbxClientV2, which is what you use to make API calls.
248+
String userLocale = Locale.getDefault().toString();
249+
DbxRequestConfig requestConfig = new DbxRequestConfig("examples-upload-file", userLocale);
250+
DbxClientV2 dbxClient = new DbxClientV2(requestConfig, authInfo.getAccessToken(), authInfo.getHost());
251+
252+
// upload the file with simple upload API if it is small enough, otherwise use chunked
253+
// upload API for better performance.
254+
if (localFile.length() < CHUNKED_UPLOAD_CHUNK_SIZE) {
255+
uploadFile(dbxClient, localFile, dropboxPath);
256+
} else {
257+
chunkedUploadFile(dbxClient, localFile, dropboxPath);
258+
}
259+
260+
System.exit(0);
261+
}
262+
}

‎src/main/java/com/dropbox/core/http/HttpRequestor.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,9 @@ public abstract class HttpRequestor
2828
*
2929
* A value of 0 indicates the timeout should be disabled.
3030
*/
31-
public static final long DEFAULT_READ_TIMEOUT_MILLIS = TimeUnit.SECONDS.toMillis(20);
31+
// Careful about lowering this value. Large file uploads can result in slow responses from the
32+
// server.
33+
public static final long DEFAULT_READ_TIMEOUT_MILLIS = TimeUnit.MINUTES.toMillis(2);
3234

3335
public abstract Response doGet(String url, Iterable<Header> headers) throws IOException;
3436
public abstract Uploader startPost(String url, Iterable<Header> headers) throws IOException;

0 commit comments

Comments
 (0)