Skip to content

Commit 1c6378e

Browse files
Boris Schrijverwido
Boris Schrijver
authored and
wido
committed
Added QCOW2 virtual size checking for S3.
- Cleaned up S3TemplateDownloader - Created static QCOW2 utils class. - Reformatted some parts of DownloadManagerImpl
1 parent 1a02773 commit 1c6378e

4 files changed

Lines changed: 244 additions & 88 deletions

File tree

‎core/src/com/cloud/storage/template/S3TemplateDownloader.java‎

Lines changed: 104 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@
2727
import java.io.InputStream;
2828
import java.util.Date;
2929

30+
import org.apache.cloudstack.managed.context.ManagedContextRunnable;
31+
import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType;
3032
import org.apache.commons.httpclient.ChunkedInputStream;
3133
import org.apache.commons.httpclient.Credentials;
3234
import org.apache.commons.httpclient.Header;
@@ -50,57 +52,55 @@
5052
import com.amazonaws.services.s3.model.ProgressListener;
5153
import com.amazonaws.services.s3.model.PutObjectRequest;
5254
import com.amazonaws.services.s3.model.StorageClass;
53-
54-
import org.apache.cloudstack.managed.context.ManagedContextRunnable;
55-
import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType;
56-
5755
import com.cloud.agent.api.storage.Proxy;
5856
import com.cloud.agent.api.to.S3TO;
5957
import com.cloud.utils.Pair;
6058
import com.cloud.utils.S3Utils;
6159
import com.cloud.utils.UriUtils;
6260

6361
/**
64-
* Download a template file using HTTP
65-
*
62+
* Download a template file using HTTP(S)
6663
*/
6764
public class S3TemplateDownloader extends ManagedContextRunnable implements TemplateDownloader {
68-
public static final Logger s_logger = Logger.getLogger(S3TemplateDownloader.class.getName());
65+
private static final Logger s_logger = Logger.getLogger(S3TemplateDownloader.class.getName());
6966
private static final MultiThreadedHttpConnectionManager s_httpClientManager = new MultiThreadedHttpConnectionManager();
7067

7168
private String downloadUrl;
7269
private String installPath;
7370
private String s3Key;
7471
private String fileName;
75-
public TemplateDownloader.Status status = TemplateDownloader.Status.NOT_STARTED;
76-
public String errorString = " ";
77-
private long remoteSize = 0;
78-
public long downloadTime = 0;
79-
public long totalBytes;
72+
private String fileExtension;
73+
private String errorString = " ";
74+
75+
private TemplateDownloader.Status status = TemplateDownloader.Status.NOT_STARTED;
76+
private ResourceType resourceType = ResourceType.TEMPLATE;
8077
private final HttpClient client;
78+
private final HttpMethodRetryHandler myretryhandler;
8179
private GetMethod request;
82-
private boolean resume = false;
8380
private DownloadCompleteCallback completionCallback;
84-
private S3TO s3;
85-
private boolean inited = true;
81+
private S3TO s3to;
8682

83+
private long remoteSize = 0;
84+
private long downloadTime = 0;
85+
private long totalBytes;
8786
private long maxTemplateSizeInByte;
88-
private ResourceType resourceType = ResourceType.TEMPLATE;
89-
private final HttpMethodRetryHandler myretryhandler;
9087

91-
public S3TemplateDownloader(S3TO storageLayer, String downloadUrl, String installPath, DownloadCompleteCallback callback, long maxTemplateSizeInBytes, String user,
92-
String password, Proxy proxy, ResourceType resourceType) {
93-
s3 = storageLayer;
88+
private boolean resume = false;
89+
private boolean inited = true;
90+
91+
public S3TemplateDownloader(S3TO s3to, String downloadUrl, String installPath, DownloadCompleteCallback callback,
92+
long maxTemplateSizeInBytes, String user, String password, Proxy proxy, ResourceType resourceType) {
93+
this.s3to = s3to;
9494
this.downloadUrl = downloadUrl;
9595
this.installPath = installPath;
96-
status = TemplateDownloader.Status.NOT_STARTED;
96+
this.status = TemplateDownloader.Status.NOT_STARTED;
9797
this.resourceType = resourceType;
98-
maxTemplateSizeInByte = maxTemplateSizeInBytes;
98+
this.maxTemplateSizeInByte = maxTemplateSizeInBytes;
9999

100-
totalBytes = 0;
101-
client = new HttpClient(s_httpClientManager);
100+
this.totalBytes = 0;
101+
this.client = new HttpClient(s_httpClientManager);
102102

103-
myretryhandler = new HttpMethodRetryHandler() {
103+
this.myretryhandler = new HttpMethodRetryHandler() {
104104
@Override
105105
public boolean retryMethod(final HttpMethod method, final IOException exception, int executionCount) {
106106
if (executionCount >= 2) {
@@ -128,6 +128,7 @@ public boolean retryMethod(final HttpMethod method, final IOException exception,
128128

129129
Pair<String, Integer> hostAndPort = UriUtils.validateUrl(downloadUrl);
130130
fileName = StringUtils.substringAfterLast(downloadUrl, "/");
131+
fileExtension = StringUtils.substringAfterLast(fileName, ".");
131132

132133
if (proxy != null) {
133134
client.getHostConfiguration().setProxy(proxy.getHost(), proxy.getPort());
@@ -139,8 +140,10 @@ public boolean retryMethod(final HttpMethod method, final IOException exception,
139140
if ((user != null) && (password != null)) {
140141
client.getParams().setAuthenticationPreemptive(true);
141142
Credentials defaultcreds = new UsernamePasswordCredentials(user, password);
142-
client.getState().setCredentials(new AuthScope(hostAndPort.first(), hostAndPort.second(), AuthScope.ANY_REALM), defaultcreds);
143-
s_logger.info("Added username=" + user + ", password=" + password + "for host " + hostAndPort.first() + ":" + hostAndPort.second());
143+
client.getState().setCredentials(
144+
new AuthScope(hostAndPort.first(), hostAndPort.second(), AuthScope.ANY_REALM), defaultcreds);
145+
s_logger.info("Added username=" + user + ", password=" + password + "for host " + hostAndPort.first()
146+
+ ":" + hostAndPort.second());
144147
} else {
145148
s_logger.info("No credentials configured for host=" + hostAndPort.first() + ":" + hostAndPort.second());
146149
}
@@ -160,11 +163,11 @@ public boolean retryMethod(final HttpMethod method, final IOException exception,
160163
@Override
161164
public long download(boolean resume, DownloadCompleteCallback callback) {
162165
switch (status) {
163-
case ABORTED:
164-
case UNRECOVERABLE_ERROR:
165-
case DOWNLOAD_FINISHED:
166-
return 0;
167-
default:
166+
case ABORTED:
167+
case UNRECOVERABLE_ERROR:
168+
case DOWNLOAD_FINISHED:
169+
return 0;
170+
default:
168171

169172
}
170173

@@ -215,10 +218,11 @@ public long download(boolean resume, DownloadCompleteCallback callback) {
215218
contentType = contentTypeHeader.getValue();
216219
}
217220

218-
InputStream in = !chunked ? new BufferedInputStream(request.getResponseBodyAsStream()) : new ChunkedInputStream(request.getResponseBodyAsStream());
221+
InputStream in = !chunked ? new BufferedInputStream(request.getResponseBodyAsStream())
222+
: new ChunkedInputStream(request.getResponseBodyAsStream());
219223

220-
s_logger.info("Starting download from " + getDownloadUrl() + " to s3 bucket " + s3.getBucketName() + " remoteSize=" + remoteSize + " , max size=" +
221-
maxTemplateSizeInByte);
224+
s_logger.info("Starting download from " + getDownloadUrl() + " to s3 bucket " + s3to.getBucketName()
225+
+ " remoteSize=" + remoteSize + " , max size=" + maxTemplateSizeInByte);
222226

223227
Date start = new Date();
224228
// compute s3 key
@@ -230,9 +234,9 @@ public long download(boolean resume, DownloadCompleteCallback callback) {
230234
if (contentType != null) {
231235
metadata.setContentType(contentType);
232236
}
233-
PutObjectRequest putObjectRequest = new PutObjectRequest(s3.getBucketName(), s3Key, in, metadata);
237+
PutObjectRequest putObjectRequest = new PutObjectRequest(s3to.getBucketName(), s3Key, in, metadata);
234238
// check if RRS is enabled
235-
if (s3.getEnableRRS()) {
239+
if (s3to.getEnableRRS()) {
236240
putObjectRequest = putObjectRequest.withStorageClass(StorageClass.ReducedRedundancy);
237241
}
238242
// register progress listenser
@@ -257,14 +261,15 @@ public void progressChanged(ProgressEvent progressEvent) {
257261

258262
});
259263

260-
if (!s3.getSingleUpload(remoteSize)) {
264+
if (!s3to.getSingleUpload(remoteSize)) {
261265
// use TransferManager to do multipart upload
262-
S3Utils.mputObject(s3, putObjectRequest);
266+
S3Utils.mputObject(s3to, putObjectRequest);
263267
} else {
264268
// single part upload, with 5GB limit in Amazon
265-
S3Utils.putObject(s3, putObjectRequest);
266-
while (status != TemplateDownloader.Status.DOWNLOAD_FINISHED && status != TemplateDownloader.Status.UNRECOVERABLE_ERROR &&
267-
status != TemplateDownloader.Status.ABORTED) {
269+
S3Utils.putObject(s3to, putObjectRequest);
270+
while (status != TemplateDownloader.Status.DOWNLOAD_FINISHED
271+
&& status != TemplateDownloader.Status.UNRECOVERABLE_ERROR
272+
&& status != TemplateDownloader.Status.ABORTED) {
268273
// wait for completion
269274
}
270275
}
@@ -324,32 +329,59 @@ public long getDownloadedBytes() {
324329
return totalBytes;
325330
}
326331

332+
/**
333+
* Returns an InputStream only when the status is DOWNLOAD_FINISHED.
334+
*
335+
* The caller of this method must close the InputStream to prevent resource leaks!
336+
*
337+
* @return S3ObjectInputStream of the object.
338+
*/
339+
public InputStream getS3ObjectInputStream() {
340+
// Check if the download is finished
341+
if (status != Status.DOWNLOAD_FINISHED) {
342+
return null;
343+
}
344+
345+
return S3Utils.getObjectStream(s3to, s3to.getBucketName(), s3Key);
346+
}
347+
348+
public void cleanupAfterError() {
349+
if (status != Status.UNRECOVERABLE_ERROR) {
350+
s_logger.debug("S3Template downloader does not have state UNRECOVERABLE_ERROR, no cleanup neccesarry.");
351+
return;
352+
}
353+
354+
s_logger.info("Cleanup after UNRECOVERABLE_ERROR, trying to remove object: " + s3Key);
355+
356+
S3Utils.deleteObject(s3to, s3to.getBucketName(), s3Key);
357+
}
358+
327359
@Override
328360
@SuppressWarnings("fallthrough")
329361
public boolean stopDownload() {
330362
switch (getStatus()) {
331-
case IN_PROGRESS:
332-
if (request != null) {
333-
request.abort();
334-
}
335-
status = TemplateDownloader.Status.ABORTED;
336-
return true;
337-
case UNKNOWN:
338-
case NOT_STARTED:
339-
case RECOVERABLE_ERROR:
340-
case UNRECOVERABLE_ERROR:
341-
case ABORTED:
342-
status = TemplateDownloader.Status.ABORTED;
343-
case DOWNLOAD_FINISHED:
344-
try {
345-
S3Utils.deleteObject(s3, s3.getBucketName(), s3Key);
346-
} catch (Exception ex) {
347-
// ignore delete exception if it is not there
348-
}
349-
return true;
363+
case IN_PROGRESS:
364+
if (request != null) {
365+
request.abort();
366+
}
367+
status = TemplateDownloader.Status.ABORTED;
368+
return true;
369+
case UNKNOWN:
370+
case NOT_STARTED:
371+
case RECOVERABLE_ERROR:
372+
case UNRECOVERABLE_ERROR:
373+
case ABORTED:
374+
status = TemplateDownloader.Status.ABORTED;
375+
case DOWNLOAD_FINISHED:
376+
try {
377+
S3Utils.deleteObject(s3to, s3to.getBucketName(), s3Key);
378+
} catch (Exception ex) {
379+
// ignore delete exception if it is not there
380+
}
381+
return true;
350382

351-
default:
352-
return true;
383+
default:
384+
return true;
353385
}
354386
}
355387

@@ -359,7 +391,7 @@ public int getDownloadPercent() {
359391
return 0;
360392
}
361393

362-
return (int)(100.0 * totalBytes / remoteSize);
394+
return (int) (100.0 * totalBytes / remoteSize);
363395
}
364396

365397
@Override
@@ -417,4 +449,11 @@ public ResourceType getResourceType() {
417449
return resourceType;
418450
}
419451

420-
}
452+
public long getTotalBytes() {
453+
return totalBytes;
454+
}
455+
456+
public String getFileExtension() {
457+
return fileExtension;
458+
}
459+
}

0 commit comments

Comments
 (0)