Skip to content

Commit f1895ea

Browse files
minchen07minchen07
committed
CLOUDSTACK-4816: Make S3 upload multipart or singlepart configurable.
1 parent a6852a3 commit f1895ea

7 files changed

Lines changed: 130 additions & 51 deletions

File tree

‎api/src/com/cloud/agent/api/to/S3TO.java‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ public final class S3TO implements S3Utils.ClientOptions, DataStoreTO {
3939
private Integer socketTimeout;
4040
private Date created;
4141
private boolean enableRRS;
42+
private boolean multipartEnabled;
4243

4344
public S3TO() {
4445

@@ -50,7 +51,7 @@ public S3TO(final Long id, final String uuid, final String accessKey,
5051
final String secretKey, final String endPoint,
5152
final String bucketName, final Boolean httpsFlag,
5253
final Integer connectionTimeout, final Integer maxErrorRetry,
53-
final Integer socketTimeout, final Date created, final boolean enableRRS) {
54+
final Integer socketTimeout, final Date created, final boolean enableRRS, final boolean multipart) {
5455

5556
super();
5657

@@ -66,6 +67,7 @@ public S3TO(final Long id, final String uuid, final String accessKey,
6667
this.socketTimeout = socketTimeout;
6768
this.created = created;
6869
this.enableRRS = enableRRS;
70+
this.multipartEnabled = multipart;
6971

7072
}
7173

@@ -268,7 +270,6 @@ public DataStoreRole getRole() {
268270
}
269271

270272

271-
272273
public boolean getEnableRRS() {
273274
return enableRRS;
274275
}
@@ -277,5 +278,14 @@ public void setEnableRRS(boolean enableRRS) {
277278
this.enableRRS = enableRRS;
278279
}
279280

281+
public boolean isMultipartEnabled() {
282+
return multipartEnabled;
283+
}
284+
285+
public void setMultipartEnabled(boolean multipartEnabled) {
286+
this.multipartEnabled = multipartEnabled;
287+
}
288+
289+
280290

281291
}

‎core/src/com/cloud/storage/template/S3TemplateDownloader.java‎

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -47,8 +47,6 @@
4747
import com.amazonaws.services.s3.model.ProgressListener;
4848
import com.amazonaws.services.s3.model.PutObjectRequest;
4949
import com.amazonaws.services.s3.model.StorageClass;
50-
import com.amazonaws.services.s3.transfer.TransferManager;
51-
import com.amazonaws.services.s3.transfer.Upload;
5250

5351
import org.apache.cloudstack.managed.context.ManagedContextRunnable;
5452
import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType;
@@ -227,9 +225,6 @@ public long download(boolean resume, DownloadCompleteCallback callback) {
227225
// compute s3 key
228226
s3Key = join(asList(installPath, fileName), S3Utils.SEPARATOR);
229227

230-
// multi-part upload using S3 api to handle > 5G input stream
231-
TransferManager tm = new TransferManager(S3Utils.acquireClient(s3));
232-
233228
// download using S3 API
234229
ObjectMetadata metadata = new ObjectMetadata();
235230
metadata.setContentLength(remoteSize);
@@ -262,11 +257,19 @@ public void progressChanged(ProgressEvent progressEvent) {
262257
}
263258

264259
});
265-
// TransferManager processes all transfers asynchronously,
266-
// so this call will return immediately.
267-
Upload upload = tm.upload(putObjectRequest);
268-
269-
upload.waitForCompletion();
260+
261+
if ( s3.isMultipartEnabled()){
262+
// use TransferManager to do multipart upload
263+
S3Utils.mputObject(s3, putObjectRequest);
264+
} else{
265+
// single part upload, with 5GB limit in Amazon
266+
S3Utils.putObject(s3, putObjectRequest);
267+
while (status != TemplateDownloader.Status.DOWNLOAD_FINISHED &&
268+
status != TemplateDownloader.Status.UNRECOVERABLE_ERROR &&
269+
status != TemplateDownloader.Status.ABORTED) {
270+
// wait for completion
271+
}
272+
}
270273

271274
// finished or aborted
272275
Date finish = new Date();

‎plugins/storage/image/s3/src/org/apache/cloudstack/storage/datastore/driver/S3ImageStoreDriverImpl.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,10 @@ public DataStoreTO getStoreTO(DataStore store) {
6666
details.get(ApiConstants.S3_SOCKET_TIMEOUT) == null ? null : Integer.valueOf(details
6767
.get(ApiConstants.S3_SOCKET_TIMEOUT)), imgStore.getCreated(),
6868
_configDao.getValue(Config.S3EnableRRS.toString()) == null ? false : Boolean.parseBoolean(_configDao
69-
.getValue(Config.S3EnableRRS.toString())));
69+
.getValue(Config.S3EnableRRS.toString())),
70+
_configDao.getValue(Config.S3EnableMultiPartUpload.toString()) == null ? true : Boolean.parseBoolean(_configDao
71+
.getValue(Config.S3EnableMultiPartUpload.toString()))
72+
);
7073

7174
}
7275

‎server/src/com/cloud/configuration/Config.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -376,6 +376,7 @@ public enum Config {
376376

377377
// object store
378378
S3EnableRRS("Advanced", ManagementServer.class, Boolean.class, "s3.rrs.enabled", "false", "enable s3 reduced redundancy storage", null),
379+
S3EnableMultiPartUpload("Advanced", ManagementServer.class, Boolean.class, "s3.multipart.enabled", "true", "enable s3 multipart upload", null),
379380

380381
// Ldap
381382
LdapBasedn("Advanced", ManagementServer.class, String.class, "ldap.basedn", null, "Sets the basedn for LDAP", null),

‎services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java‎

Lines changed: 42 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,15 @@
1616
// under the License.
1717
package org.apache.cloudstack.storage.resource;
1818

19+
20+
import static com.cloud.utils.S3Utils.mputFile;
21+
import static com.cloud.utils.S3Utils.putFile;
22+
import static com.cloud.utils.StringUtils.join;
23+
import static com.cloud.utils.db.GlobalLock.executeWithNoWaitLock;
24+
import static java.lang.String.format;
25+
import static java.util.Arrays.asList;
26+
import static org.apache.commons.lang.StringUtils.substringAfterLast;
27+
1928
import java.io.BufferedReader;
2029
import java.io.BufferedWriter;
2130
import java.io.File;
@@ -40,6 +49,19 @@
4049

4150
import javax.naming.ConfigurationException;
4251

52+
import org.apache.commons.codec.digest.DigestUtils;
53+
import org.apache.commons.lang.StringUtils;
54+
import org.apache.http.HttpEntity;
55+
import org.apache.http.HttpResponse;
56+
import org.apache.http.NameValuePair;
57+
import org.apache.http.client.HttpClient;
58+
import org.apache.http.client.methods.HttpGet;
59+
import org.apache.http.client.utils.URLEncodedUtils;
60+
import org.apache.http.impl.client.DefaultHttpClient;
61+
import org.apache.log4j.Logger;
62+
63+
import com.amazonaws.services.s3.model.S3ObjectSummary;
64+
4365
import org.apache.cloudstack.storage.command.CopyCmdAnswer;
4466
import org.apache.cloudstack.storage.command.CopyCommand;
4567
import org.apache.cloudstack.storage.command.DeleteCommand;
@@ -50,23 +72,10 @@
5072
import org.apache.cloudstack.storage.template.DownloadManagerImpl.ZfsPathParser;
5173
import org.apache.cloudstack.storage.template.UploadManager;
5274
import org.apache.cloudstack.storage.template.UploadManagerImpl;
53-
import org.apache.cloudstack.storage.to.ImageStoreTO;
54-
import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
5575
import org.apache.cloudstack.storage.to.SnapshotObjectTO;
5676
import org.apache.cloudstack.storage.to.TemplateObjectTO;
5777
import org.apache.cloudstack.storage.to.VolumeObjectTO;
58-
import org.apache.commons.codec.digest.DigestUtils;
59-
import org.apache.commons.lang.StringUtils;
60-
import org.apache.http.HttpEntity;
61-
import org.apache.http.HttpResponse;
62-
import org.apache.http.NameValuePair;
63-
import org.apache.http.client.HttpClient;
64-
import org.apache.http.client.methods.HttpGet;
65-
import org.apache.http.client.utils.URLEncodedUtils;
66-
import org.apache.http.impl.client.DefaultHttpClient;
67-
import org.apache.log4j.Logger;
6878

69-
import com.amazonaws.services.s3.model.S3ObjectSummary;
7079
import com.cloud.agent.api.Answer;
7180
import com.cloud.agent.api.CheckHealthAnswer;
7281
import com.cloud.agent.api.CheckHealthCommand;
@@ -108,7 +117,6 @@
108117
import com.cloud.resource.ServerResourceBase;
109118
import com.cloud.storage.DataStoreRole;
110119
import com.cloud.storage.Storage.ImageFormat;
111-
import com.cloud.storage.Storage.StoragePoolType;
112120
import com.cloud.storage.StorageLayer;
113121
import com.cloud.storage.VMTemplateStorageResourceAssoc;
114122
import com.cloud.storage.template.Processor;
@@ -128,14 +136,6 @@
128136
import com.cloud.utils.script.OutputInterpreter;
129137
import com.cloud.utils.script.Script;
130138
import com.cloud.vm.SecondaryStorageVm;
131-
import com.google.common.io.Files;
132-
133-
import static com.cloud.utils.S3Utils.putFile;
134-
import static com.cloud.utils.StringUtils.join;
135-
import static com.cloud.utils.db.GlobalLock.executeWithNoWaitLock;
136-
import static java.lang.String.format;
137-
import static java.util.Arrays.asList;
138-
import static org.apache.commons.lang.StringUtils.substringAfterLast;
139139

140140
public class NfsSecondaryStorageResource extends ServerResourceBase implements SecondaryStorageResource {
141141

@@ -197,7 +197,7 @@ public void disconnected() {
197197
}
198198

199199
public void setInSystemVM(boolean inSystemVM) {
200-
this._inSystemVM = inSystemVM;
200+
_inSystemVM = inSystemVM;
201201
}
202202

203203
@Override
@@ -297,7 +297,7 @@ protected CopyCmdAnswer postProcessing(File destFile, String downloadPath, Strin
297297
String finalFileName = templateFilename;
298298
String finalDownloadPath = destPath + File.separator + templateFilename;
299299
// compute the size of
300-
long size = this._storage.getSize(downloadPath + File.separator + templateFilename);
300+
long size = _storage.getSize(downloadPath + File.separator + templateFilename);
301301

302302
DataTO newDestTO = null;
303303

@@ -374,7 +374,7 @@ public String determineFileName(final String key) {
374374

375375
protected Answer copySnapshotToTemplateFromNfsToNfsXenserver(CopyCommand cmd, SnapshotObjectTO srcData,
376376
NfsTO srcDataStore, TemplateObjectTO destData, NfsTO destDataStore) {
377-
String srcMountPoint = this.getRootDir(srcDataStore.getUrl());
377+
String srcMountPoint = getRootDir(srcDataStore.getUrl());
378378
String snapshotPath = srcData.getPath();
379379
int index = snapshotPath.lastIndexOf("/");
380380
String snapshotName = snapshotPath.substring(index + 1);
@@ -384,16 +384,16 @@ protected Answer copySnapshotToTemplateFromNfsToNfsXenserver(CopyCommand cmd, Sn
384384
snapshotPath = snapshotPath.substring(0, index);
385385

386386
snapshotPath = srcMountPoint + File.separator + snapshotPath;
387-
String destMountPoint = this.getRootDir(destDataStore.getUrl());
387+
String destMountPoint = getRootDir(destDataStore.getUrl());
388388
String destPath = destMountPoint + File.separator + destData.getPath();
389389

390390
String errMsg = null;
391391
try {
392-
this._storage.mkdir(destPath);
392+
_storage.mkdir(destPath);
393393

394394
String templateUuid = UUID.randomUUID().toString();
395395
String templateName = templateUuid + ".vhd";
396-
Script command = new Script(this.createTemplateFromSnapshotXenScript, cmd.getWait() * 1000, s_logger);
396+
Script command = new Script(createTemplateFromSnapshotXenScript, cmd.getWait() * 1000, s_logger);
397397
command.add("-p", snapshotPath);
398398
command.add("-s", snapshotName);
399399
command.add("-n", templateName);
@@ -468,7 +468,7 @@ protected Answer copySnapshotToTemplateFromNfsToNfs(CopyCommand cmd, SnapshotObj
468468
bufferWriter.write("\n");
469469
bufferWriter.write("filename=" + fileName);
470470
bufferWriter.write("\n");
471-
long size = this._storage.getSize(destFileFullPath);
471+
long size = _storage.getSize(destFileFullPath);
472472
bufferWriter.write("size=" + size);
473473
bufferWriter.close();
474474
writer.close();
@@ -630,7 +630,7 @@ protected Answer execute(CopyCommand cmd) {
630630
NfsTO destImageStore = (NfsTO) destDataStore;
631631
if (srcDataStore instanceof S3TO) {
632632
S3TO s3 = (S3TO) srcDataStore;
633-
return this.copyFromS3ToNfs(cmd, srcData, s3, destData, destImageStore);
633+
return copyFromS3ToNfs(cmd, srcData, s3, destData, destImageStore);
634634
} else if (srcDataStore instanceof SwiftTO) {
635635
return copyFromSwiftToNfs(cmd, srcData, (SwiftTO)srcDataStore, destData, destImageStore);
636636
}
@@ -857,9 +857,13 @@ protected Answer copyFromNfsToS3(CopyCommand cmd) {
857857
}
858858
}
859859
}
860-
ImageFormat format = this.getTemplateFormat(srcFile.getName());
860+
ImageFormat format = getTemplateFormat(srcFile.getName());
861861
String key = destData.getPath() + S3Utils.SEPARATOR + srcFile.getName();
862-
putFile(s3, srcFile, bucket, key);
862+
if (s3.isMultipartEnabled()){
863+
mputFile(s3, srcFile, bucket, key);
864+
} else{
865+
putFile(s3, srcFile, bucket, key);
866+
}
863867

864868
DataTO retObj = null;
865869
if (destData.getObjectType() == DataObjectType.TEMPLATE) {
@@ -1271,9 +1275,9 @@ private String deleteSnapshotBackupFromLocalFileSystem(final String secondarySto
12711275
int index = name.lastIndexOf(File.separator);
12721276
String snapshotPath = name.substring(0, index);
12731277
if (deleteAllFlag) {
1274-
lPath = this.getRootDir(secondaryStorageUrl) + File.separator + snapshotPath + File.separator + "*";
1278+
lPath = getRootDir(secondaryStorageUrl) + File.separator + snapshotPath + File.separator + "*";
12751279
} else {
1276-
lPath = this.getRootDir(secondaryStorageUrl) + File.separator + name + "*";
1280+
lPath = getRootDir(secondaryStorageUrl) + File.separator + name + "*";
12771281
}
12781282

12791283
final String result = deleteLocalFile(lPath);
@@ -1461,7 +1465,7 @@ Map<String, TemplateProp> swiftListTemplate(SwiftTO swift) {
14611465
Map<String, TemplateProp> s3ListTemplate(S3TO s3) {
14621466
String bucket = s3.getBucketName();
14631467
// List the objects in the source directory on S3
1464-
final List<S3ObjectSummary> objectSummaries = S3Utils.getDirectory(s3, bucket, this.TEMPLATE_ROOT_DIR);
1468+
final List<S3ObjectSummary> objectSummaries = S3Utils.getDirectory(s3, bucket, TEMPLATE_ROOT_DIR);
14651469
if (objectSummaries == null) {
14661470
return null;
14671471
}
@@ -1470,7 +1474,7 @@ Map<String, TemplateProp> s3ListTemplate(S3TO s3) {
14701474
String key = objectSummary.getKey();
14711475
// String installPath = StringUtils.substringBeforeLast(key,
14721476
// S3Utils.SEPARATOR);
1473-
String uniqueName = this.determineS3TemplateNameFromKey(key);
1477+
String uniqueName = determineS3TemplateNameFromKey(key);
14741478
// TODO: isPublic value, where to get?
14751479
TemplateProp tInfo = new TemplateProp(uniqueName, key, objectSummary.getSize(), objectSummary.getSize(),
14761480
true, false);
@@ -1483,7 +1487,7 @@ Map<String, TemplateProp> s3ListTemplate(S3TO s3) {
14831487
Map<Long, TemplateProp> s3ListVolume(S3TO s3) {
14841488
String bucket = s3.getBucketName();
14851489
// List the objects in the source directory on S3
1486-
final List<S3ObjectSummary> objectSummaries = S3Utils.getDirectory(s3, bucket, this.VOLUME_ROOT_DIR);
1490+
final List<S3ObjectSummary> objectSummaries = S3Utils.getDirectory(s3, bucket, VOLUME_ROOT_DIR);
14871491
if (objectSummaries == null) {
14881492
return null;
14891493
}
@@ -1492,7 +1496,7 @@ Map<Long, TemplateProp> s3ListVolume(S3TO s3) {
14921496
String key = objectSummary.getKey();
14931497
// String installPath = StringUtils.substringBeforeLast(key,
14941498
// S3Utils.SEPARATOR);
1495-
Long id = this.determineS3VolumeIdFromKey(key);
1499+
Long id = determineS3VolumeIdFromKey(key);
14961500
// TODO: how to get volume template name
14971501
TemplateProp tInfo = new TemplateProp(id.toString(), key, objectSummary.getSize(), objectSummary.getSize(),
14981502
true, false);

‎setup/db/db/schema-420to430.sql‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -389,4 +389,6 @@ CREATE VIEW `cloud`.`volume_view` AS
389389
`cloud`.`async_job` ON async_job.instance_id = volumes.id
390390
and async_job.instance_type = 'Volume'
391391
and async_job.job_status = 0;
392+
393+
INSERT IGNORE INTO `cloud`.`configuration`(category, instance, component, name, value, description, default_value) VALUES ('Advanced', 'DEFAULT', 'management-server', 's3.multipart.enabled', 'true', 'enable s3 multipart upload', 'true');
392394

0 commit comments

Comments
 (0)