Skip to content

Commit 25acfba

Browse files
minchen07minchen07
committed
CLOUDSTACK-4816:provide configurable option to choose single vs
multipart upload to S3 object storage based on object size.
1 parent 85f83a4 commit 25acfba

7 files changed

Lines changed: 47 additions & 15 deletions

File tree

‎api/src/com/cloud/agent/api/to/S3TO.java‎

Lines changed: 23 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ public final class S3TO implements S3Utils.ClientOptions, DataStoreTO {
4040
private Date created;
4141
private boolean enableRRS;
4242
private boolean multipartEnabled;
43+
private long maxSingleUploadSizeInBytes;
4344

4445
public S3TO() {
4546

@@ -51,7 +52,7 @@ public S3TO(final Long id, final String uuid, final String accessKey,
5152
final String secretKey, final String endPoint,
5253
final String bucketName, final Boolean httpsFlag,
5354
final Integer connectionTimeout, final Integer maxErrorRetry,
54-
final Integer socketTimeout, final Date created, final boolean enableRRS, final boolean multipart) {
55+
final Integer socketTimeout, final Date created, final boolean enableRRS, final long maxUploadSize) {
5556

5657
super();
5758

@@ -67,7 +68,7 @@ public S3TO(final Long id, final String uuid, final String accessKey,
6768
this.socketTimeout = socketTimeout;
6869
this.created = created;
6970
this.enableRRS = enableRRS;
70-
this.multipartEnabled = multipart;
71+
this.maxSingleUploadSizeInBytes = maxUploadSize;
7172

7273
}
7374

@@ -278,14 +279,28 @@ public void setEnableRRS(boolean enableRRS) {
278279
this.enableRRS = enableRRS;
279280
}
280281

281-
public boolean isMultipartEnabled() {
282-
return multipartEnabled;
282+
public long getMaxSingleUploadSizeInBytes() {
283+
return maxSingleUploadSizeInBytes;
283284
}
284285

285-
public void setMultipartEnabled(boolean multipartEnabled) {
286-
this.multipartEnabled = multipartEnabled;
286+
public void setMaxSingleUploadSizeInBytes(long maxSingleUploadSizeInBytes) {
287+
this.maxSingleUploadSizeInBytes = maxSingleUploadSizeInBytes;
287288
}
288289

289-
290-
290+
public boolean getSingleUpload(long objSize){
291+
if ( maxSingleUploadSizeInBytes < 0 ){
292+
// always use single part upload
293+
return true;
294+
} else if ( maxSingleUploadSizeInBytes == 0 ){
295+
// always use multi part upload
296+
return false;
297+
} else {
298+
// check object size to set flag
299+
if (objSize < maxSingleUploadSizeInBytes){
300+
return true;
301+
} else{
302+
return false;
303+
}
304+
}
305+
}
291306
}

‎core/src/com/cloud/storage/template/S3TemplateDownloader.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -258,7 +258,8 @@ public void progressChanged(ProgressEvent progressEvent) {
258258

259259
});
260260

261-
if ( s3.isMultipartEnabled()){
261+
262+
if ( !s3.getSingleUpload(remoteSize) ){
262263
// use TransferManager to do multipart upload
263264
S3Utils.mputObject(s3, putObjectRequest);
264265
} else{

‎plugins/storage/image/s3/src/org/apache/cloudstack/storage/datastore/driver/S3ImageStoreDriverImpl.java‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,13 +67,21 @@ public DataStoreTO getStoreTO(DataStore store) {
6767
.get(ApiConstants.S3_SOCKET_TIMEOUT)), imgStore.getCreated(),
6868
_configDao.getValue(Config.S3EnableRRS.toString()) == null ? false : Boolean.parseBoolean(_configDao
6969
.getValue(Config.S3EnableRRS.toString())),
70-
_configDao.getValue(Config.S3EnableMultiPartUpload.toString()) == null ? true : Boolean.parseBoolean(_configDao
71-
.getValue(Config.S3EnableMultiPartUpload.toString()))
70+
getMaxSingleUploadSizeInBytes()
7271
);
7372

7473
}
7574

7675

76+
private long getMaxSingleUploadSizeInBytes() {
77+
try {
78+
return Long.parseLong(_configDao.getValue(Config.S3MaxSingleUploadSize.toString())) * 1024L * 1024L * 1024L;
79+
} catch (NumberFormatException e) {
80+
// use default 5GB
81+
return 5L * 1024L * 1024L * 1024L;
82+
}
83+
}
84+
7785
@Override
7886
public String createEntityExtractUrl(DataStore store, String installPath, ImageFormat format, DataObject dataObject) {
7987
// for S3, no need to do anything, just return template url for

‎server/src/com/cloud/configuration/Config.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -377,7 +377,8 @@ public enum Config {
377377

378378
// object store
379379
S3EnableRRS("Advanced", ManagementServer.class, Boolean.class, "s3.rrs.enabled", "false", "enable s3 reduced redundancy storage", null),
380-
S3EnableMultiPartUpload("Advanced", ManagementServer.class, Boolean.class, "s3.multipart.enabled", "true", "enable s3 multipart upload", null),
380+
S3MaxSingleUploadSize("Advanced", ManagementServer.class, Integer.class, "s3.singleupload.max.size", "5", "The maximum size limit for S3 single part upload API(in GB). If it is set to 0, then it means always use multi-part upload to upload object to S3. " +
381+
"If it is set to -1, then it means always use single-part upload to upload object to S3. ", null),
381382

382383
// Ldap
383384
LdapBasedn("Advanced", ManagementServer.class, String.class, "ldap.basedn", null, "Sets the basedn for LDAP", null),

‎services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -857,10 +857,12 @@ protected Answer copyFromNfsToS3(CopyCommand cmd) {
857857
}
858858
}
859859
}
860+
861+
long srcSize = srcFile.length();
860862
ImageFormat format = getTemplateFormat(srcFile.getName());
861863
String key = destData.getPath() + S3Utils.SEPARATOR + srcFile.getName();
862-
if (s3.isMultipartEnabled()){
863-
mputFile(s3, srcFile, bucket, key);
864+
if (!s3.getSingleUpload(srcSize)){
865+
mputFile(s3, srcFile, bucket, key);
864866
} else{
865867
putFile(s3, srcFile, bucket, key);
866868
}

‎setup/db/db/schema-420to421.sql‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,8 @@
2020
--;
2121

2222

23-
INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Advanced', 'DEFAULT', 'management-server', 's3.multipart.enabled', 'true', 'enable s3 multipart upload');
23+
INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Advanced', 'DEFAULT', 'management-server', 's3.singleupload.max.size', '5',
24+
'The maximum size limit for S3 single part upload API(in GB). If it is set to 0, then it means always use multi-part upload to upload object to S3. If it is set to -1, then it means always use single-part upload to upload object to S3.');
2425

2526
INSERT IGNORE INTO `cloud`.`configuration` VALUES ("Storage", 'DEFAULT', 'management-server', "enable.ha.storage.migration", "true", "Enable/disable storage migration across primary storage during HA");
2627

‎utils/src/com/cloud/utils/S3Utils.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,10 @@ public static void putObject(final ClientOptions clientOptions,
171171
assert clientOptions != null;
172172
assert req != null;
173173

174+
if (LOGGER.isDebugEnabled()) {
175+
LOGGER.debug(format("Sending stream as S3 object using PutObjectRequest"));
176+
}
177+
174178
acquireClient(clientOptions).putObject(req);
175179

176180
}

0 commit comments

Comments
 (0)