Skip to content

Commit 48f8a95

Browse files
SakshamsSateesh Chodapuneedi
authored andcommitted
CLOUDSTACK-6092: Storage OverProvisioning as a Per Primary Basis Allow storage.overprovisioning.factor to be specified at storape pool level.
Signed-off-by: Sateesh Chodapuneedi <sateesh@apache.org>
1 parent 6a4927f commit 48f8a95

7 files changed

Lines changed: 141 additions & 10 deletions

File tree

api/src/org/apache/cloudstack/api/response/StoragePoolResponse.java

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,10 @@ public class StoragePoolResponse extends BaseResponse {
107107
@Param(description = "the scope of the storage pool")
108108
private String scope;
109109

110+
@SerializedName("overprovisionfactor")
111+
@Param(description = "the overprovisionfactor for the storage pool")
112+
private String overprovisionfactor;
113+
110114
@SerializedName(ApiConstants.HYPERVISOR)
111115
@Param(description = "the hypervisor type of the storage pool")
112116
private String hypervisor;
@@ -301,4 +305,8 @@ public void setState(StoragePoolStatus state) {
301305
public void setSuitableForMigration(Boolean suitableForMigration) {
302306
this.suitableForMigration = suitableForMigration;
303307
}
308+
309+
public void setOverProvisionFactor(String overProvisionFactor) {
310+
this.overprovisionfactor = overProvisionFactor;
311+
}
304312
}

engine/components-api/src/com/cloud/capacity/CapacityManager.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ public interface CapacityManager {
4444
"Percentage (as a value between 0 and 1) of storage utilization above which allocators will disable using the pool for low storage available.", true,
4545
ConfigKey.Scope.Zone);
4646
static final ConfigKey<Double> StorageOverprovisioningFactor = new ConfigKey<Double>("Storage", Double.class, StorageOverprovisioningFactorCK, "2",
47-
"Used for storage overprovisioning calculation; available storage will be (actualStorageSize * storage.overprovisioning.factor)", true, ConfigKey.Scope.Zone);
47+
"Used for storage overprovisioning calculation; available storage will be (actualStorageSize * storage.overprovisioning.factor)", true, ConfigKey.Scope.StoragePool);
4848
static final ConfigKey<Double> StorageAllocatedCapacityDisableThreshold =
4949
new ConfigKey<Double>(
5050
"Alert",

server/src/com/cloud/api/query/dao/StoragePoolJoinDaoImpl.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,13 +24,13 @@
2424

2525
import org.apache.log4j.Logger;
2626
import org.springframework.stereotype.Component;
27-
2827
import org.apache.cloudstack.api.response.StoragePoolResponse;
2928
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
3029

3130
import com.cloud.api.ApiDBUtils;
3231
import com.cloud.api.query.vo.StoragePoolJoinVO;
3332
import com.cloud.capacity.Capacity;
33+
import com.cloud.capacity.CapacityManager;
3434
import com.cloud.storage.ScopeType;
3535
import com.cloud.storage.StoragePool;
3636
import com.cloud.storage.StorageStats;
@@ -103,6 +103,7 @@ public StoragePoolResponse newStoragePoolResponse(StoragePoolJoinVO pool) {
103103
poolResponse.setClusterId(pool.getClusterUuid());
104104
poolResponse.setClusterName(pool.getClusterName());
105105
poolResponse.setTags(pool.getTag());
106+
poolResponse.setOverProvisionFactor(Double.toString(CapacityManager.StorageOverprovisioningFactor.valueIn(pool.getId())));
106107

107108
// set async job
108109
if (pool.getJobId() != null) {
@@ -156,6 +157,7 @@ public StoragePoolResponse newStoragePoolForMigrationResponse(StoragePoolJoinVO
156157
poolResponse.setDiskSizeTotal(pool.getCapacityBytes());
157158
poolResponse.setDiskSizeAllocated(allocatedSize);
158159
poolResponse.setCapacityIops(pool.getCapacityIops());
160+
poolResponse.setOverProvisionFactor(Double.toString(CapacityManager.StorageOverprovisioningFactor.valueIn(pool.getId())));
159161

160162
// TODO: StatsCollector does not persist data
161163
StorageStats stats = ApiDBUtils.getStoragePoolStatistics(pool.getId());

server/src/com/cloud/capacity/StorageCapacityListener.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ public void processConnect(Host server, StartupCommand startup, boolean forRebal
6767

6868
StartupStorageCommand ssCmd = (StartupStorageCommand)startup;
6969
if (ssCmd.getResourceType() == Storage.StorageResourceType.STORAGE_HOST) {
70-
BigDecimal overProvFactor = _storageMgr.getStorageOverProvisioningFactor(server.getDataCenterId());
70+
BigDecimal overProvFactor = BigDecimal.valueOf(CapacityManager.StorageOverprovisioningFactor.value());
7171
CapacityVO capacity =
7272
new CapacityVO(server.getId(), server.getDataCenterId(), server.getPodId(), server.getClusterId(), 0L, (overProvFactor.multiply(new BigDecimal(
7373
server.getTotalSize()))).longValue(), Capacity.CAPACITY_TYPE_STORAGE_ALLOCATED);

server/src/com/cloud/storage/StorageManagerImpl.java

Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -854,8 +854,8 @@ public void connectHostToSharedPool(long hostId, long poolId) throws StorageUnav
854854
}
855855

856856
@Override
857-
public BigDecimal getStorageOverProvisioningFactor(Long dcId) {
858-
return new BigDecimal(CapacityManager.StorageOverprovisioningFactor.valueIn(dcId));
857+
public BigDecimal getStorageOverProvisioningFactor(Long poolId) {
858+
return new BigDecimal(CapacityManager.StorageOverprovisioningFactor.valueIn(poolId));
859859
}
860860

861861
@Override
@@ -869,13 +869,18 @@ public void createCapacityEntry(StoragePoolVO storagePool, short capacityType, l
869869

870870
long totalOverProvCapacity;
871871
if (storagePool.getPoolType() == StoragePoolType.NetworkFilesystem || storagePool.getPoolType() == StoragePoolType.VMFS) {
872-
BigDecimal overProvFactor = getStorageOverProvisioningFactor(storagePool.getDataCenterId());
873-
totalOverProvCapacity = overProvFactor.multiply(new BigDecimal(storagePool.getCapacityBytes())).longValue();
874872
// All this is for the inaccuracy of floats for big number multiplication.
873+
BigDecimal overProvFactor = getStorageOverProvisioningFactor(storagePool.getId());
874+
totalOverProvCapacity = overProvFactor.multiply(new BigDecimal(storagePool.getCapacityBytes())).longValue();
875+
s_logger.debug("Found storage pool " + storagePool.getName() + " of type " + storagePool.getPoolType().toString() + " with overprovisioning factor "
876+
+ overProvFactor.toString());
877+
s_logger.debug("Total over provisioned capacity calculated is " + overProvFactor + " * " + storagePool.getCapacityBytes());
875878
} else {
879+
s_logger.debug("Found storage pool " + storagePool.getName() + " of type " + storagePool.getPoolType().toString());
876880
totalOverProvCapacity = storagePool.getCapacityBytes();
877881
}
878882

883+
s_logger.debug("Total over provisioned capacity of the pool " + storagePool.getName() + " id: " + storagePool.getId() + " is " + totalOverProvCapacity);
879884
if (capacities.size() == 0) {
880885
CapacityVO capacity =
881886
new CapacityVO(storagePool.getId(), storagePool.getDataCenterId(), storagePool.getPodId(), storagePool.getClusterId(), allocated, totalOverProvCapacity,
@@ -1552,16 +1557,22 @@ public boolean storagePoolHasEnoughSpace(List<Volume> volumes, StoragePool pool)
15521557

15531558
long totalOverProvCapacity;
15541559
if (pool.getPoolType() == StoragePoolType.NetworkFilesystem || pool.getPoolType() == StoragePoolType.VMFS) {
1555-
totalOverProvCapacity = getStorageOverProvisioningFactor(pool.getDataCenterId()).multiply(new BigDecimal(pool.getCapacityBytes())).longValue();
1560+
BigDecimal overProvFactor = getStorageOverProvisioningFactor(pool.getId());
1561+
totalOverProvCapacity = overProvFactor.multiply(new BigDecimal(pool.getCapacityBytes())).longValue();
1562+
s_logger.debug("Found storage pool " + poolVO.getName() + " of type " + pool.getPoolType().toString() + " with overprovisioning factor "
1563+
+ overProvFactor.toString());
1564+
s_logger.debug("Total over provisioned capacity calculated is " + overProvFactor + " * " + pool.getCapacityBytes());
15561565
} else {
15571566
totalOverProvCapacity = pool.getCapacityBytes();
1567+
s_logger.debug("Found storage pool " + poolVO.getName() + " of type " + pool.getPoolType().toString());
15581568
}
15591569

1570+
s_logger.debug("Total capacity of the pool " + poolVO.getName() + " id: " + pool.getId() + " is " + totalOverProvCapacity);
15601571
double storageAllocatedThreshold = CapacityManager.StorageAllocatedCapacityDisableThreshold.valueIn(pool.getDataCenterId());
15611572
if (s_logger.isDebugEnabled()) {
15621573
s_logger.debug("Checking pool: " + pool.getId() + " for volume allocation " + volumes.toString() + ", maxSize : " + totalOverProvCapacity +
1563-
", totalAllocatedSize : " + allocatedSizeWithtemplate + ", askingSize : " + totalAskingSize + ", allocated disable threshold: " +
1564-
storageAllocatedThreshold);
1574+
", totalAllocatedSize : " + allocatedSizeWithtemplate + ", askingSize : " + totalAskingSize + ", allocated disable threshold: " +
1575+
storageAllocatedThreshold);
15651576
}
15661577

15671578
double usedPercentage = (allocatedSizeWithtemplate + totalAskingSize) / (double)(totalOverProvCapacity);

setup/db/db/schema-430to440.sql

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -742,3 +742,6 @@ UPDATE `cloud`.`guest_os` SET `created` = now();
742742
ALTER TABLE `cloud`.`vm_reservation` ADD COLUMN `deployment_planner` varchar(40) DEFAULT NULL COMMENT 'Preferred deployment planner for the vm';
743743
ALTER TABLE `cloud`.`vpc_offerings` ADD COLUMN supports_distributed_router boolean default false;
744744
ALTER TABLE `cloud`.`vpc` ADD COLUMN uses_distributed_router boolean default false;
745+
746+
INSERT INTO `cloud`.`storage_pool_details` (pool_id,name,value,display) SELECT storage_pool.id,data_center_details.name,data_center_details.value,data_center_details.display FROM `cloud`.`storage_pool` JOIN `cloud`.`data_center_details` ON data_center_details.dc_id=storage_pool.data_center_id WHERE data_center_details.name = "storage.overprovisioning.factor";
747+
DELETE FROM `cloud`.`data_center_details` WHERE name="storage.overprovisioning.factor";
Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
""" Test for storage.overprovisioning.factor update
18+
"""
19+
#Import Local Modules
20+
from marvin.cloudstackTestCase import *
21+
from marvin.cloudstackAPI import *
22+
from marvin.integration.lib.utils import *
23+
from marvin.integration.lib.base import *
24+
from marvin.integration.lib.common import *
25+
from nose.plugins.attrib import attr
26+
#Import System modules
27+
28+
class TestUpdateOverProvision(cloudstackTestCase):
29+
"""
30+
Test to update a storage.overprovisioning.factor
31+
"""
32+
def setUp(self):
33+
self.apiClient = self.testClient.getApiClient()
34+
35+
@attr(tags=["simulator", "devcloud", "basic", "advanced"])
36+
def test_UpdateStorageOverProvisioningFactor(self):
37+
"""
38+
test update configuration setting at storage scope
39+
@return:
40+
"""
41+
42+
""" 1. list storagepools for id """
43+
""" 2. list overprovisioning factor for storage pool """
44+
""" 3. update setting for the pool"""
45+
""" 4. list overprovisioning factor for storage pool and assert"""
46+
47+
""" list storagepools """
48+
storage_pools = StoragePool.list(
49+
self.apiClient,
50+
listall=True
51+
)
52+
self.assertEqual(
53+
isinstance(storage_pools, list),
54+
True,
55+
"List storage pools should not return empty response"
56+
)
57+
58+
if len(storage_pools) < 1:
59+
raise self.skipTest(
60+
"The environment don't have storage pools required for test")
61+
62+
for pool in storage_pools:
63+
if pool.type == "NetworkFilesystem" or pool.type == "VMFS":
64+
break
65+
if pool.type != "NetworkFilesystem" and pool.type != "VMFS":
66+
raise self.skipTest("Storage overprovisioning currently not supported on " + pool.type + " pools")
67+
68+
self.poolId = pool.id
69+
""" list overprovisioning factor for storage pool """
70+
factorOld = float(pool.overprovisionfactor)
71+
factorNew = str(factorOld + 1.0)
72+
73+
""" update setting for the pool"""
74+
updateConfigurationCmd = updateConfiguration.updateConfigurationCmd()
75+
updateConfigurationCmd.name = "storage.overprovisioning.factor"
76+
updateConfigurationCmd.value = factorNew
77+
updateConfigurationCmd.storageid = pool.id
78+
79+
updateConfigurationResponse = self.apiClient.updateConfiguration(updateConfigurationCmd)
80+
81+
self.debug("updated the parameter %s with value %s"%(updateConfigurationResponse.name, updateConfigurationResponse.value))
82+
83+
storage_pools = StoragePool.list(
84+
self.apiClient,
85+
id = self.poolId
86+
)
87+
pool = storage_pools[0]
88+
factorNew = float(pool.overprovisionfactor)
89+
self.assertNotEqual(int(factorNew), int(factorOld)," Check if overprovision factor of storage pool has changed")
90+
self.assertEqual(int(factorNew), int(factorOld + 1.0)," Check if overprovision factor of storage pool has increased by 1")
91+
92+
def tearDown(self):
93+
"""Reset the storage.overprovisioning.factor back to its original value
94+
@return:
95+
"""
96+
storage_pools = StoragePool.list(
97+
self.apiClient,
98+
id = self.poolId
99+
)
100+
pool = storage_pools[0]
101+
updateConfigurationCmd = updateConfiguration.updateConfigurationCmd()
102+
updateConfigurationCmd.name = "storage.overprovisioning.factor"
103+
factorOld = float(pool.overprovisionfactor)
104+
factorNew = str(factorOld - 1.0)
105+
updateConfigurationCmd.value = factorNew
106+
updateConfigurationCmd.storageid = pool.id
107+
updateConfigurationResponse = self.apiClient.updateConfiguration(updateConfigurationCmd)

0 commit comments

Comments
 (0)