Skip to content

Commit fa9ca72

Browse files
author
Prachi Damle
committed
CLOUDSTACK-2155: Anti-Affinity -When Vm deployment is done in parallel , anti-affinity rule is not honored.
Changes: - Locking the group and save reservation mechanism done by DPM - Added admin operation to cleanup VM reservations - DPM will also cleanup VM reservations on startup
1 parent 6610889 commit fa9ca72

12 files changed

Lines changed: 239 additions & 71 deletions

File tree

‎api/src/com/cloud/event/EventTypes.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -424,7 +424,7 @@ public class EventTypes {
424424
public static final String EVENT_AFFINITY_GROUP_ASSIGN = "AG.ASSIGN";
425425
public static final String EVENT_AFFINITY_GROUP_REMOVE = "AG.REMOVE";
426426
public static final String EVENT_VM_AFFINITY_GROUP_UPDATE = "VM.AG.UPDATE";
427-
427+
428428
public static final String EVENT_INTERNAL_LB_VM_START = "INTERNALLBVM.START";
429429
public static final String EVENT_INTERNAL_LB_VM_STOP = "INTERNALLBVM.STOP";
430430

@@ -442,6 +442,8 @@ public class EventTypes {
442442
public static final String EVENT_DEDICATE_RESOURCE = "DEDICATE.RESOURCE";
443443
public static final String EVENT_DEDICATE_RESOURCE_RELEASE = "DEDICATE.RESOURCE.RELEASE";
444444

445+
public static final String EVENT_CLEANUP_VM_RESERVATION = "VM.RESERVATION.CLEANUP";
446+
445447
static {
446448

447449
// TODO: need a way to force author adding event types to declare the entity details as well, with out braking

‎api/src/com/cloud/server/ManagementService.java‎

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,9 @@
1616
// under the License.
1717
package com.cloud.server;
1818

19-
import java.net.URISyntaxException;
2019
import java.util.ArrayList;
2120
import java.util.List;
2221
import java.util.Map;
23-
import java.util.Set;
24-
2522
import com.cloud.exception.*;
2623
import org.apache.cloudstack.api.ServerApiException;
2724
import org.apache.cloudstack.api.command.admin.cluster.ListClustersCmd;
@@ -43,18 +40,12 @@
4340
import org.apache.cloudstack.api.command.user.event.DeleteEventsCmd;
4441
import org.apache.cloudstack.api.command.user.guest.ListGuestOsCategoriesCmd;
4542
import org.apache.cloudstack.api.command.user.guest.ListGuestOsCmd;
46-
import org.apache.cloudstack.api.command.user.iso.ListIsosCmd;
47-
import org.apache.cloudstack.api.command.user.iso.UpdateIsoCmd;
4843
import org.apache.cloudstack.api.command.user.ssh.CreateSSHKeyPairCmd;
4944
import org.apache.cloudstack.api.command.user.ssh.DeleteSSHKeyPairCmd;
5045
import org.apache.cloudstack.api.command.user.ssh.ListSSHKeyPairsCmd;
5146
import org.apache.cloudstack.api.command.user.ssh.RegisterSSHKeyPairCmd;
52-
import org.apache.cloudstack.api.command.user.template.ListTemplatesCmd;
53-
import org.apache.cloudstack.api.command.user.template.UpdateTemplateCmd;
5447
import org.apache.cloudstack.api.command.user.vm.GetVMPasswordCmd;
5548
import org.apache.cloudstack.api.command.user.vmgroup.UpdateVMGroupCmd;
56-
import org.apache.cloudstack.api.command.user.volume.ExtractVolumeCmd;
57-
5849
import com.cloud.alert.Alert;
5950
import com.cloud.capacity.Capacity;
6051
import com.cloud.configuration.Configuration;
@@ -69,7 +60,6 @@
6960
import com.cloud.storage.GuestOS;
7061
import com.cloud.storage.GuestOsCategory;
7162
import com.cloud.storage.StoragePool;
72-
import com.cloud.template.VirtualMachineTemplate;
7363
import com.cloud.user.SSHKeyPair;
7464
import com.cloud.utils.Pair;
7565
import com.cloud.utils.Ternary;
@@ -371,6 +361,8 @@ Ternary<Pair<List<? extends Host>, Integer>, List<? extends Host>, Map<Host, Boo
371361
List<String> listDeploymentPlanners();
372362

373363
VirtualMachine upgradeSystemVM(ScaleSystemVMCmd cmd) throws ResourceUnavailableException, ManagementServerException, VirtualMachineMigrationException, ConcurrentOperationException;
374-
364+
375365
boolean getExecuteInSequence();
366+
367+
void cleanupVMReservations();
376368
}

‎api/src/org/apache/cloudstack/api/command/admin/resource/CleanVMReservationsCmd.java‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
package org.apache.cloudstack.api.command.admin.resource;
18+
19+
import org.apache.cloudstack.api.APICommand;
20+
import org.apache.cloudstack.api.ApiErrorCode;
21+
import org.apache.cloudstack.api.BaseAsyncCmd;
22+
import org.apache.cloudstack.api.ServerApiException;
23+
import org.apache.cloudstack.api.response.SuccessResponse;
24+
import org.apache.log4j.Logger;
25+
26+
import com.cloud.event.EventTypes;
27+
import com.cloud.user.Account;
28+
import com.cloud.user.UserContext;
29+
30+
@APICommand(name = "cleanVMReservations", description = "Cleanups VM reservations in the database.", responseObject = SuccessResponse.class)
31+
public class CleanVMReservationsCmd extends BaseAsyncCmd {
32+
public static final Logger s_logger = Logger.getLogger(CleanVMReservationsCmd.class.getName());
33+
34+
private static final String s_name = "cleanvmreservationresponse";
35+
36+
/////////////////////////////////////////////////////
37+
/////////////////// Accessors ///////////////////////
38+
/////////////////////////////////////////////////////
39+
40+
41+
/////////////////////////////////////////////////////
42+
/////////////// API Implementation///////////////////
43+
/////////////////////////////////////////////////////
44+
45+
@Override
46+
public String getCommandName() {
47+
return s_name;
48+
}
49+
50+
@Override
51+
public long getEntityOwnerId() {
52+
Account account = UserContext.current().getCaller();
53+
if (account != null) {
54+
return account.getId();
55+
}
56+
57+
return Account.ACCOUNT_ID_SYSTEM;
58+
}
59+
60+
@Override
61+
public String getEventType() {
62+
return EventTypes.EVENT_CLEANUP_VM_RESERVATION;
63+
}
64+
65+
@Override
66+
public String getEventDescription() {
67+
return "cleaning vm reservations in database";
68+
}
69+
70+
@Override
71+
public void execute(){
72+
try {
73+
_mgr.cleanupVMReservations();
74+
SuccessResponse response = new SuccessResponse(getCommandName());
75+
this.setResponseObject(response);
76+
} catch (Exception ex) {
77+
throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to clean vm reservations");
78+
}
79+
}
80+
}

‎client/tomcatconf/commands.properties.in‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -214,6 +214,7 @@ ldapConfig=1
214214
ldapRemove=1
215215
listCapabilities=15
216216
listDeploymentPlanners=1
217+
cleanVMReservations=1
217218

218219
#### pod commands
219220
createPod=1

‎engine/orchestration/src/org/apache/cloudstack/engine/cloud/entity/api/VMEntityManagerImpl.java‎

Lines changed: 10 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@
1616
// under the License.
1717
package org.apache.cloudstack.engine.cloud.entity.api;
1818

19-
import java.util.HashMap;
2019
import java.util.List;
2120
import java.util.Map;
2221
import java.util.UUID;
@@ -30,6 +29,7 @@
3029
import org.apache.cloudstack.engine.cloud.entity.api.db.dao.VMReservationDao;
3130
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
3231
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
32+
import org.apache.log4j.Logger;
3333
import org.springframework.stereotype.Component;
3434

3535
import com.cloud.dc.DataCenter;
@@ -46,12 +46,10 @@
4646
import com.cloud.exception.InsufficientServerCapacityException;
4747
import com.cloud.exception.OperationTimedoutException;
4848
import com.cloud.exception.ResourceUnavailableException;
49-
import com.cloud.hypervisor.Hypervisor.HypervisorType;
5049
import com.cloud.network.dao.NetworkDao;
5150
import com.cloud.org.Cluster;
5251
import com.cloud.service.dao.ServiceOfferingDao;
5352
import com.cloud.storage.StoragePool;
54-
import com.cloud.storage.Volume;
5553
import com.cloud.storage.VolumeVO;
5654
import com.cloud.storage.dao.DiskOfferingDao;
5755
import com.cloud.storage.dao.VMTemplateDao;
@@ -69,6 +67,8 @@
6967
@Component
7068
public class VMEntityManagerImpl implements VMEntityManager {
7169

70+
private static final Logger s_logger = Logger.getLogger(VMEntityManagerImpl.class);
71+
7272
@Inject
7373
protected VMInstanceDao _vmDao;
7474
@Inject
@@ -190,28 +190,15 @@ public String reserveVirtualMachine(VMEntityVO vmEntityVO, String plannerToUse,
190190
}
191191

192192
if (dest != null) {
193-
if (_dpMgr.finalizeReservation(dest, vmProfile, plan, exclude)) {
194-
// save destination with VMEntityVO
195-
VMReservationVO vmReservation = new VMReservationVO(vm.getId(), dest.getDataCenter().getId(), dest
196-
.getPod().getId(), dest.getCluster().getId(), dest.getHost().getId());
197-
Map<Long, Long> volumeReservationMap = new HashMap<Long, Long>();
198-
199-
if (vm.getHypervisorType() != HypervisorType.BareMetal) {
200-
for (Volume vo : dest.getStorageForDisks().keySet()) {
201-
volumeReservationMap.put(vo.getId(), dest.getStorageForDisks().get(vo).getId());
202-
}
203-
vmReservation.setVolumeReservation(volumeReservationMap);
204-
}
205-
206-
vmEntityVO.setVmReservation(vmReservation);
207-
_vmEntityDao.persist(vmEntityVO);
208-
209-
return vmReservation.getUuid();
193+
String reservationId = _dpMgr.finalizeReservation(dest, vmProfile, plan, exclude);
194+
if(reservationId != null){
195+
return reservationId;
210196
} else {
211-
try {
212-
Thread.sleep(10000);
213-
} catch (final InterruptedException e) {
197+
if (s_logger.isDebugEnabled()) {
198+
s_logger.debug("Cannot finalize the VM reservation for this destination found, retrying");
214199
}
200+
201+
exclude.addHost(dest.getHost().getId());
215202
continue;
216203
}
217204
} else if (planChangedByReadyVolume) {

‎engine/schema/src/org/apache/cloudstack/affinity/dao/AffinityGroupVMMapDao.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@
2323
import com.cloud.utils.Pair;
2424
import com.cloud.utils.db.Filter;
2525
import com.cloud.utils.db.GenericDao;
26-
import com.cloud.vm.VirtualMachine.State;
2726

2827
public interface AffinityGroupVMMapDao extends GenericDao<AffinityGroupVMMapVO, Long> {
2928

@@ -44,4 +43,6 @@ public interface AffinityGroupVMMapDao extends GenericDao<AffinityGroupVMMapVO,
4443
List<AffinityGroupVMMapVO> findByVmIdType(long instanceId, String type);
4544

4645
void updateMap(Long vmId, List<Long> affinityGroupIds);
46+
47+
List<Long> listAffinityGroupIdsByVmId(long instanceId);
4748
}

‎engine/schema/src/org/apache/cloudstack/affinity/dao/AffinityGroupVMMapDaoImpl.java‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,6 @@
2424

2525
import org.apache.cloudstack.affinity.AffinityGroupVMMapVO;
2626
import org.apache.cloudstack.affinity.AffinityGroupVO;
27-
import org.springframework.stereotype.Component;
28-
29-
import com.cloud.host.HostTagVO;
3027
import com.cloud.utils.Pair;
3128
import com.cloud.utils.db.Filter;
3229
import com.cloud.utils.db.GenericDaoBase;
@@ -46,6 +43,7 @@ public class AffinityGroupVMMapDaoImpl extends GenericDaoBase<AffinityGroupVMMap
4643
private GenericSearchBuilder<AffinityGroupVMMapVO, Long> ListVmIdByAffinityGroup;
4744
private SearchBuilder<AffinityGroupVMMapVO> ListByAffinityGroup;
4845
private SearchBuilder<AffinityGroupVMMapVO> ListByVmIdType;
46+
private GenericSearchBuilder<AffinityGroupVMMapVO, Long> ListAffinityGroupIdByVm;
4947

5048
@Inject
5149
protected AffinityGroupDao _affinityGroupDao;
@@ -87,6 +85,12 @@ protected void init() {
8785
CountSGForVm.select(null, Func.COUNT, null);
8886
CountSGForVm.and("vmId", CountSGForVm.entity().getInstanceId(), SearchCriteria.Op.EQ);
8987
CountSGForVm.done();
88+
89+
ListAffinityGroupIdByVm = createSearchBuilder(Long.class);
90+
ListAffinityGroupIdByVm.and("instanceId", ListAffinityGroupIdByVm.entity().getInstanceId(),
91+
SearchCriteria.Op.EQ);
92+
ListAffinityGroupIdByVm.selectField(ListAffinityGroupIdByVm.entity().getAffinityGroupId());
93+
ListAffinityGroupIdByVm.done();
9094
}
9195

9296
@Override
@@ -147,6 +151,13 @@ public List<AffinityGroupVMMapVO> findByVmIdType(long instanceId, String type) {
147151
return listBy(sc);
148152
}
149153

154+
@Override
155+
public List<Long> listAffinityGroupIdsByVmId(long instanceId) {
156+
SearchCriteria<Long> sc = ListAffinityGroupIdByVm.create();
157+
sc.setParameters("instanceId", instanceId);
158+
return customSearchIncludingRemoved(sc, null);
159+
}
160+
150161
@Override
151162
public void updateMap(Long vmId, List<Long> affinityGroupIds) {
152163
Transaction txn = Transaction.currentTxn();

‎plugins/affinity-group-processors/host-anti-affinity/src/org/apache/cloudstack/affinity/HostAntiAffinityProcessor.java‎

Lines changed: 14 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,6 @@ public boolean configure(final String name, final Map<String, Object> params) th
117117
return true;
118118
}
119119

120-
@DB
121120
@Override
122121
public boolean check(VirtualMachineProfile<? extends VirtualMachine> vmProfile, DeployDestination plannedDestination)
123122
throws AffinityConflictException {
@@ -132,31 +131,24 @@ public boolean check(VirtualMachineProfile<? extends VirtualMachine> vmProfile,
132131
List<AffinityGroupVMMapVO> vmGroupMappings = _affinityGroupVMMapDao.findByVmIdType(vm.getId(), getType());
133132

134133
for (AffinityGroupVMMapVO vmGroupMapping : vmGroupMappings) {
135-
final Transaction txn = Transaction.currentTxn();
136-
try {
137-
txn.start();
138-
// lock the group
139-
AffinityGroupVO group = _affinityGroupDao.lockRow(vmGroupMapping.getAffinityGroupId(), true);
140-
// if more than 1 VM's are present in the group then check for
141-
// conflict due to parallel deployment
142-
List<Long> groupVMIds = _affinityGroupVMMapDao.listVmIdsByAffinityGroup(vmGroupMapping
143-
.getAffinityGroupId());
144-
groupVMIds.remove(vm.getId());
145-
146-
for (Long groupVMId : groupVMIds) {
147-
VMReservationVO vmReservation = _reservationDao.findByVmId(groupVMId);
148-
if (vmReservation.getHostId() != null && vmReservation.getHostId().equals(plannedHostId)) {
149-
return false;
134+
// if more than 1 VM's are present in the group then check for
135+
// conflict due to parallel deployment
136+
List<Long> groupVMIds = _affinityGroupVMMapDao
137+
.listVmIdsByAffinityGroup(vmGroupMapping.getAffinityGroupId());
138+
groupVMIds.remove(vm.getId());
139+
140+
for (Long groupVMId : groupVMIds) {
141+
VMReservationVO vmReservation = _reservationDao.findByVmId(groupVMId);
142+
if (vmReservation != null && vmReservation.getHostId() != null
143+
&& vmReservation.getHostId().equals(plannedHostId)) {
144+
if (s_logger.isDebugEnabled()) {
145+
s_logger.debug("Planned destination for VM " + vm.getId() + " conflicts with an existing VM "
146+
+ vmReservation.getVmId() + " reserved on the same host " + plannedHostId);
150147
}
148+
return false;
151149
}
152-
153-
return true;
154-
155-
} finally {
156-
txn.commit();
157150
}
158151
}
159-
160152
return true;
161153
}
162154

‎server/src/com/cloud/deploy/DeploymentPlanningManager.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,9 @@ public interface DeploymentPlanningManager extends Manager {
4343
DeployDestination planDeployment(VirtualMachineProfile<? extends VirtualMachine> vmProfile, DeploymentPlan plan,
4444
ExcludeList avoids) throws InsufficientServerCapacityException, AffinityConflictException;
4545

46-
boolean finalizeReservation(DeployDestination plannedDestination,
46+
String finalizeReservation(DeployDestination plannedDestination,
4747
VirtualMachineProfile<? extends VirtualMachine> vmProfile, DeploymentPlan plan, ExcludeList avoids)
4848
throws InsufficientServerCapacityException, AffinityConflictException;
49+
50+
void cleanupVMReservations();
4951
}

0 commit comments

Comments
 (0)