Skip to content

Commit 5820b07

Browse files
kelvenykelveny
committed
CLOUDSTACK-4659: Add the missing feature back for GC VMware worker VMs
1 parent 2b4e994 commit 5820b07

16 files changed

Lines changed: 197 additions & 182 deletions

File tree

‎agent/src/com/cloud/agent/resource/consoleproxy/ConsoleProxyResource.java‎

Lines changed: 14 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,7 @@ public Answer executeRequest(final Command cmd) {
112112
}
113113

114114
private Answer execute(StartConsoleProxyAgentHttpHandlerCommand cmd) {
115+
s_logger.info("Invoke launchConsoleProxy() in responding to StartConsoleProxyAgentHttpHandlerCommand");
115116
launchConsoleProxy(cmd.getKeystoreBits(), cmd.getKeystorePassword(), cmd.getEncryptorPassword());
116117
return new Answer(cmd);
117118
}
@@ -361,29 +362,31 @@ public void run() {
361362
try {
362363
Class<?> consoleProxyClazz = Class.forName("com.cloud.consoleproxy.ConsoleProxy");
363364
try {
365+
s_logger.info("Invoke setEncryptorPassword(), ecnryptorPassword: " + encryptorPassword);
364366
Method methodSetup = consoleProxyClazz.getMethod(
365367
"setEncryptorPassword", String.class);
366368
methodSetup.invoke(null, encryptorPassword);
367369

370+
s_logger.info("Invoke startWithContext()");
368371
Method method = consoleProxyClazz.getMethod(
369372
"startWithContext", Properties.class,
370373
Object.class, byte[].class, String.class);
371374
method.invoke(null, _properties, resource, ksBits,
372375
ksPassword);
373376
} catch (SecurityException e) {
374-
s_logger.error("Unable to launch console proxy due to SecurityException");
377+
s_logger.error("Unable to launch console proxy due to SecurityException", e);
375378
System.exit(ExitStatus.Error.value());
376379
} catch (NoSuchMethodException e) {
377-
s_logger.error("Unable to launch console proxy due to NoSuchMethodException");
380+
s_logger.error("Unable to launch console proxy due to NoSuchMethodException", e);
378381
System.exit(ExitStatus.Error.value());
379382
} catch (IllegalArgumentException e) {
380-
s_logger.error("Unable to launch console proxy due to IllegalArgumentException");
383+
s_logger.error("Unable to launch console proxy due to IllegalArgumentException", e);
381384
System.exit(ExitStatus.Error.value());
382385
} catch (IllegalAccessException e) {
383-
s_logger.error("Unable to launch console proxy due to IllegalAccessException");
386+
s_logger.error("Unable to launch console proxy due to IllegalAccessException", e);
384387
System.exit(ExitStatus.Error.value());
385388
} catch (InvocationTargetException e) {
386-
s_logger.error("Unable to launch console proxy due to InvocationTargetException");
389+
s_logger.error("Unable to launch console proxy due to InvocationTargetException " + e.getTargetException().toString(), e);
387390
System.exit(ExitStatus.Error.value());
388391
}
389392
} catch (final ClassNotFoundException e) {
@@ -402,22 +405,22 @@ public void run() {
402405
Method methodSetup = consoleProxyClazz.getMethod("setEncryptorPassword", String.class);
403406
methodSetup.invoke(null, encryptorPassword);
404407
} catch (SecurityException e) {
405-
s_logger.error("Unable to launch console proxy due to SecurityException");
408+
s_logger.error("Unable to launch console proxy due to SecurityException", e);
406409
System.exit(ExitStatus.Error.value());
407410
} catch (NoSuchMethodException e) {
408-
s_logger.error("Unable to launch console proxy due to NoSuchMethodException");
411+
s_logger.error("Unable to launch console proxy due to NoSuchMethodException", e);
409412
System.exit(ExitStatus.Error.value());
410413
} catch (IllegalArgumentException e) {
411-
s_logger.error("Unable to launch console proxy due to IllegalArgumentException");
414+
s_logger.error("Unable to launch console proxy due to IllegalArgumentException", e);
412415
System.exit(ExitStatus.Error.value());
413416
} catch (IllegalAccessException e) {
414-
s_logger.error("Unable to launch console proxy due to IllegalAccessException");
417+
s_logger.error("Unable to launch console proxy due to IllegalAccessException", e);
415418
System.exit(ExitStatus.Error.value());
416419
} catch (InvocationTargetException e) {
417-
s_logger.error("Unable to launch console proxy due to InvocationTargetException");
420+
s_logger.error("Unable to launch console proxy due to InvocationTargetException " + e.getTargetException().toString(), e);
418421
System.exit(ExitStatus.Error.value());
419422
} catch (final ClassNotFoundException e) {
420-
s_logger.error("Unable to launch console proxy due to ClassNotFoundException");
423+
s_logger.error("Unable to launch console proxy due to ClassNotFoundException", e);
421424
System.exit(ExitStatus.Error.value());
422425
}
423426
}

‎client/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
<groupId>org.apache.cloudstack</groupId>
1818
<artifactId>cloudstack</artifactId>
1919
<version>4.3.0-SNAPSHOT</version>
20-
</parent>
20+
</parent>
2121
<dependencies>
2222
<dependency>
2323
<groupId>org.apache.cloudstack</groupId>

‎framework/cluster/src/com/cloud/cluster/ClusterManager.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,8 @@ public interface ClusterManager extends Manager {
5555
ManagementServerHost getPeer(String peerName);
5656

5757
String getSelfPeerName();
58+
long getManagementNodeId();
59+
long getCurrentRunId();
5860

5961
public interface Dispatcher {
6062
String getName();

‎framework/cluster/src/com/cloud/cluster/ClusterManagerImpl.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1076,6 +1076,10 @@ public boolean configure(String name, Map<String, Object> params) throws Configu
10761076
}
10771077
return true;
10781078
}
1079+
1080+
public long getManagementNodeId() {
1081+
return _msId;
1082+
}
10791083

10801084
public long getCurrentRunId() {
10811085
return _runId;

‎plugins/hypervisors/vmware/src/com/cloud/hypervisor/guru/VMwareGuru.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@
5454
import com.cloud.agent.api.to.DataTO;
5555
import com.cloud.agent.api.to.NicTO;
5656
import com.cloud.agent.api.to.VirtualMachineTO;
57+
import com.cloud.cluster.ClusterManager;
5758
import com.cloud.configuration.Config;
5859
import com.cloud.exception.InsufficientAddressCapacityException;
5960
import com.cloud.host.Host;
@@ -115,6 +116,7 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru {
115116
PhysicalNetworkDao _physicalNetworkDao;
116117
@Inject
117118
PhysicalNetworkTrafficTypeDao _physicalNetworkTrafficTypeDao;
119+
@Inject ClusterManager _clusterMgr;
118120

119121
protected VMwareGuru() {
120122
super();
@@ -365,6 +367,7 @@ public Pair<Boolean, Long> getCommandHostDelegation(long hostId, Command cmd) {
365367
CommandExecLogVO execLog = new CommandExecLogVO(cmdTarget.first().getId(), cmdTarget.second().getId(), cmd.getClass().getSimpleName(), 1);
366368
_cmdExecLogDao.persist(execLog);
367369
cmd.setContextParam("execid", String.valueOf(execLog.getId()));
370+
cmd.setContextParam("noderuninfo", String.format("%d-%d", _clusterMgr.getManagementNodeId(), _clusterMgr.getCurrentRunId()));
368371

369372
if(cmd instanceof BackupSnapshotCommand ||
370373
cmd instanceof CreatePrivateTemplateFromVolumeCommand ||

‎plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareManager.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ List<ManagedObjectReference> addHostToPodCluster(VmwareContext serviceContext, l
5252

5353
VmwareStorageManager getStorageManager();
5454
void gcLeftOverVMs(VmwareContext context);
55+
boolean needRecycle(String workerTag);
5556

5657
Pair<Integer, Integer> getAddiionalVncPortRange();
5758

‎plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java‎

Lines changed: 50 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,9 @@
5959
import com.cloud.agent.api.Command;
6060
import com.cloud.agent.api.StartupCommand;
6161
import com.cloud.agent.api.StartupRoutingCommand;
62+
import com.cloud.cluster.ClusterManager;
63+
import com.cloud.cluster.ManagementServerHost;
64+
import com.cloud.cluster.dao.ManagementServerHostPeerDao;
6265
import com.cloud.configuration.Config;
6366
import com.cloud.dc.ClusterDetailsDao;
6467
import com.cloud.dc.ClusterVO;
@@ -153,6 +156,8 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw
153156
@Inject VmwareDatacenterDao _vmwareDcDao;
154157
@Inject VmwareDatacenterZoneMapDao _vmwareDcZoneMapDao;
155158
@Inject LegacyZoneDao _legacyZoneDao;
159+
@Inject ManagementServerHostPeerDao _mshostPeerDao;
160+
@Inject ClusterManager _clusterMgr;
156161

157162
String _mountParent;
158163
StorageLayer _storage;
@@ -166,6 +171,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw
166171
String _managemetPortGroupName;
167172
String _defaultSystemVmNicAdapterType = VirtualEthernetCardType.E1000.toString();
168173
String _recycleHungWorker = "false";
174+
long _hungWorkerTimeout = 7200000; // 2 hour
169175
int _additionalPortRangeStart;
170176
int _additionalPortRangeSize;
171177
int _routerExtraPublicNics = 2;
@@ -284,6 +290,10 @@ public boolean configure(String name, Map<String, Object> params) throws Configu
284290
if(_recycleHungWorker == null || _recycleHungWorker.isEmpty()) {
285291
_recycleHungWorker = "false";
286292
}
293+
294+
value = _configDao.getValue(Config.VmwareHungWorkerTimeout.key());
295+
if(value != null)
296+
_hungWorkerTimeout = Long.parseLong(value) * 1000;
287297

288298
_rootDiskController = _configDao.getValue(Config.VmwareRootDiskControllerType.key());
289299
if(_rootDiskController == null || _rootDiskController.isEmpty()) {
@@ -529,11 +539,50 @@ public VmwareStorageManager getStorageManager() {
529539
return _storageMgr;
530540
}
531541

532-
533542
@Override
534543
public void gcLeftOverVMs(VmwareContext context) {
535544
VmwareCleanupMaid.gcLeftOverVMs(context);
536545
}
546+
547+
@Override
548+
public boolean needRecycle(String workerTag) {
549+
if(s_logger.isInfoEnabled())
550+
s_logger.info("Check to see if a worker VM with tag " + workerTag + " needs to be recycled");
551+
552+
if(workerTag == null || workerTag.isEmpty()) {
553+
s_logger.error("Invalid worker VM tag " + workerTag);
554+
return false;
555+
}
556+
557+
String tokens[] = workerTag.split("-");
558+
if(tokens.length != 3) {
559+
s_logger.error("Invalid worker VM tag " + workerTag);
560+
return false;
561+
}
562+
563+
long startTick = Long.parseLong(tokens[0]);
564+
long msid = Long.parseLong(tokens[1]);
565+
long runid = Long.parseLong(tokens[2]);
566+
567+
if(_mshostPeerDao.countStateSeenInPeers(msid, runid, ManagementServerHost.State.Down) > 0) {
568+
if(s_logger.isInfoEnabled())
569+
s_logger.info("Worker VM's owner management server node has been detected down from peer nodes, recycle it");
570+
return true;
571+
}
572+
573+
if(msid == _clusterMgr.getManagementNodeId() && runid != _clusterMgr.getCurrentRunId()) {
574+
if(s_logger.isInfoEnabled())
575+
s_logger.info("Worker VM's owner management server has changed runid, recycle it");
576+
return true;
577+
}
578+
579+
if(System.currentTimeMillis() - startTick > _hungWorkerTimeout) {
580+
if(s_logger.isInfoEnabled())
581+
s_logger.info("Worker VM expired, seconds elapsed: " + (System.currentTimeMillis() - startTick) / 1000);
582+
return true;
583+
}
584+
return false;
585+
}
537586

538587
@Override
539588
public void prepareSecondaryStorageStore(String storageUrl) {

‎plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareStorageManagerImpl.java‎

Lines changed: 37 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -315,46 +315,41 @@ public Answer execute(VmwareHostService hostService, BackupSnapshotCommand cmd)
315315
String prevBackupUuid = cmd.getPrevBackupUuid();
316316
VirtualMachineMO workerVm=null;
317317
String workerVMName = null;
318-
String volumePath = cmd.getVolumePath();
319-
ManagedObjectReference morDs = null;
320-
DatastoreMO dsMo=null;
321-
322-
// By default assume failure
323-
String details = null;
324-
boolean success = false;
325-
String snapshotBackupUuid = null;
326-
327-
VmwareContext context = hostService.getServiceContext(cmd);
328-
VirtualMachineMO vmMo = null;
329-
try {
330-
VmwareHypervisorHost hyperHost = hostService.getHyperHost(context, cmd);
331-
morDs = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, cmd.getPool().getUuid());
332-
333-
try {
334-
vmMo = hyperHost.findVmOnHyperHost(cmd.getVmName());
335-
if (vmMo == null) {
336-
if(s_logger.isDebugEnabled())
337-
s_logger.debug("Unable to find owner VM for BackupSnapshotCommand on host " + hyperHost.getHyperHostName() + ", will try within datacenter");
338-
339-
vmMo = hyperHost.findVmOnPeerHyperHost(cmd.getVmName());
340-
if(vmMo == null) {
341-
dsMo = new DatastoreMO(hyperHost.getContext(), morDs);
342-
343-
workerVMName = hostService.getWorkerName(context, cmd, 0);
344-
345-
// attach a volume to dummay wrapper VM for taking snapshot and exporting the VM for backup
346-
if (!hyperHost.createBlankVm(workerVMName, null, 1, 512, 0, false, 4, 0, VirtualMachineGuestOsIdentifier.OTHER_GUEST.value(), morDs, false)) {
347-
String msg = "Unable to create worker VM to execute BackupSnapshotCommand";
348-
s_logger.error(msg);
349-
throw new Exception(msg);
350-
}
351-
vmMo = hyperHost.findVmOnHyperHost(workerVMName);
352-
if (vmMo == null) {
353-
throw new Exception("Failed to find the newly create or relocated VM. vmName: " + workerVMName);
354-
}
355-
workerVm = vmMo;
356-
357-
// attach volume to worker VM
318+
String volumePath = cmd.getVolumePath();
319+
ManagedObjectReference morDs = null;
320+
DatastoreMO dsMo=null;
321+
322+
// By default assume failure
323+
String details = null;
324+
boolean success = false;
325+
String snapshotBackupUuid = null;
326+
327+
VmwareContext context = hostService.getServiceContext(cmd);
328+
VirtualMachineMO vmMo = null;
329+
try {
330+
VmwareHypervisorHost hyperHost = hostService.getHyperHost(context, cmd);
331+
morDs = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, cmd.getPool().getUuid());
332+
333+
try {
334+
vmMo = hyperHost.findVmOnHyperHost(cmd.getVmName());
335+
if (vmMo == null) {
336+
if(s_logger.isDebugEnabled()) {
337+
s_logger.debug("Unable to find owner VM for BackupSnapshotCommand on host " + hyperHost.getHyperHostName() + ", will try within datacenter");
338+
}
339+
340+
vmMo = hyperHost.findVmOnPeerHyperHost(cmd.getVmName());
341+
if(vmMo == null) {
342+
dsMo = new DatastoreMO(hyperHost.getContext(), morDs);
343+
344+
workerVMName = hostService.getWorkerName(context, cmd, 0);
345+
vmMo = HypervisorHostHelper.createWorkerVM(hyperHost, dsMo, workerVMName);
346+
347+
if (vmMo == null) {
348+
throw new Exception("Failed to find the newly create or relocated VM. vmName: " + workerVMName);
349+
}
350+
workerVm = vmMo;
351+
352+
// attach volume to worker VM
358353
String datastoreVolumePath = getVolumePathInDatastore(dsMo, volumePath + ".vmdk");
359354
vmMo.attachDisk(new String[] { datastoreVolumePath }, morDs);
360355
}
@@ -1071,28 +1066,8 @@ private Pair<String, String> copyVolumeToSecStorage(VmwareHostService hostServic
10711066
if (vmMo == null) {
10721067
// create a dummy worker vm for attaching the volume
10731068
DatastoreMO dsMo = new DatastoreMO(hyperHost.getContext(), morDs);
1074-
//restrict VM name to 32 chars, (else snapshot descriptor file name will be truncated to 32 chars of vm name)
1075-
VirtualMachineConfigSpec vmConfig = new VirtualMachineConfigSpec();
1076-
vmConfig.setName(workerVmName);
1077-
vmConfig.setMemoryMB((long) 4);
1078-
vmConfig.setNumCPUs(1);
1079-
vmConfig.setGuestId(VirtualMachineGuestOsIdentifier.OTHER_GUEST.value());
1080-
VirtualMachineFileInfo fileInfo = new VirtualMachineFileInfo();
1081-
fileInfo.setVmPathName(String.format("[%s]", dsMo.getName()));
1082-
vmConfig.setFiles(fileInfo);
1083-
1084-
// Scsi controller
1085-
VirtualLsiLogicController scsiController = new VirtualLsiLogicController();
1086-
scsiController.setSharedBus(VirtualSCSISharing.NO_SHARING);
1087-
scsiController.setBusNumber(0);
1088-
scsiController.setKey(1);
1089-
VirtualDeviceConfigSpec scsiControllerSpec = new VirtualDeviceConfigSpec();
1090-
scsiControllerSpec.setDevice(scsiController);
1091-
scsiControllerSpec.setOperation(VirtualDeviceConfigSpecOperation.ADD);
1092-
vmConfig.getDeviceChange().add(scsiControllerSpec);
1093-
1094-
hyperHost.createVm(vmConfig);
1095-
workerVm = hyperHost.findVmOnHyperHost(workerVmName);
1069+
workerVm = HypervisorHostHelper.createWorkerVM(hyperHost, dsMo, workerVmName);
1070+
10961071
if (workerVm == null) {
10971072
String msg = "Unable to create worker VM to execute CopyVolumeCommand";
10981073
s_logger.error(msg);

‎plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareContextFactory.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import org.apache.log4j.Logger;
2323
import org.springframework.stereotype.Component;
2424

25+
import com.cloud.cluster.ClusterManager;
2526
import com.cloud.hypervisor.vmware.manager.VmwareManager;
2627
import com.cloud.hypervisor.vmware.util.VmwareClient;
2728
import com.cloud.hypervisor.vmware.util.VmwareContext;
@@ -34,9 +35,11 @@ public class VmwareContextFactory {
3435

3536
private static volatile int s_seq = 1;
3637
private static VmwareManager s_vmwareMgr;
38+
private static ClusterManager s_clusterMgr;
3739
private static VmwareContextPool s_pool;
3840

3941
@Inject VmwareManager _vmwareMgr;
42+
@Inject ClusterManager _clusterMgr;
4043

4144
static {
4245
// skip certificate check
@@ -47,6 +50,7 @@ public class VmwareContextFactory {
4750
@PostConstruct
4851
void init() {
4952
s_vmwareMgr = _vmwareMgr;
53+
s_clusterMgr = _clusterMgr;
5054
}
5155

5256
public static VmwareContext create(String vCenterAddress, String vCenterUserName, String vCenterPassword) throws Exception {
@@ -66,6 +70,7 @@ public static VmwareContext create(String vCenterAddress, String vCenterUserName
6670

6771
context.registerStockObject("serviceconsole", s_vmwareMgr.getServiceConsolePortGroupName());
6872
context.registerStockObject("manageportgroup", s_vmwareMgr.getManagementPortGroupName());
73+
context.registerStockObject("noderuninfo", String.format("%d-%d", s_clusterMgr.getManagementNodeId(), s_clusterMgr.getCurrentRunId()));
6974

7075
context.setPoolInfo(s_pool, VmwareContextPool.composePoolKey(vCenterAddress, vCenterUserName));
7176
s_pool.registerOutstandingContext(context);
@@ -83,6 +88,7 @@ public static VmwareContext getContext(String vCenterAddress, String vCenterUser
8388

8489
context.registerStockObject("serviceconsole", s_vmwareMgr.getServiceConsolePortGroupName());
8590
context.registerStockObject("manageportgroup", s_vmwareMgr.getManagementPortGroupName());
91+
context.registerStockObject("noderuninfo", String.format("%d-%d", s_clusterMgr.getManagementNodeId(), s_clusterMgr.getCurrentRunId()));
8692
}
8793

8894
return context;

0 commit comments

Comments
 (0)