Skip to content

Commit 670fc7e

Browse files
kelvenykelveny
committed
CLOUDSTACK-4659: Add the missing feature back for GC VMware worker VMs
1 parent 93d8b96 commit 670fc7e

13 files changed

Lines changed: 153 additions & 152 deletions

File tree

‎agent/src/com/cloud/agent/resource/consoleproxy/ConsoleProxyResource.java‎

Lines changed: 14 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,7 @@ public Answer executeRequest(final Command cmd) {
112112
}
113113

114114
private Answer execute(StartConsoleProxyAgentHttpHandlerCommand cmd) {
115+
s_logger.info("Invoke launchConsoleProxy() in responding to StartConsoleProxyAgentHttpHandlerCommand");
115116
launchConsoleProxy(cmd.getKeystoreBits(), cmd.getKeystorePassword(), cmd.getEncryptorPassword());
116117
return new Answer(cmd);
117118
}
@@ -361,29 +362,31 @@ public void run() {
361362
try {
362363
Class<?> consoleProxyClazz = Class.forName("com.cloud.consoleproxy.ConsoleProxy");
363364
try {
365+
s_logger.info("Invoke setEncryptorPassword(), ecnryptorPassword: " + encryptorPassword);
364366
Method methodSetup = consoleProxyClazz.getMethod(
365367
"setEncryptorPassword", String.class);
366368
methodSetup.invoke(null, encryptorPassword);
367369

370+
s_logger.info("Invoke startWithContext()");
368371
Method method = consoleProxyClazz.getMethod(
369372
"startWithContext", Properties.class,
370373
Object.class, byte[].class, String.class);
371374
method.invoke(null, _properties, resource, ksBits,
372375
ksPassword);
373376
} catch (SecurityException e) {
374-
s_logger.error("Unable to launch console proxy due to SecurityException");
377+
s_logger.error("Unable to launch console proxy due to SecurityException", e);
375378
System.exit(ExitStatus.Error.value());
376379
} catch (NoSuchMethodException e) {
377-
s_logger.error("Unable to launch console proxy due to NoSuchMethodException");
380+
s_logger.error("Unable to launch console proxy due to NoSuchMethodException", e);
378381
System.exit(ExitStatus.Error.value());
379382
} catch (IllegalArgumentException e) {
380-
s_logger.error("Unable to launch console proxy due to IllegalArgumentException");
383+
s_logger.error("Unable to launch console proxy due to IllegalArgumentException", e);
381384
System.exit(ExitStatus.Error.value());
382385
} catch (IllegalAccessException e) {
383-
s_logger.error("Unable to launch console proxy due to IllegalAccessException");
386+
s_logger.error("Unable to launch console proxy due to IllegalAccessException", e);
384387
System.exit(ExitStatus.Error.value());
385388
} catch (InvocationTargetException e) {
386-
s_logger.error("Unable to launch console proxy due to InvocationTargetException");
389+
s_logger.error("Unable to launch console proxy due to InvocationTargetException " + e.getTargetException().toString(), e);
387390
System.exit(ExitStatus.Error.value());
388391
}
389392
} catch (final ClassNotFoundException e) {
@@ -402,22 +405,22 @@ public void run() {
402405
Method methodSetup = consoleProxyClazz.getMethod("setEncryptorPassword", String.class);
403406
methodSetup.invoke(null, encryptorPassword);
404407
} catch (SecurityException e) {
405-
s_logger.error("Unable to launch console proxy due to SecurityException");
408+
s_logger.error("Unable to launch console proxy due to SecurityException", e);
406409
System.exit(ExitStatus.Error.value());
407410
} catch (NoSuchMethodException e) {
408-
s_logger.error("Unable to launch console proxy due to NoSuchMethodException");
411+
s_logger.error("Unable to launch console proxy due to NoSuchMethodException", e);
409412
System.exit(ExitStatus.Error.value());
410413
} catch (IllegalArgumentException e) {
411-
s_logger.error("Unable to launch console proxy due to IllegalArgumentException");
414+
s_logger.error("Unable to launch console proxy due to IllegalArgumentException", e);
412415
System.exit(ExitStatus.Error.value());
413416
} catch (IllegalAccessException e) {
414-
s_logger.error("Unable to launch console proxy due to IllegalAccessException");
417+
s_logger.error("Unable to launch console proxy due to IllegalAccessException", e);
415418
System.exit(ExitStatus.Error.value());
416419
} catch (InvocationTargetException e) {
417-
s_logger.error("Unable to launch console proxy due to InvocationTargetException");
420+
s_logger.error("Unable to launch console proxy due to InvocationTargetException " + e.getTargetException().toString(), e);
418421
System.exit(ExitStatus.Error.value());
419422
} catch (final ClassNotFoundException e) {
420-
s_logger.error("Unable to launch console proxy due to ClassNotFoundException");
423+
s_logger.error("Unable to launch console proxy due to ClassNotFoundException", e);
421424
System.exit(ExitStatus.Error.value());
422425
}
423426
}

‎plugins/hypervisors/vmware/src/com/cloud/hypervisor/guru/VMwareGuru.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -374,6 +374,7 @@ public Pair<Boolean, Long> getCommandHostDelegation(long hostId, Command cmd) {
374374
CommandExecLogVO execLog = new CommandExecLogVO(cmdTarget.first().getId(), cmdTarget.second().getId(), cmd.getClass().getSimpleName(), 1);
375375
_cmdExecLogDao.persist(execLog);
376376
cmd.setContextParam("execid", String.valueOf(execLog.getId()));
377+
cmd.setContextParam("noderuninfo", String.format("%d-%d", _clusterMgr.getManagementNodeId(), _clusterMgr.getCurrentRunId()));
377378

378379
if(cmd instanceof BackupSnapshotCommand ||
379380
cmd instanceof CreatePrivateTemplateFromVolumeCommand ||

‎plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareManager.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ List<ManagedObjectReference> addHostToPodCluster(VmwareContext serviceContext, l
5252

5353
VmwareStorageManager getStorageManager();
5454
void gcLeftOverVMs(VmwareContext context);
55+
boolean needRecycle(String workerTag);
5556

5657
Pair<Integer, Integer> getAddiionalVncPortRange();
5758

‎plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java‎

Lines changed: 48 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,8 @@
5252
import com.cloud.agent.api.StartupCommand;
5353
import com.cloud.agent.api.StartupRoutingCommand;
5454
import com.cloud.cluster.ClusterManager;
55+
import com.cloud.cluster.ManagementServerHost;
56+
import com.cloud.cluster.dao.ManagementServerHostPeerDao;
5557
import com.cloud.configuration.Config;
5658
import com.cloud.configuration.dao.ConfigurationDao;
5759
import com.cloud.dc.ClusterDetailsDao;
@@ -154,6 +156,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw
154156
@Inject VmwareDatacenterDao _vmwareDcDao;
155157
@Inject VmwareDatacenterZoneMapDao _vmwareDcZoneMapDao;
156158
@Inject LegacyZoneDao _legacyZoneDao;
159+
@Inject ManagementServerHostPeerDao _mshostPeerDao;
157160

158161
String _mountParent;
159162
StorageLayer _storage;
@@ -167,6 +170,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw
167170
String _managemetPortGroupName;
168171
String _defaultSystemVmNicAdapterType = VirtualEthernetCardType.E1000.toString();
169172
String _recycleHungWorker = "false";
173+
long _hungWorkerTimeout = 7200000; // 2 hour
170174
int _additionalPortRangeStart;
171175
int _additionalPortRangeSize;
172176
int _routerExtraPublicNics = 2;
@@ -285,6 +289,10 @@ public boolean configure(String name, Map<String, Object> params) throws Configu
285289
if(_recycleHungWorker == null || _recycleHungWorker.isEmpty()) {
286290
_recycleHungWorker = "false";
287291
}
292+
293+
value = _configDao.getValue(Config.VmwareHungWorkerTimeout.key());
294+
if(value != null)
295+
_hungWorkerTimeout = Long.parseLong(value) * 1000;
288296

289297
_rootDiskController = _configDao.getValue(Config.VmwareRootDiskControllerType.key());
290298
if(_rootDiskController == null || _rootDiskController.isEmpty()) {
@@ -528,11 +536,50 @@ public VmwareStorageManager getStorageManager() {
528536
return _storageMgr;
529537
}
530538

531-
532539
@Override
533540
public void gcLeftOverVMs(VmwareContext context) {
534541
VmwareCleanupMaid.gcLeftOverVMs(context);
535542
}
543+
544+
@Override
545+
public boolean needRecycle(String workerTag) {
546+
if(s_logger.isInfoEnabled())
547+
s_logger.info("Check to see if a worker VM with tag " + workerTag + " needs to be recycled");
548+
549+
if(workerTag == null || workerTag.isEmpty()) {
550+
s_logger.error("Invalid worker VM tag " + workerTag);
551+
return false;
552+
}
553+
554+
String tokens[] = workerTag.split("-");
555+
if(tokens.length != 3) {
556+
s_logger.error("Invalid worker VM tag " + workerTag);
557+
return false;
558+
}
559+
560+
long startTick = Long.parseLong(tokens[0]);
561+
long msid = Long.parseLong(tokens[1]);
562+
long runid = Long.parseLong(tokens[2]);
563+
564+
if(_mshostPeerDao.countStateSeenInPeers(msid, runid, ManagementServerHost.State.Down) > 0) {
565+
if(s_logger.isInfoEnabled())
566+
s_logger.info("Worker VM's owner management server node has been detected down from peer nodes, recycle it");
567+
return true;
568+
}
569+
570+
if(msid == _clusterMgr.getManagementNodeId() && runid != _clusterMgr.getCurrentRunId()) {
571+
if(s_logger.isInfoEnabled())
572+
s_logger.info("Worker VM's owner management server has changed runid, recycle it");
573+
return true;
574+
}
575+
576+
if(System.currentTimeMillis() - startTick > _hungWorkerTimeout) {
577+
if(s_logger.isInfoEnabled())
578+
s_logger.info("Worker VM expired, seconds elapsed: " + (System.currentTimeMillis() - startTick) / 1000);
579+
return true;
580+
}
581+
return false;
582+
}
536583

537584
@Override
538585
public void prepareSecondaryStorageStore(String storageUrl) {

‎plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareStorageManagerImpl.java‎

Lines changed: 4 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -329,14 +329,8 @@ public Answer execute(VmwareHostService hostService, BackupSnapshotCommand cmd)
329329
dsMo = new DatastoreMO(hyperHost.getContext(), morDs);
330330

331331
workerVMName = hostService.getWorkerName(context, cmd, 0);
332-
333-
// attach a volume to dummay wrapper VM for taking snapshot and exporting the VM for backup
334-
if (!hyperHost.createBlankVm(workerVMName, null, 1, 512, 0, false, 4, 0, VirtualMachineGuestOsIdentifier.OTHER_GUEST.value(), morDs, false)) {
335-
String msg = "Unable to create worker VM to execute BackupSnapshotCommand";
336-
s_logger.error(msg);
337-
throw new Exception(msg);
338-
}
339-
vmMo = hyperHost.findVmOnHyperHost(workerVMName);
332+
vmMo = HypervisorHostHelper.createWorkerVM(hyperHost, dsMo, workerVMName);
333+
340334
if (vmMo == null) {
341335
throw new Exception("Failed to find the newly create or relocated VM. vmName: " + workerVMName);
342336
}
@@ -1059,28 +1053,8 @@ private Pair<String, String> copyVolumeToSecStorage(VmwareHostService hostServic
10591053
if (vmMo == null) {
10601054
// create a dummy worker vm for attaching the volume
10611055
DatastoreMO dsMo = new DatastoreMO(hyperHost.getContext(), morDs);
1062-
//restrict VM name to 32 chars, (else snapshot descriptor file name will be truncated to 32 chars of vm name)
1063-
VirtualMachineConfigSpec vmConfig = new VirtualMachineConfigSpec();
1064-
vmConfig.setName(workerVmName);
1065-
vmConfig.setMemoryMB((long) 4);
1066-
vmConfig.setNumCPUs(1);
1067-
vmConfig.setGuestId(VirtualMachineGuestOsIdentifier.OTHER_GUEST.value());
1068-
VirtualMachineFileInfo fileInfo = new VirtualMachineFileInfo();
1069-
fileInfo.setVmPathName(String.format("[%s]", dsMo.getName()));
1070-
vmConfig.setFiles(fileInfo);
1071-
1072-
// Scsi controller
1073-
VirtualLsiLogicController scsiController = new VirtualLsiLogicController();
1074-
scsiController.setSharedBus(VirtualSCSISharing.NO_SHARING);
1075-
scsiController.setBusNumber(0);
1076-
scsiController.setKey(1);
1077-
VirtualDeviceConfigSpec scsiControllerSpec = new VirtualDeviceConfigSpec();
1078-
scsiControllerSpec.setDevice(scsiController);
1079-
scsiControllerSpec.setOperation(VirtualDeviceConfigSpecOperation.ADD);
1080-
vmConfig.getDeviceChange().add(scsiControllerSpec);
1081-
1082-
hyperHost.createVm(vmConfig);
1083-
workerVm = hyperHost.findVmOnHyperHost(workerVmName);
1056+
workerVm = HypervisorHostHelper.createWorkerVM(hyperHost, dsMo, workerVmName);
1057+
10841058
if (workerVm == null) {
10851059
String msg = "Unable to create worker VM to execute CopyVolumeCommand";
10861060
s_logger.error(msg);

‎plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareContextFactory.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import org.apache.log4j.Logger;
2323
import org.springframework.stereotype.Component;
2424

25+
import com.cloud.cluster.ClusterManager;
2526
import com.cloud.hypervisor.vmware.manager.VmwareManager;
2627
import com.cloud.hypervisor.vmware.util.VmwareClient;
2728
import com.cloud.hypervisor.vmware.util.VmwareContext;
@@ -34,9 +35,11 @@ public class VmwareContextFactory {
3435

3536
private static volatile int s_seq = 1;
3637
private static VmwareManager s_vmwareMgr;
38+
private static ClusterManager s_clusterMgr;
3739
private static VmwareContextPool s_pool;
3840

3941
@Inject VmwareManager _vmwareMgr;
42+
@Inject ClusterManager _clusterMgr;
4043

4144
static {
4245
// skip certificate check
@@ -47,6 +50,7 @@ public class VmwareContextFactory {
4750
@PostConstruct
4851
void init() {
4952
s_vmwareMgr = _vmwareMgr;
53+
s_clusterMgr = _clusterMgr;
5054
}
5155

5256
public static VmwareContext create(String vCenterAddress, String vCenterUserName, String vCenterPassword) throws Exception {
@@ -66,6 +70,7 @@ public static VmwareContext create(String vCenterAddress, String vCenterUserName
6670

6771
context.registerStockObject("serviceconsole", s_vmwareMgr.getServiceConsolePortGroupName());
6872
context.registerStockObject("manageportgroup", s_vmwareMgr.getManagementPortGroupName());
73+
context.registerStockObject("noderuninfo", String.format("%d-%d", s_clusterMgr.getManagementNodeId(), s_clusterMgr.getCurrentRunId()));
6974

7075
context.setPoolInfo(s_pool, VmwareContextPool.composePoolKey(vCenterAddress, vCenterUserName));
7176
s_pool.registerOutstandingContext(context);
@@ -83,6 +88,7 @@ public static VmwareContext getContext(String vCenterAddress, String vCenterUser
8388

8489
context.registerStockObject("serviceconsole", s_vmwareMgr.getServiceConsolePortGroupName());
8590
context.registerStockObject("manageportgroup", s_vmwareMgr.getManagementPortGroupName());
91+
context.registerStockObject("noderuninfo", String.format("%d-%d", s_clusterMgr.getManagementNodeId(), s_clusterMgr.getCurrentRunId()));
8692
}
8793

8894
return context;

0 commit comments

Comments
 (0)