Skip to content

Commit 867e916

Browse files
marcaurelemarcaurele
committed
Live migration fixes (#20)
Multiple bugs fixes: * Incorrect disk size: when provisioning the volume at the destination, the size was taken from the template and not from the original volume itself. * Incorrect details when returning an error: now if an exception is raised by Libvirt at the agent level, the error message is forwarded correctly all the way into the job response text. * Incorrect volume state: when a migration failed, the volume state was inconsistent and stayed in "Migrating", those interfering with future commands on the VM. As a result, the event message wasn't sent either.
1 parent ae0931e commit 867e916

11 files changed

Lines changed: 69 additions & 66 deletions

File tree

‎engine/api/src/com/cloud/vm/VirtualMachineManager.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import java.util.List;
2222
import java.util.Map;
2323

24+
import com.cloud.exception.VirtualMachineMigrationException;
2425
import org.apache.cloudstack.framework.config.ConfigKey;
2526

2627
import com.cloud.agent.api.to.NicTO;
@@ -114,7 +115,7 @@ void orchestrateStart(String vmUuid, Map<VirtualMachineProfile.Param, Object> pa
114115

115116
void migrate(String vmUuid, long srcHostId, DeployDestination dest) throws ResourceUnavailableException, ConcurrentOperationException;
116117

117-
void migrateWithStorage(String vmUuid, long srcId, long destId, Map<Long, Long> volumeToPool) throws ResourceUnavailableException, ConcurrentOperationException;
118+
void migrateWithStorage(String vmUuid, long srcId, long destId, Map<Long, Long> volumeToPool) throws ResourceUnavailableException, ConcurrentOperationException, VirtualMachineMigrationException;
118119

119120
void reboot(String vmUuid, Map<VirtualMachineProfile.Param, Object> params) throws InsufficientCapacityException, ResourceUnavailableException;
120121

‎engine/api/src/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import java.util.Map;
2222
import java.util.Set;
2323

24+
import com.cloud.exception.VirtualMachineMigrationException;
2425
import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
2526
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
2627
import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo;
@@ -102,7 +103,7 @@ VolumeInfo moveVolume(VolumeInfo volume, long destPoolDcId, Long destPoolPodId,
102103

103104
void revokeAccess(long vmId, long hostId);
104105

105-
void migrateVolumes(VirtualMachine vm, VirtualMachineTO vmTo, Host srcHost, Host destHost, Map<Volume, StoragePool> volumeToPool);
106+
void liveMigrateVolumes(VirtualMachine vm, VirtualMachineTO vmTo, Host srcHost, Host destHost, Map<Volume, StoragePool> volumeToPool) throws VirtualMachineMigrationException;
106107

107108
boolean storageMigration(VirtualMachineProfile vm, StoragePool destPool) throws StorageUnavailableException;
108109

‎engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/CopyCommandResult.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,4 +39,10 @@ public String getPath() {
3939
public Answer getAnswer() {
4040
return this.answer;
4141
}
42+
43+
@Override
44+
public String toString() {
45+
String sup = super.toString();
46+
return sup.substring(0, sup.length()-1) + ", path: " + (path == null ? "<null>" : path) + ", answer: " + (answer == null ? "<null>" : answer) + "}";
47+
}
4248
}

‎engine/api/src/org/apache/cloudstack/storage/command/CommandResult.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,4 +59,9 @@ public void setResult(String result) {
5959
this.success = false;
6060
}
6161
}
62+
63+
@Override
64+
public String toString() {
65+
return "CommandResult={success: " + success + ", aborted: " + aborted + ", result: " + (result == null ? "<null>" : result) + "}";
66+
}
6267
}

‎engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java‎

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,8 @@
3838
import javax.naming.ConfigurationException;
3939

4040
import com.cloud.agent.api.AttachOrDettachConfigDriveCommand;
41+
import com.cloud.exception.VirtualMachineMigrationException;
42+
4143
import org.apache.cloudstack.affinity.dao.AffinityGroupVMMapDao;
4244
import org.apache.cloudstack.context.CallContext;
4345
import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
@@ -798,7 +800,7 @@ public void advanceStart(final String vmUuid, final Map<VirtualMachineProfile.Pa
798800
} catch (final InterruptedException e) {
799801
throw new RuntimeException("Operation is interrupted", e);
800802
} catch (final java.util.concurrent.ExecutionException e) {
801-
throw new RuntimeException("Execution excetion", e);
803+
throw new RuntimeException("Execution exception", e);
802804
}
803805

804806
final Object jobResult = _jobMgr.unmarshallResultObject(outcome.getJob());
@@ -2148,7 +2150,7 @@ private <T extends VMInstanceVO> void moveVmOutofMigratingStateOnSuccess(final T
21482150

21492151
@Override
21502152
public void migrateWithStorage(final String vmUuid, final long srcHostId, final long destHostId, final Map<Long, Long> volumeToPool)
2151-
throws ResourceUnavailableException, ConcurrentOperationException {
2153+
throws ResourceUnavailableException, ConcurrentOperationException, VirtualMachineMigrationException {
21522154

21532155
final AsyncJobExecutionContext jobContext = AsyncJobExecutionContext.getCurrentExecutionContext();
21542156
if (jobContext.isJobDispatchedBy(VmWorkConstants.VM_WORK_JOB_DISPATCHER)) {
@@ -2192,7 +2194,7 @@ public void migrateWithStorage(final String vmUuid, final long srcHostId, final
21922194
}
21932195

21942196
private void orchestrateMigrateWithStorage(final String vmUuid, final long srcHostId, final long destHostId, final Map<Long, Long> volumeToPool) throws ResourceUnavailableException,
2195-
ConcurrentOperationException {
2197+
ConcurrentOperationException, VirtualMachineMigrationException {
21962198

21972199
final VMInstanceVO vm = _vmDao.findByUuid(vmUuid);
21982200

@@ -2204,8 +2206,7 @@ private void orchestrateMigrateWithStorage(final String vmUuid, final long srcHo
22042206
final HostPodVO pod = _podDao.findById(destHost.getPodId());
22052207
final Cluster cluster = _clusterDao.findById(destHost.getClusterId());
22062208
final DeployDestination destination = new DeployDestination(dc, pod, cluster, destHost);
2207-
2208-
VirtualMachineProfile vmSrc = new VirtualMachineProfileImpl(vm);
2209+
final VirtualMachineProfile vmSrc = new VirtualMachineProfileImpl(vm);
22092210
for (NicProfile nic : _networkMgr.getNicProfiles(vm)) {
22102211
vmSrc.addNic(nic);
22112212
}
@@ -2288,7 +2289,7 @@ private void orchestrateMigrateWithStorage(final String vmUuid, final long srcHo
22882289
}
22892290

22902291
// Migrate the vm and its volume.
2291-
volumeMgr.migrateVolumes(vm, to, srcHost, destHost, volumeToPoolMap);
2292+
volumeMgr.liveMigrateVolumes(vm, to, srcHost, destHost, volumeToPoolMap);
22922293
migrated = true;
22932294

22942295
try {
@@ -2325,10 +2326,13 @@ private void orchestrateMigrateWithStorage(final String vmUuid, final long srcHo
23252326

23262327
volumeMgr.confirmMigration(profile, srcHostId, destHostId, migrated);
23272328

2328-
23292329
work.setStep(Step.Done);
23302330
_workDao.update(work.getId(), work);
23312331
}
2332+
2333+
if (!migrated) {
2334+
throw new VirtualMachineMigrationException("Migration failed");
2335+
}
23322336
}
23332337

23342338
@Override

‎engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java‎

Lines changed: 5 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -30,9 +30,6 @@
3030
import javax.inject.Inject;
3131
import javax.naming.ConfigurationException;
3232

33-
import com.cloud.storage.StorageManager;
34-
import com.cloud.storage.VMTemplateVO;
35-
import com.cloud.storage.dao.VMTemplateDao;
3633
import org.apache.log4j.Logger;
3734

3835
import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService;
@@ -86,6 +83,7 @@
8683
import com.cloud.exception.InsufficientStorageCapacityException;
8784
import com.cloud.exception.InvalidParameterValueException;
8885
import com.cloud.exception.StorageUnavailableException;
86+
import com.cloud.exception.VirtualMachineMigrationException;
8987
import com.cloud.host.Host;
9088
import com.cloud.host.HostVO;
9189
import com.cloud.host.dao.HostDao;
@@ -100,11 +98,13 @@
10098
import com.cloud.storage.Storage.ImageFormat;
10199
import com.cloud.storage.StorageManager;
102100
import com.cloud.storage.StoragePool;
101+
import com.cloud.storage.VMTemplateVO;
103102
import com.cloud.storage.VMTemplateStorageResourceAssoc;
104103
import com.cloud.storage.Volume;
105104
import com.cloud.storage.Volume.Type;
106105
import com.cloud.storage.VolumeVO;
107106
import com.cloud.storage.dao.SnapshotDao;
107+
import com.cloud.storage.dao.VMTemplateDao;
108108
import com.cloud.storage.dao.VolumeDao;
109109
import com.cloud.storage.dao.VolumeDetailsDao;
110110
import com.cloud.template.TemplateManager;
@@ -980,28 +980,8 @@ public Volume migrateVolume(Volume volume, StoragePool destPool) throws StorageU
980980
}
981981
}
982982

983-
@DB
984-
protected Volume liveMigrateVolume(Volume volume, StoragePool destPool) {
985-
VolumeInfo vol = volFactory.getVolume(volume.getId());
986-
AsyncCallFuture<VolumeApiResult> future = volService.migrateVolume(vol, (DataStore)destPool);
987-
try {
988-
VolumeApiResult result = future.get();
989-
if (result.isFailed()) {
990-
s_logger.debug("migrate volume failed:" + result.getResult());
991-
return null;
992-
}
993-
return result.getVolume();
994-
} catch (InterruptedException e) {
995-
s_logger.debug("migrate volume failed", e);
996-
return null;
997-
} catch (ExecutionException e) {
998-
s_logger.debug("migrate volume failed", e);
999-
return null;
1000-
}
1001-
}
1002-
1003983
@Override
1004-
public void migrateVolumes(VirtualMachine vm, VirtualMachineTO vmTo, Host srcHost, Host destHost, Map<Volume, StoragePool> volumeToPool) {
984+
public void liveMigrateVolumes(VirtualMachine vm, VirtualMachineTO vmTo, Host srcHost, Host destHost, Map<Volume, StoragePool> volumeToPool) throws VirtualMachineMigrationException {
1005985
// Check if all the vms being migrated belong to the vm.
1006986
// Check if the storage pool is of the right type.
1007987
// Create a VolumeInfo to DataStore map too.
@@ -1027,7 +1007,7 @@ public void migrateVolumes(VirtualMachine vm, VirtualMachineTO vmTo, Host srcHos
10271007
CommandResult result = future.get();
10281008
if (result.isFailed()) {
10291009
s_logger.debug("Failed to migrate vm " + vm + " along with its volumes. " + result.getResult());
1030-
throw new CloudRuntimeException("Failed to migrate vm " + vm + " along with its volumes.");
1010+
throw new VirtualMachineMigrationException("Failed to migrate vm " + vm + " along with its volumes.");
10311011
}
10321012
} catch (InterruptedException e) {
10331013
s_logger.debug("Failed to migrated vm " + vm + " along with its volumes.", e);

‎engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java‎

Lines changed: 13 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1573,29 +1573,23 @@ protected Void migrateVmWithVolumesCallBack(AsyncCallbackDispatcher<VolumeServic
15731573
CopyCommandResult result = callback.getResult();
15741574
AsyncCallFuture<CommandResult> future = context.future;
15751575
CommandResult res = new CommandResult();
1576-
try {
1577-
if (result.isFailed()) {
1578-
MigrateWithStorageAnswer answer = (MigrateWithStorageAnswer)result.getAnswer();
1579-
res.setSuccess(false);
1576+
1577+
if (result.isFailed()) {
1578+
res.setSuccess(false);
1579+
MigrateWithStorageAnswer answer = (MigrateWithStorageAnswer)result.getAnswer();
1580+
if (answer != null) {
15801581
res.setAborted(answer.isAborted());
1581-
res.setResult(result.getResult());
1582-
for (Map.Entry<VolumeInfo, DataStore> entry : volumeToPool.entrySet()) {
1583-
VolumeInfo volume = entry.getKey();
1584-
volume.processEvent(Event.OperationFailed);
1585-
}
1586-
} else {
1587-
for (Map.Entry<VolumeInfo, DataStore> entry : volumeToPool.entrySet()) {
1588-
VolumeInfo volume = entry.getKey();
1589-
volume.processEvent(Event.OperationSuccessed);
1590-
}
15911582
}
1592-
future.complete(res);
1593-
} catch (Exception e) {
1594-
s_logger.error("Failed to process migration volume callback", e);
1595-
res.setResult(e.toString());
1596-
future.complete(res);
1583+
res.setResult(result.getResult());
15971584
}
15981585

1586+
for (Map.Entry<VolumeInfo, DataStore> entry : volumeToPool.entrySet()) {
1587+
VolumeInfo volume = entry.getKey();
1588+
volume.processEvent(result.isFailed() || result.isAborted() ? Event.OperationFailed : Event.OperationSuccessed);
1589+
}
1590+
1591+
future.complete(res);
1592+
15991593
return null;
16001594
}
16011595

‎plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateCommandWrapper.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,10 @@ public Answer execute(final CreateCommand command, final LibvirtComputingResourc
5959
vol = libvirtComputingResource.templateToPrimaryDownload(command.getTemplateUrl(), primaryPool, dskch.getPath());
6060
} else {
6161
baseVol = primaryPool.getPhysicalDisk(command.getTemplateUrl());
62-
vol = storagePoolMgr.createDiskFromTemplate(baseVol, dskch.getPath(), dskch.getProvisioningType(), primaryPool, 0);
62+
if (baseVol.getSize() > disksize) {
63+
disksize = baseVol.getSize();
64+
}
65+
vol = storagePoolMgr.createDiskFromTemplate(baseVol, dskch.getPath(), dskch.getProvisioningType(), primaryPool, disksize, 0);
6366
}
6467
if (vol == null) {
6568
return new Answer(command, false, " Can't create storage volume on storage pool");

‎plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -336,11 +336,6 @@ public boolean deleteStoragePool(StoragePoolType type, String uuid) {
336336
return true;
337337
}
338338

339-
public KVMPhysicalDisk createDiskFromTemplate(KVMPhysicalDisk template, String name, Storage.ProvisioningType provisioningType,
340-
KVMStoragePool destPool, int timeout) {
341-
return createDiskFromTemplate(template, name, provisioningType, destPool, template.getSize(), timeout);
342-
}
343-
344339
public KVMPhysicalDisk createDiskFromTemplate(KVMPhysicalDisk template, String name, Storage.ProvisioningType provisioningType,
345340
KVMStoragePool destPool, long size, int timeout) {
346341
StorageAdaptor adaptor = getStorageAdaptor(destPool.getType());

‎plugins/hypervisors/kvm/src/org/apache/cloudstack/storage/motion/KVMStorageMotionStrategy.java‎

Lines changed: 20 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -121,21 +121,31 @@ public void copyAsync(Map<VolumeInfo, DataStore> volumeMap, VirtualMachineTO vmT
121121
if (instance != null) {
122122
answer = migrateVmWithVolumes(instance, vmTo, srcHost, destHost, volumeMap);
123123
errMsg = answer.getDetails();
124+
125+
if (s_logger.isDebugEnabled()) {
126+
s_logger.debug("Got answer from migration, result: " + (answer.getResult() ? "ok" : "failed, reason: " + (answer.getDetails() == null ? "<null>" : answer.getDetails())));
127+
}
128+
124129
} else {
125130
throw new CloudRuntimeException("Unsupported operation requested for moving data.");
126131
}
127132
} catch (Exception e) {
128133
s_logger.error("copyAsync failed", e);
129-
errMsg = e.toString();
134+
errMsg = e.getMessage();
130135
}
131136

132-
if (s_logger.isDebugEnabled()) {
133-
s_logger.debug("Got answer from migration, result: " + (answer.getResult() ? "ok" : "failed, reason: " + answer.getDetails()));
137+
CopyCommandResult result = new CopyCommandResult("", answer);
138+
139+
if (answer != null) {
140+
result.setSuccess(answer.getResult());
141+
if (!answer.getResult()) {
142+
result.setResult(errMsg);
143+
}
144+
} else {
145+
result.setSuccess(false);
146+
result.setResult(errMsg);
134147
}
135148

136-
CopyCommandResult result = new CopyCommandResult(null, answer);
137-
result.setResult(errMsg);
138-
result.setSuccess(answer.getResult());
139149
callback.complete(result);
140150
}
141151

@@ -161,6 +171,10 @@ private Answer migrateVmWithVolumes(VMInstanceVO vm, VirtualMachineTO to, Host s
161171
StoragePool destStoragePool = storageManager.findLocalStorageOnHost(destHost.getId());
162172
TemplateInfo templateImage = tmplFactory.getTemplate(rootVolume.getTemplateId(), DataStoreRole.Image);
163173

174+
if (s_logger.isDebugEnabled()) {
175+
s_logger.debug("Provisioning disk " + diskProfile.toString() + " on destination host");
176+
}
177+
164178
try {
165179
CreateCommand provisioningCommand = new CreateCommand(diskProfile, templateImage.getUuid(), destStoragePool, true);
166180
CreateAnswer provisioningAnwer = (CreateAnswer) agentMgr.send(destHost.getId(), provisioningCommand);

0 commit comments

Comments
 (0)