Skip to content

Commit 693f0c2

Browse files
minchen07minchen07
committed
CLOUDSTACK-5841:Snapshots taken before migration NFS to S3 can not be
used cross-zone.
1 parent 62aa147 commit 693f0c2

10 files changed

Lines changed: 178 additions & 23 deletions

File tree

‎engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotDataFactory.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,4 +30,6 @@ public interface SnapshotDataFactory {
3030
SnapshotInfo getSnapshot(long snapshotId, DataStoreRole role);
3131

3232
List<SnapshotInfo> listSnapshotOnCache(long snapshotId);
33+
34+
SnapshotInfo getReadySnapshotOnCache(long snapshotId);
3335
}

‎engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotService.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,4 +25,6 @@ public interface SnapshotService {
2525
boolean deleteSnapshot(SnapshotInfo snapshot);
2626

2727
boolean revertSnapshot(Long snapshotId);
28+
29+
void syncVolumeSnapshotsToRegionStore(long volumeId, DataStore store);
2830
}

‎engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,12 +31,14 @@
3131
import javax.naming.ConfigurationException;
3232

3333
import org.apache.log4j.Logger;
34+
3435
import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService;
3536
import org.apache.cloudstack.engine.subsystem.api.storage.ChapInfo;
3637
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
3738
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
3839
import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory;
3940
import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
41+
import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotService;
4042
import org.apache.cloudstack.engine.subsystem.api.storage.StoragePoolAllocator;
4143
import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory;
4244
import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo;
@@ -144,6 +146,8 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati
144146
ConfigDepot _configDepot;
145147
@Inject
146148
HostDao _hostDao;
149+
@Inject
150+
SnapshotService _snapshotSrv;
147151

148152
private final StateMachine2<Volume.State, Volume.Event, Volume> _volStateMachine;
149153
protected List<StoragePoolAllocator> _storagePoolAllocators;
@@ -346,6 +350,16 @@ public VolumeInfo createVolumeFromSnapshot(Volume volume, Snapshot snapshot, Use
346350
VolumeInfo vol = volFactory.getVolume(volume.getId());
347351
DataStore store = dataStoreMgr.getDataStore(pool.getId(), DataStoreRole.Primary);
348352
SnapshotInfo snapInfo = snapshotFactory.getSnapshot(snapshot.getId(), DataStoreRole.Image);
353+
// sync snapshot to region store if necessary
354+
DataStore snapStore = snapInfo.getDataStore();
355+
long snapVolId = snapInfo.getVolumeId();
356+
try {
357+
_snapshotSrv.syncVolumeSnapshotsToRegionStore(snapVolId, snapStore);
358+
} catch (Exception ex) {
359+
// log but ignore the sync error to avoid any potential S3 down issue, it should be sync next time
360+
s_logger.warn(ex.getMessage(), ex);
361+
}
362+
// create volume on primary from snapshot
349363
AsyncCallFuture<VolumeApiResult> future = volService.createVolumeFromSnapshot(vol, store, snapInfo);
350364
try {
351365
VolumeApiResult result = future.get();
@@ -771,7 +785,7 @@ protected VolumeVO switchVolume(final VolumeVO existingVolume, final VirtualMach
771785
templateIdToUse = vmTemplateId;
772786
}
773787

774-
final Long templateIdToUseFinal = templateIdToUse;
788+
final Long templateIdToUseFinal = templateIdToUse;
775789
return Transaction.execute(new TransactionCallback<VolumeVO>() {
776790
@Override
777791
public VolumeVO doInTransaction(TransactionStatus status) {

‎engine/schema/src/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,8 @@ public interface SnapshotDataStoreDao extends GenericDao<SnapshotDataStoreVO, Lo
4747
// delete the snapshot entry on primary data store to make sure that next snapshot will be full snapshot
4848
void deleteSnapshotRecordsOnPrimary();
4949

50+
SnapshotDataStoreVO findReadyOnCache(long snapshotId);
51+
5052
List<SnapshotDataStoreVO> listOnCache(long snapshotId);
5153

5254
void updateStoreRoleToCache(long storeId);

‎engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -704,6 +704,9 @@ protected Void syncTemplateCallBack(AsyncCallbackDispatcher<TemplateServiceImpl,
704704
@Override
705705
public void syncTemplateToRegionStore(long templateId, DataStore store) {
706706
if (_storeMgr.isRegionStore(store)) {
707+
if (s_logger.isDebugEnabled()) {
708+
s_logger.debug("Sync template " + templateId + " from cache to object store...");
709+
}
707710
// if template is on region wide object store, check if it is really downloaded there (by checking install_path). Sync template to region
708711
// wide store if it is not there physically.
709712
TemplateInfo tmplOnStore = _templateFactory.getTemplate(templateId, store);

‎engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotDataFactoryImpl.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,18 @@ public SnapshotInfo getSnapshot(long snapshotId, DataStoreRole role) {
7979
return so;
8080
}
8181

82+
@Override
83+
public SnapshotInfo getReadySnapshotOnCache(long snapshotId) {
84+
SnapshotDataStoreVO snapStore = snapshotStoreDao.findReadyOnCache(snapshotId);
85+
if (snapStore != null) {
86+
DataStore store = storeMgr.getDataStore(snapStore.getDataStoreId(), DataStoreRole.ImageCache);
87+
return getSnapshot(snapshotId, store);
88+
} else {
89+
return null;
90+
}
91+
92+
}
93+
8294
@Override
8395
public List<SnapshotInfo> listSnapshotOnCache(long snapshotId) {
8496
List<SnapshotDataStoreVO> cacheSnapshots = snapshotStoreDao.listOnCache(snapshotId);

‎engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotObject.java‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,18 +19,18 @@
1919
package org.apache.cloudstack.storage.snapshot;
2020

2121
import java.util.Date;
22-
import java.util.List;
2322

2423
import javax.inject.Inject;
2524

25+
import org.apache.log4j.Logger;
26+
2627
import org.apache.cloudstack.engine.subsystem.api.storage.DataObjectInStore;
2728
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
2829
import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine;
2930
import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory;
3031
import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
3132
import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotStrategy;
3233
import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotStrategy.SnapshotOperation;
33-
import org.apache.cloudstack.engine.subsystem.api.storage.StrategyPriority;
3434
import org.apache.cloudstack.engine.subsystem.api.storage.StorageStrategyFactory;
3535
import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory;
3636
import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo;
@@ -40,7 +40,6 @@
4040
import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao;
4141
import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO;
4242
import org.apache.cloudstack.storage.to.SnapshotObjectTO;
43-
import org.apache.log4j.Logger;
4443

4544
import com.cloud.agent.api.Answer;
4645
import com.cloud.agent.api.to.DataObjectType;
@@ -79,6 +78,7 @@ public class SnapshotObject implements SnapshotInfo {
7978
SnapshotDataStoreDao snapshotStoreDao;
8079
@Inject
8180
StorageStrategyFactory storageStrategyFactory;
81+
private String installPath; // temporarily set installPath before passing to resource for entries with empty installPath for object store migration case
8282

8383
public SnapshotObject() {
8484

@@ -200,13 +200,20 @@ public long getVolumeId() {
200200

201201
@Override
202202
public String getPath() {
203+
if (installPath != null)
204+
return installPath;
205+
203206
DataObjectInStore objectInStore = objectInStoreMgr.findObject(this, getDataStore());
204207
if (objectInStore != null) {
205208
return objectInStore.getInstallPath();
206209
}
207210
return null;
208211
}
209212

213+
public void setPath(String installPath) {
214+
this.installPath = installPath;
215+
}
216+
210217
@Override
211218
public String getName() {
212219
return snapshot.getName();
@@ -360,12 +367,12 @@ public ObjectInDataStoreStateMachine.State getStatus() {
360367

361368
@Override
362369
public void addPayload(Object data) {
363-
this.payload = data;
370+
payload = data;
364371
}
365372

366373
@Override
367374
public Object getPayload() {
368-
return this.payload;
375+
return payload;
369376
}
370377

371378
@Override

‎engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java‎

Lines changed: 116 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -17,49 +17,61 @@
1717

1818
package org.apache.cloudstack.storage.snapshot;
1919

20-
import com.cloud.storage.DataStoreRole;
21-
import com.cloud.storage.Snapshot;
22-
import com.cloud.utils.exception.CloudRuntimeException;
23-
import com.cloud.utils.fsm.NoTransitionException;
20+
import java.util.List;
21+
import java.util.concurrent.ExecutionException;
22+
23+
import javax.inject.Inject;
24+
25+
import org.apache.log4j.Logger;
26+
import org.springframework.stereotype.Component;
27+
2428
import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult;
2529
import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult;
2630
import org.apache.cloudstack.engine.subsystem.api.storage.DataMotionService;
2731
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
2832
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
2933
import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine;
3034
import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.Event;
35+
import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore;
3136
import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreDriver;
3237
import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory;
3338
import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
3439
import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotResult;
3540
import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotService;
41+
import org.apache.cloudstack.engine.subsystem.api.storage.StorageCacheManager;
3642
import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo;
3743
import org.apache.cloudstack.framework.async.AsyncCallFuture;
3844
import org.apache.cloudstack.framework.async.AsyncCallbackDispatcher;
3945
import org.apache.cloudstack.framework.async.AsyncCompletionCallback;
4046
import org.apache.cloudstack.framework.async.AsyncRpcContext;
4147
import org.apache.cloudstack.storage.command.CommandResult;
4248
import org.apache.cloudstack.storage.command.CopyCmdAnswer;
43-
import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore;
4449
import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao;
4550
import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO;
46-
import org.apache.log4j.Logger;
47-
import org.springframework.stereotype.Component;
4851

49-
import javax.inject.Inject;
50-
import java.util.concurrent.ExecutionException;
52+
import com.cloud.storage.DataStoreRole;
53+
import com.cloud.storage.Snapshot;
54+
import com.cloud.storage.SnapshotVO;
55+
import com.cloud.storage.dao.SnapshotDao;
56+
import com.cloud.storage.template.TemplateConstants;
57+
import com.cloud.utils.exception.CloudRuntimeException;
58+
import com.cloud.utils.fsm.NoTransitionException;
5159

5260
@Component
5361
public class SnapshotServiceImpl implements SnapshotService {
5462
private static final Logger s_logger = Logger.getLogger(SnapshotServiceImpl.class);
5563
@Inject
64+
protected SnapshotDao _snapshotDao;
65+
@Inject
5666
protected SnapshotDataStoreDao _snapshotStoreDao;
5767
@Inject
58-
SnapshotDataFactory snapshotfactory;
68+
SnapshotDataFactory _snapshotFactory;
5969
@Inject
6070
DataStoreManager dataStoreMgr;
6171
@Inject
6272
DataMotionService motionSrv;
73+
@Inject
74+
StorageCacheManager _cacheMgr;
6375

6476
static private class CreateSnapshotContext<T> extends AsyncRpcContext<T> {
6577
final SnapshotInfo snapshot;
@@ -249,7 +261,7 @@ public SnapshotInfo backupSnapshot(SnapshotInfo snapshot) {
249261
try {
250262
snapObj.processEvent(Snapshot.Event.BackupToSecondary);
251263

252-
DataStore imageStore = this.findSnapshotImageStore(snapshot);
264+
DataStore imageStore = findSnapshotImageStore(snapshot);
253265
if (imageStore == null) {
254266
throw new CloudRuntimeException("can not find an image stores");
255267
}
@@ -262,7 +274,7 @@ public SnapshotInfo backupSnapshot(SnapshotInfo snapshot) {
262274
AsyncCallbackDispatcher<SnapshotServiceImpl, CopyCommandResult> caller = AsyncCallbackDispatcher
263275
.create(this);
264276
caller.setCallback(caller.getTarget().copySnapshotAsyncCallback(null, null)).setContext(context);
265-
this.motionSrv.copyAsync(snapshot, snapshotOnImageStore, caller);
277+
motionSrv.copyAsync(snapshot, snapshotOnImageStore, caller);
266278
} catch (Exception e) {
267279
s_logger.debug("Failed to copy snapshot", e);
268280
result.setResult("Failed to copy snapshot:" + e.toString());
@@ -314,7 +326,7 @@ protected Void copySnapshotAsyncCallback(AsyncCallbackDispatcher<SnapshotService
314326
CopyCmdAnswer answer = (CopyCmdAnswer) result.getAnswer();
315327
destSnapshot.processEvent(Event.OperationSuccessed, result.getAnswer());
316328
srcSnapshot.processEvent(Snapshot.Event.OperationSucceeded);
317-
snapResult = new SnapshotResult(this.snapshotfactory.getSnapshot(destSnapshot.getId(),
329+
snapResult = new SnapshotResult(_snapshotFactory.getSnapshot(destSnapshot.getId(),
318330
destSnapshot.getDataStore()), answer);
319331
future.complete(snapResult);
320332
} catch (Exception e) {
@@ -403,7 +415,7 @@ public boolean deleteSnapshot(SnapshotInfo snapInfo) {
403415

404416
@Override
405417
public boolean revertSnapshot(Long snapshotId) {
406-
SnapshotInfo snapshot = snapshotfactory.getSnapshot(snapshotId, DataStoreRole.Primary);
418+
SnapshotInfo snapshot = _snapshotFactory.getSnapshot(snapshotId, DataStoreRole.Primary);
407419
PrimaryDataStore store = (PrimaryDataStore)snapshot.getDataStore();
408420

409421
AsyncCallFuture<SnapshotResult> future = new AsyncCallFuture<SnapshotResult>();
@@ -429,4 +441,94 @@ public boolean revertSnapshot(Long snapshotId) {
429441
return false;
430442
}
431443

444+
445+
// This routine is used to push snapshots currently on cache store, but not in region store to region store.
446+
// used in migrating existing NFS secondary storage to S3. We chose to push all volume related snapshots to handle delta snapshots smoothly.
447+
@Override
448+
public void syncVolumeSnapshotsToRegionStore(long volumeId, DataStore store) {
449+
if (dataStoreMgr.isRegionStore(store)) {
450+
// list all backed up snapshots for the given volume
451+
List<SnapshotVO> snapshots = _snapshotDao.listByStatus(volumeId, Snapshot.State.BackedUp);
452+
if (snapshots != null ){
453+
for (SnapshotVO snapshot : snapshots){
454+
syncSnapshotToRegionStore(snapshot.getId(), store);
455+
}
456+
}
457+
}
458+
}
459+
460+
// push one individual snapshots currently on cache store to region store if it is not there already
461+
private void syncSnapshotToRegionStore(long snapshotId, DataStore store){
462+
if (s_logger.isDebugEnabled()) {
463+
s_logger.debug("sync snapshot " + snapshotId + " from cache to object store...");
464+
}
465+
// if snapshot is already on region wide object store, check if it is really downloaded there (by checking install_path). Sync snapshot to region
466+
// wide store if it is not there physically.
467+
SnapshotInfo snapOnStore = _snapshotFactory.getSnapshot(snapshotId, store);
468+
if (snapOnStore == null) {
469+
throw new CloudRuntimeException("Cannot find an entry in snapshot_store_ref for snapshot " + snapshotId + " on region store: " + store.getName());
470+
}
471+
if (snapOnStore.getPath() == null || snapOnStore.getPath().length() == 0) {
472+
// snapshot is not on region store yet, sync to region store
473+
SnapshotInfo srcSnapshot = _snapshotFactory.getReadySnapshotOnCache(snapshotId);
474+
if (srcSnapshot == null) {
475+
throw new CloudRuntimeException("Cannot find snapshot " + snapshotId + " on cache store");
476+
}
477+
AsyncCallFuture<SnapshotResult> future = syncToRegionStoreAsync(srcSnapshot, store);
478+
try {
479+
SnapshotResult result = future.get();
480+
if (result.isFailed()) {
481+
throw new CloudRuntimeException("sync snapshot from cache to region wide store failed for image store " + store.getName() + ":"
482+
+ result.getResult());
483+
}
484+
_cacheMgr.releaseCacheObject(srcSnapshot); // reduce reference count for template on cache, so it can recycled by schedule
485+
} catch (Exception ex) {
486+
throw new CloudRuntimeException("sync snapshot from cache to region wide store failed for image store " + store.getName());
487+
}
488+
}
489+
490+
}
491+
492+
493+
private AsyncCallFuture<SnapshotResult> syncToRegionStoreAsync(SnapshotInfo snapshot, DataStore store) {
494+
AsyncCallFuture<SnapshotResult> future = new AsyncCallFuture<SnapshotResult>();
495+
// no need to create entry on snapshot_store_ref here, since entries are already created when updateCloudToUseObjectStore is invoked.
496+
// But we need to set default install path so that sync can be done in the right s3 path
497+
SnapshotInfo snapshotOnStore = _snapshotFactory.getSnapshot(snapshot, store);
498+
String installPath = TemplateConstants.DEFAULT_SNAPSHOT_ROOT_DIR + "/"
499+
+ snapshot.getAccountId() + "/" + snapshot.getVolumeId();
500+
((SnapshotObject)snapshotOnStore).setPath(installPath);
501+
CopySnapshotContext<CommandResult> context = new CopySnapshotContext<CommandResult>(null, snapshot,
502+
snapshotOnStore, future);
503+
AsyncCallbackDispatcher<SnapshotServiceImpl, CopyCommandResult> caller = AsyncCallbackDispatcher
504+
.create(this);
505+
caller.setCallback(caller.getTarget().syncSnapshotCallBack(null, null)).setContext(context);
506+
motionSrv.copyAsync(snapshot, snapshotOnStore, caller);
507+
return future;
508+
}
509+
510+
protected Void syncSnapshotCallBack(AsyncCallbackDispatcher<SnapshotServiceImpl, CopyCommandResult> callback,
511+
CopySnapshotContext<CommandResult> context) {
512+
CopyCommandResult result = callback.getResult();
513+
SnapshotInfo destSnapshot = context.destSnapshot;
514+
SnapshotResult res = new SnapshotResult(destSnapshot, null);
515+
516+
AsyncCallFuture<SnapshotResult> future = context.future;
517+
try {
518+
if (result.isFailed()) {
519+
res.setResult(result.getResult());
520+
// no change to existing snapshot_store_ref, will try to re-sync later if other call triggers this sync operation
521+
} else {
522+
// this will update install path properly, next time it will not sync anymore.
523+
destSnapshot.processEvent(Event.OperationSuccessed, result.getAnswer());
524+
}
525+
future.complete(res);
526+
} catch (Exception e) {
527+
s_logger.debug("Failed to process sync snapshot callback", e);
528+
res.setResult(e.toString());
529+
future.complete(res);
530+
}
531+
532+
return null;
533+
}
432534
}

0 commit comments

Comments
 (0)