Skip to content

Commit b70c1a5

Browse files
author
Edison Su
committed
Backs NFS-based secondary storage with an S3-compatible object store. Periodically, a reaper thread synchronizes templates and ISOs stored on a NFS secondary storage mount with a configured S3 object store. It also pushes snapshots to the object store when they are created and downloads them in other zones on-demand. In addition to permitting the use of commodity or IaaS storage solutions for static assets, it provides a means of automatically synchronizing template and ISO assets across multiple zones.
1 parent 0ba355f commit b70c1a5

66 files changed

Lines changed: 4707 additions & 53 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ awsapi/modules/*
6767
.settings/
6868
db.properties.override
6969
awsapi/overlays/
70+
tools/marvin/marvin/cloudstackAPI/*
7071
*.egg-info/
7172
docs/tmp
7273
docs/publish

‎api/src/com/cloud/agent/api/BackupSnapshotCommand.java‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
package com.cloud.agent.api;
1818

1919
import com.cloud.agent.api.LogLevel.Log4jLevel;
20+
import com.cloud.agent.api.to.S3TO;
2021
import com.cloud.agent.api.to.StorageFilerTO;
2122
import com.cloud.agent.api.to.SwiftTO;
2223
import com.cloud.storage.StoragePool;
@@ -32,6 +33,7 @@ public class BackupSnapshotCommand extends SnapshotCommand {
3233
private Long snapshotId;
3334
@LogLevel(Log4jLevel.Off)
3435
private SwiftTO swift;
36+
private S3TO s3;
3537
StorageFilerTO pool;
3638

3739
protected BackupSnapshotCommand() {
@@ -88,7 +90,7 @@ public boolean isVolumeInactive() {
8890
}
8991

9092
public String getVmName() {
91-
return vmName;
93+
return vmName;
9294
}
9395

9496
public SwiftTO getSwift() {
@@ -99,6 +101,14 @@ public void setSwift(SwiftTO swift) {
99101
this.swift = swift;
100102
}
101103

104+
public S3TO getS3() {
105+
return s3;
106+
}
107+
108+
public void setS3(S3TO s3) {
109+
this.s3 = s3;
110+
}
111+
102112
public Long getSnapshotId() {
103113
return snapshotId;
104114
}

‎api/src/com/cloud/agent/api/DeleteSnapshotBackupCommand.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
package com.cloud.agent.api;
1818

1919
import com.cloud.agent.api.LogLevel.Log4jLevel;
20+
import com.cloud.agent.api.to.S3TO;
2021
import com.cloud.agent.api.to.SwiftTO;
2122

2223
/**
@@ -26,6 +27,7 @@
2627
public class DeleteSnapshotBackupCommand extends SnapshotCommand {
2728
@LogLevel(Log4jLevel.Off)
2829
private SwiftTO swift;
30+
private S3TO s3;
2931
private Boolean all;
3032

3133
public SwiftTO getSwift() {
@@ -44,6 +46,10 @@ public void setSwift(SwiftTO swift) {
4446
this.swift = swift;
4547
}
4648

49+
public S3TO getS3() {
50+
return s3;
51+
}
52+
4753
protected DeleteSnapshotBackupCommand() {
4854
}
4955

@@ -73,6 +79,7 @@ protected DeleteSnapshotBackupCommand() {
7379
* @param childUUID The child VHD file of the backup whose parent is reset to its grandparent.
7480
*/
7581
public DeleteSnapshotBackupCommand(SwiftTO swift,
82+
S3TO s3,
7683
String secondaryStoragePoolURL,
7784
Long dcId,
7885
Long accountId,
@@ -81,6 +88,7 @@ public DeleteSnapshotBackupCommand(SwiftTO swift,
8188
{
8289
super(null, secondaryStoragePoolURL, backupUUID, null, dcId, accountId, volumeId);
8390
setSwift(swift);
91+
this.s3 = s3;
8492
setAll(all);
8593
}
8694
}

‎api/src/com/cloud/agent/api/DeleteTemplateFromS3Command.java‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package com.cloud.agent.api;
20+
21+
import com.cloud.agent.api.to.S3TO;
22+
23+
public class DeleteTemplateFromS3Command extends Command {
24+
25+
private S3TO s3;
26+
private Long templateId;
27+
private Long accountId;
28+
29+
protected DeleteTemplateFromS3Command() {
30+
super();
31+
}
32+
33+
public DeleteTemplateFromS3Command(final S3TO s3, final Long accountId,
34+
final Long templateId) {
35+
36+
super();
37+
38+
this.s3 = s3;
39+
this.accountId = accountId;
40+
this.templateId = templateId;
41+
42+
}
43+
44+
@Override
45+
public int hashCode() {
46+
final int prime = 31;
47+
int result = 1;
48+
result = prime * result
49+
+ ((accountId == null) ? 0 : accountId.hashCode());
50+
result = prime * result + ((s3 == null) ? 0 : s3.hashCode());
51+
result = prime * result
52+
+ ((templateId == null) ? 0 : templateId.hashCode());
53+
return result;
54+
}
55+
56+
@Override
57+
public boolean equals(Object thatObject) {
58+
59+
if (this == thatObject) {
60+
return true;
61+
}
62+
63+
if (thatObject == null) {
64+
return false;
65+
}
66+
67+
if (getClass() != thatObject.getClass()) {
68+
return false;
69+
}
70+
71+
final DeleteTemplateFromS3Command thatCommand = (DeleteTemplateFromS3Command) thatObject;
72+
73+
if (!(accountId == thatCommand.accountId)
74+
|| (this.accountId != null && this.accountId
75+
.equals(thatCommand.accountId))) {
76+
return false;
77+
}
78+
79+
if (!(templateId == thatCommand.templateId)
80+
|| (this.templateId != null && this.templateId
81+
.equals(thatCommand.templateId))) {
82+
return false;
83+
}
84+
85+
return true;
86+
87+
}
88+
89+
public S3TO getS3() {
90+
return s3;
91+
}
92+
93+
public Long getTemplateId() {
94+
return templateId;
95+
}
96+
97+
public Long getAccountId() {
98+
return accountId;
99+
}
100+
101+
@Override
102+
public boolean executeInSequence() {
103+
return true;
104+
}
105+
106+
}

‎api/src/com/cloud/agent/api/DownloadSnapshotFromS3Command.java‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package com.cloud.agent.api;
20+
21+
import com.cloud.agent.api.to.S3TO;
22+
23+
public class DownloadSnapshotFromS3Command extends SnapshotCommand {
24+
25+
private S3TO s3;
26+
private String parent;
27+
28+
protected DownloadSnapshotFromS3Command() {
29+
super();
30+
}
31+
32+
public DownloadSnapshotFromS3Command(S3TO s3, String parent,
33+
String secondaryStorageUrl, Long dcId, Long accountId,
34+
Long volumeId, String backupUuid, int wait) {
35+
36+
super("", secondaryStorageUrl, backupUuid, "", dcId, accountId,
37+
volumeId);
38+
39+
this.s3 = s3;
40+
this.parent = parent;
41+
setWait(wait);
42+
43+
}
44+
45+
public S3TO getS3() {
46+
return s3;
47+
}
48+
49+
public void setS3(S3TO s3) {
50+
this.s3 = s3;
51+
}
52+
53+
public String getParent() {
54+
return parent;
55+
}
56+
57+
public void setParent(String parent) {
58+
this.parent = parent;
59+
}
60+
61+
}

‎api/src/com/cloud/agent/api/DownloadTemplateFromS3ToSecondaryStorageCommand.java‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package com.cloud.agent.api;
20+
21+
import com.cloud.agent.api.to.S3TO;
22+
23+
public final class DownloadTemplateFromS3ToSecondaryStorageCommand extends Command {
24+
25+
private final S3TO s3;
26+
private final Long accountId;
27+
private final Long templateId;
28+
private final String storagePath;
29+
30+
public DownloadTemplateFromS3ToSecondaryStorageCommand(final S3TO s3,
31+
final Long accountId, final Long templateId,
32+
final String storagePath, final int wait) {
33+
34+
super();
35+
36+
this.s3 = s3;
37+
this.accountId = accountId;
38+
this.templateId = templateId;
39+
this.storagePath = storagePath;
40+
41+
setWait(wait);
42+
43+
}
44+
45+
public S3TO getS3() {
46+
return this.s3;
47+
}
48+
49+
public Long getAccountId() {
50+
return this.accountId;
51+
}
52+
53+
public Long getTemplateId() {
54+
return this.templateId;
55+
}
56+
57+
public String getStoragePath() {
58+
return this.storagePath;
59+
}
60+
61+
@Override
62+
public boolean executeInSequence() {
63+
return true;
64+
}
65+
66+
}

0 commit comments

Comments
 (0)