Skip to content

Commit b6d420b

Browse files
khos2owyadvr
khos2ow
authored and
yadvr
committed
CLOUDSTACK-9677: Adding storage policy support for swift as secondary storage (apache#2412)
Original-Author: @pdube on PR Fixes apache#1830.
1 parent 9a13227 commit b6d420b

11 files changed

Lines changed: 499 additions & 150 deletions

File tree

‎api/src/com/cloud/agent/api/to/SwiftTO.java‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,17 +26,19 @@ public class SwiftTO implements DataStoreTO, SwiftUtil.SwiftClientCfg {
2626

2727
String userName;
2828
String key;
29+
String storagePolicy;
2930
private static final String pathSeparator = "/";
3031

3132
public SwiftTO() {
3233
}
3334

34-
public SwiftTO(Long id, String url, String account, String userName, String key) {
35+
public SwiftTO(Long id, String url, String account, String userName, String key, String storagePolicy) {
3536
this.id = id;
3637
this.url = url;
3738
this.account = account;
3839
this.userName = userName;
3940
this.key = key;
41+
this.storagePolicy = storagePolicy;
4042
}
4143

4244
public Long getId() {
@@ -63,6 +65,11 @@ public String getKey() {
6365
return key;
6466
}
6567

68+
@Override
69+
public String getStoragePolicy() {
70+
return this.storagePolicy;
71+
}
72+
6673
@Override
6774
public DataStoreRole getRole() {
6875
return DataStoreRole.Image;

‎api/src/org/apache/cloudstack/api/ApiConstants.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -287,6 +287,7 @@ public class ApiConstants {
287287
public static final String STATE = "state";
288288
public static final String STATUS = "status";
289289
public static final String STORAGE_TYPE = "storagetype";
290+
public static final String STORAGE_POLICY = "storagepolicy";
290291
public static final String STORAGE_MOTION_ENABLED = "storagemotionenabled";
291292
public static final String STORAGE_CAPABILITIES = "storagecapabilities";
292293
public static final String SYSTEM_VM_TYPE = "systemvmtype";

‎plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java‎

Lines changed: 78 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -18,23 +18,24 @@
1818
*/
1919
package com.cloud.hypervisor.xenserver.resource;
2020

21-
import com.cloud.agent.api.Answer;
22-
import com.cloud.agent.api.to.DataObjectType;
23-
import com.cloud.agent.api.to.DataStoreTO;
24-
import com.cloud.agent.api.to.DataTO;
25-
import com.cloud.agent.api.to.DiskTO;
26-
import com.cloud.agent.api.to.NfsTO;
27-
import com.cloud.agent.api.to.S3TO;
28-
import com.cloud.agent.api.to.SwiftTO;
29-
import com.cloud.exception.InternalErrorException;
30-
import com.cloud.hypervisor.Hypervisor.HypervisorType;
31-
import com.cloud.hypervisor.xenserver.resource.CitrixResourceBase.SRType;
32-
import com.cloud.storage.DataStoreRole;
33-
import com.cloud.storage.Storage;
34-
import com.cloud.storage.Storage.ImageFormat;
35-
import com.cloud.storage.resource.StorageProcessor;
36-
import com.cloud.utils.exception.CloudRuntimeException;
37-
import com.cloud.utils.storage.S3.ClientOptions;
21+
import static com.cloud.utils.ReflectUtil.flattenProperties;
22+
import static com.google.common.collect.Lists.newArrayList;
23+
24+
import java.io.File;
25+
import java.net.URI;
26+
import java.util.ArrayList;
27+
import java.util.Arrays;
28+
import java.util.HashMap;
29+
import java.util.List;
30+
import java.util.Map;
31+
import java.util.Set;
32+
import java.util.UUID;
33+
34+
import org.apache.commons.lang3.BooleanUtils;
35+
import org.apache.log4j.Logger;
36+
import org.apache.xmlrpc.XmlRpcException;
37+
38+
import com.google.common.annotations.VisibleForTesting;
3839
import com.xensource.xenapi.Connection;
3940
import com.xensource.xenapi.SR;
4041
import com.xensource.xenapi.Types;
@@ -44,6 +45,7 @@
4445
import com.xensource.xenapi.VBD;
4546
import com.xensource.xenapi.VDI;
4647
import com.xensource.xenapi.VM;
48+
4749
import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand;
4850
import org.apache.cloudstack.storage.command.AttachAnswer;
4951
import org.apache.cloudstack.storage.command.AttachCommand;
@@ -65,20 +67,24 @@
6567
import org.apache.cloudstack.storage.to.SnapshotObjectTO;
6668
import org.apache.cloudstack.storage.to.TemplateObjectTO;
6769
import org.apache.cloudstack.storage.to.VolumeObjectTO;
68-
import org.apache.log4j.Logger;
69-
import org.apache.xmlrpc.XmlRpcException;
70-
71-
import java.io.File;
72-
import java.net.URI;
73-
import java.util.Arrays;
74-
import java.util.HashMap;
75-
import java.util.List;
76-
import java.util.Map;
77-
import java.util.Set;
78-
import java.util.UUID;
7970

80-
import static com.cloud.utils.ReflectUtil.flattenProperties;
81-
import static com.google.common.collect.Lists.newArrayList;
71+
import com.cloud.agent.api.Answer;
72+
import com.cloud.agent.api.to.DataObjectType;
73+
import com.cloud.agent.api.to.DataStoreTO;
74+
import com.cloud.agent.api.to.DataTO;
75+
import com.cloud.agent.api.to.DiskTO;
76+
import com.cloud.agent.api.to.NfsTO;
77+
import com.cloud.agent.api.to.S3TO;
78+
import com.cloud.agent.api.to.SwiftTO;
79+
import com.cloud.exception.InternalErrorException;
80+
import com.cloud.hypervisor.Hypervisor.HypervisorType;
81+
import com.cloud.hypervisor.xenserver.resource.CitrixResourceBase.SRType;
82+
import com.cloud.storage.DataStoreRole;
83+
import com.cloud.storage.Storage;
84+
import com.cloud.storage.Storage.ImageFormat;
85+
import com.cloud.storage.resource.StorageProcessor;
86+
import com.cloud.utils.exception.CloudRuntimeException;
87+
import com.cloud.utils.storage.S3.ClientOptions;
8288

8389
public class XenServerStorageProcessor implements StorageProcessor {
8490
private static final Logger s_logger = Logger.getLogger(XenServerStorageProcessor.class);
@@ -914,20 +920,55 @@ public Answer copyVolumeFromPrimaryToSecondary(final CopyCommand cmd) {
914920

915921
private boolean swiftUpload(final Connection conn, final SwiftTO swift, final String container, final String ldir, final String lfilename, final Boolean isISCSI,
916922
final int wait) {
917-
String result = null;
923+
924+
List<String> params = getSwiftParams(swift, container, ldir, lfilename, isISCSI);
925+
918926
try {
919-
result =
920-
hypervisorResource.callHostPluginAsync(conn, "swiftxenserver", "swift", wait, "op", "upload", "url", swift.getUrl(), "account", swift.getAccount(), "username",
921-
swift.getUserName(), "key", swift.getKey(), "container", container, "ldir", ldir, "lfilename", lfilename, "isISCSI", isISCSI.toString());
922-
if (result != null && result.equals("true")) {
923-
return true;
924-
}
927+
String result = hypervisorResource.callHostPluginAsync(conn, "swiftxenserver", "swift", wait, params.toArray(new String[params.size()]));
928+
return BooleanUtils.toBoolean(result);
925929
} catch (final Exception e) {
926930
s_logger.warn("swift upload failed due to " + e.toString(), e);
927931
}
928932
return false;
929933
}
930934

935+
@VisibleForTesting
936+
List<String> getSwiftParams(SwiftTO swift, String container, String ldir, String lfilename, Boolean isISCSI) {
937+
// ORDER IS IMPORTANT
938+
List<String> params = new ArrayList<>();
939+
940+
//operation
941+
params.add("op");
942+
params.add("upload");
943+
944+
//auth
945+
params.add("url");
946+
params.add(swift.getUrl());
947+
params.add("account");
948+
params.add(swift.getAccount());
949+
params.add("username");
950+
params.add(swift.getUserName());
951+
params.add("key");
952+
params.add(swift.getKey());
953+
954+
// object info
955+
params.add("container");
956+
params.add(container);
957+
params.add("ldir");
958+
params.add(ldir);
959+
params.add("lfilename");
960+
params.add(lfilename);
961+
params.add("isISCSI");
962+
params.add(isISCSI.toString());
963+
964+
if (swift.getStoragePolicy() != null) {
965+
params.add("storagepolicy");
966+
params.add(swift.getStoragePolicy());
967+
}
968+
969+
return params;
970+
}
971+
931972
protected String deleteSnapshotBackup(final Connection conn, final String localMountPoint, final String path, final String secondaryStorageMountPath, final String backupUUID) {
932973

933974
// If anybody modifies the formatting below again, I'll skin them

‎plugins/hypervisors/xenserver/test/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessorTest.java‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package com.cloud.hypervisor.xenserver.resource;
20+
21+
import static org.mockito.Mockito.when;
22+
23+
import java.util.List;
24+
25+
import org.junit.Assert;
26+
import org.junit.Test;
27+
import org.mockito.Mockito;
28+
29+
import com.cloud.agent.api.to.SwiftTO;
30+
31+
public class XenServerStorageProcessorTest {
32+
@Test
33+
public void testOrderOfSwiftUplodScriptParamsWithoutStoragePolicy() {
34+
CitrixResourceBase resource = Mockito.mock(CitrixResourceBase.class);
35+
XenServerStorageProcessor mock = new XenServerStorageProcessor(resource);
36+
37+
SwiftTO swift = Mockito.mock(SwiftTO.class);
38+
when(swift.getStoragePolicy()).thenReturn(null);
39+
40+
String container = "sample-container-name";
41+
String ldir = "sample-ldir";
42+
String lfilename = "sample-lfilename";
43+
Boolean isISCSI = true;
44+
45+
List<String> params = mock.getSwiftParams(swift, container, ldir, lfilename, isISCSI);
46+
47+
// make sure the params not null and has correct number of items in it
48+
Assert.assertNotNull("params is null", params);
49+
Assert.assertTrue("Expected param list size is 18 but it was" + params.size(), params.size() == 18);
50+
51+
// check the order of params
52+
Assert.assertEquals("unexpected param.", "op", params.get(0));
53+
Assert.assertEquals("unexpected param.", "upload", params.get(1));
54+
Assert.assertEquals("unexpected param.", "url", params.get(2));
55+
Assert.assertEquals("unexpected param.", swift.getUrl(), params.get(3));
56+
Assert.assertEquals("unexpected param.", "account", params.get(4));
57+
Assert.assertEquals("unexpected param.", swift.getAccount(), params.get(5));
58+
Assert.assertEquals("unexpected param.", "username", params.get(6));
59+
Assert.assertEquals("unexpected param.", swift.getUserName(), params.get(7));
60+
Assert.assertEquals("unexpected param.", "key", params.get(8));
61+
Assert.assertEquals("unexpected param.", swift.getKey(), params.get(9));
62+
Assert.assertEquals("unexpected param.", "container", params.get(10));
63+
Assert.assertEquals("unexpected param.", container, params.get(11));
64+
Assert.assertEquals("unexpected param.", "ldir", params.get(12));
65+
Assert.assertEquals("unexpected param.", ldir, params.get(13));
66+
Assert.assertEquals("unexpected param.", "lfilename", params.get(14));
67+
Assert.assertEquals("unexpected param.", lfilename, params.get(15));
68+
Assert.assertEquals("unexpected param.", "isISCSI", params.get(16));
69+
Assert.assertEquals("unexpected param.", isISCSI.toString(), params.get(17));
70+
}
71+
72+
@Test
73+
public void testOrderOfSwiftUplodScriptParamsWithStoragePolicy() {
74+
CitrixResourceBase resource = Mockito.mock(CitrixResourceBase.class);
75+
XenServerStorageProcessor mock = new XenServerStorageProcessor(resource);
76+
77+
SwiftTO swift = Mockito.mock(SwiftTO.class);
78+
when(swift.getStoragePolicy()).thenReturn("sample-storagepolicy");
79+
80+
String container = "sample-container-name";
81+
String ldir = "sample-ldir";
82+
String lfilename = "sample-lfilename";
83+
Boolean isISCSI = true;
84+
85+
List<String> params = mock.getSwiftParams(swift, container, ldir, lfilename, isISCSI);
86+
87+
// make sure the params not null and has correct number of items in it
88+
Assert.assertNotNull("params is null", params);
89+
Assert.assertTrue("Expected param list size is 20 but it was" + params.size(), params.size() == 20);
90+
91+
// check the order of params
92+
Assert.assertEquals("unexpected param.", "op", params.get(0));
93+
Assert.assertEquals("unexpected param.", "upload", params.get(1));
94+
Assert.assertEquals("unexpected param.", "url", params.get(2));
95+
Assert.assertEquals("unexpected param.", swift.getUrl(), params.get(3));
96+
Assert.assertEquals("unexpected param.", "account", params.get(4));
97+
Assert.assertEquals("unexpected param.", swift.getAccount(), params.get(5));
98+
Assert.assertEquals("unexpected param.", "username", params.get(6));
99+
Assert.assertEquals("unexpected param.", swift.getUserName(), params.get(7));
100+
Assert.assertEquals("unexpected param.", "key", params.get(8));
101+
Assert.assertEquals("unexpected param.", swift.getKey(), params.get(9));
102+
Assert.assertEquals("unexpected param.", "container", params.get(10));
103+
Assert.assertEquals("unexpected param.", container, params.get(11));
104+
Assert.assertEquals("unexpected param.", "ldir", params.get(12));
105+
Assert.assertEquals("unexpected param.", ldir, params.get(13));
106+
Assert.assertEquals("unexpected param.", "lfilename", params.get(14));
107+
Assert.assertEquals("unexpected param.", lfilename, params.get(15));
108+
Assert.assertEquals("unexpected param.", "isISCSI", params.get(16));
109+
Assert.assertEquals("unexpected param.", isISCSI.toString(), params.get(17));
110+
Assert.assertEquals("unexpected param.", "storagepolicy", params.get(18));
111+
Assert.assertEquals("unexpected param.", "sample-storagepolicy", params.get(19));
112+
}
113+
}

‎plugins/storage/image/swift/src/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ public class SwiftImageStoreDriverImpl extends BaseImageStoreDriverImpl {
6868
public DataStoreTO getStoreTO(DataStore store) {
6969
ImageStoreImpl imgStore = (ImageStoreImpl)store;
7070
Map<String, String> details = _imageStoreDetailsDao.getDetails(imgStore.getId());
71-
return new SwiftTO(imgStore.getId(), imgStore.getUri(), details.get(ApiConstants.ACCOUNT), details.get(ApiConstants.USERNAME), details.get(ApiConstants.KEY));
71+
return new SwiftTO(imgStore.getId(), imgStore.getUri(), details.get(ApiConstants.ACCOUNT), details.get(ApiConstants.USERNAME), details.get(ApiConstants.KEY), details.get(ApiConstants.STORAGE_POLICY));
7272
}
7373

7474
@Override

‎scripts/storage/secondary/swift‎

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1473,8 +1473,9 @@ post [options] [container] [object]
14731473
Updates meta information for the account, container, or object depending on
14741474
the args given. If the container is not found, it will be created
14751475
automatically; but this is not true for accounts and objects. Containers
1476-
also allow the -r (or --read-acl) and -w (or --write-acl) options. The -m
1477-
or --meta option is allowed on all and used to define the user meta data
1476+
also allow the -r (or --read-acl) and -w (or --write-acl) options.
1477+
The --storage-policy will set a storage policy to the container if the container does not exist.
1478+
The -m or --meta option is allowed on all and used to define the user meta data
14781479
items to set in the form Name:Value. This option can be repeated. Example:
14791480
post -m Color:Blue -m Size:Large'''.strip('\n')
14801481

@@ -1493,6 +1494,8 @@ def st_post(options, args, print_queue, error_queue):
14931494
parser.add_option('-m', '--meta', action='append', dest='meta', default=[],
14941495
help='Sets a meta data item with the syntax name:value. This option '
14951496
'may be repeated. Example: -m Color:Blue -m Size:Large')
1497+
parser.add_option('', '--storage-policy', action='store', dest='storage_policy',
1498+
help='Sets a storage policy to the container if the container does not exist')
14961499
(options, args) = parse_args(parser, args)
14971500
args = args[1:]
14981501
if (options.read_acl or options.write_acl or options.sync_to or
@@ -1529,6 +1532,8 @@ def st_post(options, args, print_queue, error_queue):
15291532
headers['X-Container-Sync-To'] = options.sync_to
15301533
if options.sync_key is not None:
15311534
headers['X-Container-Sync-Key'] = options.sync_key
1535+
if options.storage_policy is not None:
1536+
headers['X-Storage-Policy'] = options.storage_policy
15321537
try:
15331538
conn.post_container(args[0], headers=headers)
15341539
except ClientException, err:
@@ -1558,7 +1563,8 @@ upload [options] container file_or_directory [file_or_directory] [...]
15581563
Uploads to the given container the files and directories specified by the
15591564
remaining args. -c or --changed is an option that will only upload files
15601565
that have changed since the last upload. -S <size> or --segment-size <size>
1561-
and --leave-segments are options as well (see --help for more).
1566+
and --leave-segments are options as well (see --help for more). --storage-policy
1567+
Sets a storage policy to the container if the container does not exist.
15621568
'''.strip('\n')
15631569

15641570

@@ -1576,6 +1582,8 @@ def st_upload(options, args, print_queue, error_queue):
15761582
dest='leave_segments', default=False, help='Indicates that you want '
15771583
'the older segments of manifest objects left alone (in the case of '
15781584
'overwrites)')
1585+
parser.add_option('', '--storage-policy', action='store', dest='storage_policy',
1586+
help='Sets a storage policy to the container if the container does not exist')
15791587
(options, args) = parse_args(parser, args)
15801588
args = args[1:]
15811589
if len(args) < 2:
@@ -1749,9 +1757,12 @@ def st_upload(options, args, print_queue, error_queue):
17491757
# permissions, so we'll ignore any error. If there's really a problem,
17501758
# it'll surface on the first object PUT.
17511759
try:
1752-
conn.put_container(args[0])
1760+
container_headers = {}
1761+
if options.storage_policy is not None:
1762+
container_headers['X-Storage-Policy'] = options.storage_policy
1763+
conn.put_container(args[0],headers=container_headers)
17531764
if options.segment_size is not None:
1754-
conn.put_container(args[0] + '_segments')
1765+
conn.put_container(args[0] + '_segments',headers=container_headers)
17551766
except Exception:
17561767
pass
17571768
try:

0 commit comments

Comments
 (0)