Skip to content

Commit fe83a85

Browse files
nixpanicwido
nixpanic
authored and
wido
committed
Add support for Primary Storage on Gluster using the libvirt backend
The support for Gluster as Primary Storage is mostly based on the implementation for NFS. Like NFS, libvirt can address a Gluster environment through the 'netfs' pool-type.
1 parent ca68395 commit fe83a85

8 files changed

Lines changed: 111 additions & 15 deletions

File tree

‎api/src/com/cloud/storage/Storage.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,8 @@ public static enum StoragePoolType {
9898
PreSetup(true), // for XenServer, Storage Pool is set up by customers.
9999
EXT(false), // XenServer local EXT SR
100100
OCFS2(true),
101-
SMB(true);
101+
SMB(true),
102+
Gluster(true);
102103

103104
boolean shared;
104105

‎plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1750,8 +1750,11 @@ private String getResizeScriptType(KVMStoragePool pool, KVMPhysicalDisk vol) {
17501750

17511751
if (pool.getType() == StoragePoolType.CLVM && volFormat == PhysicalDiskFormat.RAW) {
17521752
return "CLVM";
1753-
} else if ((poolType == StoragePoolType.NetworkFilesystem || poolType == StoragePoolType.SharedMountPoint || poolType == StoragePoolType.Filesystem) &&
1754-
volFormat == PhysicalDiskFormat.QCOW2) {
1753+
} else if ((poolType == StoragePoolType.NetworkFilesystem
1754+
|| poolType == StoragePoolType.SharedMountPoint
1755+
|| poolType == StoragePoolType.Filesystem
1756+
|| poolType == StoragePoolType.Gluster)
1757+
&& volFormat == PhysicalDiskFormat.QCOW2 ) {
17551758
return "QCOW2";
17561759
}
17571760
return null;
@@ -3702,6 +3705,12 @@ public int compare(DiskTO arg0, DiskTO arg1) {
37023705
*/
37033706
disk.defNetworkBasedDisk(physicalDisk.getPath().replace("rbd:", ""), pool.getSourceHost(), pool.getSourcePort(), pool.getAuthUserName(),
37043707
pool.getUuid(), devId, diskBusType, diskProtocol.RBD);
3708+
} else if (pool.getType() == StoragePoolType.Gluster) {
3709+
String mountpoint = pool.getLocalPath();
3710+
String path = physicalDisk.getPath();
3711+
String glusterVolume = pool.getSourceDir().replace("/", "");
3712+
disk.defNetworkBasedDisk(glusterVolume + path.replace(mountpoint, ""), pool.getSourceHost(), pool.getSourcePort(), null,
3713+
null, devId, diskBusType, diskProtocol.GLUSTER);
37053714
} else if (pool.getType() == StoragePoolType.CLVM || physicalDisk.getFormat() == PhysicalDiskFormat.RAW) {
37063715
disk.defBlockBasedDisk(physicalDisk.getPath(), devId, diskBusType);
37073716
} else {
@@ -3859,6 +3868,9 @@ protected synchronized String attachOrDetachDisk(Connect conn,
38593868
if (attachingPool.getType() == StoragePoolType.RBD) {
38603869
diskdef.defNetworkBasedDisk(attachingDisk.getPath(), attachingPool.getSourceHost(), attachingPool.getSourcePort(), attachingPool.getAuthUserName(),
38613870
attachingPool.getUuid(), devId, DiskDef.diskBus.VIRTIO, diskProtocol.RBD);
3871+
} else if (attachingPool.getType() == StoragePoolType.Gluster) {
3872+
diskdef.defNetworkBasedDisk(attachingDisk.getPath(), attachingPool.getSourceHost(), attachingPool.getSourcePort(), null,
3873+
null, devId, DiskDef.diskBus.VIRTIO, diskProtocol.GLUSTER);
38623874
} else if (attachingDisk.getFormat() == PhysicalDiskFormat.QCOW2) {
38633875
diskdef.defFileBasedDisk(attachingDisk.getPath(), devId, DiskDef.diskBus.VIRTIO, DiskDef.diskFmtType.QCOW2);
38643876
} else if (attachingDisk.getFormat() == PhysicalDiskFormat.RAW) {

‎plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolDef.java‎

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818

1919
public class LibvirtStoragePoolDef {
2020
public enum poolType {
21-
ISCSI("iscsi"), NETFS("netfs"), LOGICAL("logical"), DIR("dir"), RBD("rbd");
21+
ISCSI("iscsi"), NETFS("netfs"), LOGICAL("logical"), DIR("dir"), RBD("rbd"), GLUSTERFS("glusterfs");
2222
String _poolType;
2323

2424
poolType(String poolType) {
@@ -127,7 +127,15 @@ public authType getAuthType() {
127127
@Override
128128
public String toString() {
129129
StringBuilder storagePoolBuilder = new StringBuilder();
130-
storagePoolBuilder.append("<pool type='" + _poolType + "'>\n");
130+
if (_poolType == poolType.GLUSTERFS) {
131+
/* libvirt mounts a Gluster volume, similar to NFS */
132+
storagePoolBuilder.append("<pool type='netfs'>\n");
133+
} else {
134+
storagePoolBuilder.append("<pool type='");
135+
storagePoolBuilder.append(_poolType);
136+
storagePoolBuilder.append("'>\n");
137+
}
138+
131139
storagePoolBuilder.append("<name>" + _poolName + "</name>\n");
132140
if (_uuid != null)
133141
storagePoolBuilder.append("<uuid>" + _uuid + "</uuid>\n");
@@ -148,6 +156,23 @@ public String toString() {
148156
}
149157
storagePoolBuilder.append("</source>\n");
150158
}
159+
if (_poolType == poolType.GLUSTERFS) {
160+
storagePoolBuilder.append("<source>\n");
161+
storagePoolBuilder.append("<host name='");
162+
storagePoolBuilder.append(_sourceHost);
163+
if (_sourcePort != 0) {
164+
storagePoolBuilder.append("' port='");
165+
storagePoolBuilder.append(_sourcePort);
166+
}
167+
storagePoolBuilder.append("'/>\n");
168+
storagePoolBuilder.append("<dir path='");
169+
storagePoolBuilder.append(_sourceDir);
170+
storagePoolBuilder.append("'/>\n");
171+
storagePoolBuilder.append("<format type='");
172+
storagePoolBuilder.append(_poolType);
173+
storagePoolBuilder.append("'/>\n");
174+
storagePoolBuilder.append("</source>\n");
175+
}
151176
if (_poolType != poolType.RBD) {
152177
storagePoolBuilder.append("<target>\n");
153178
storagePoolBuilder.append("<path>" + _targetPath + "</path>\n");

‎plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolXMLParser.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ public LibvirtStoragePoolDef parseStoragePoolXML(String poolXML) {
5252

5353
Element source = (Element)rootElement.getElementsByTagName("source").item(0);
5454
String host = getAttrValue("host", "name", source);
55+
String format = getAttrValue("format", "type", source);
5556

5657
if (type.equalsIgnoreCase("rbd")) {
5758
int port = Integer.parseInt(getAttrValue("host", "port", source));
@@ -67,6 +68,23 @@ public LibvirtStoragePoolDef parseStoragePoolXML(String poolXML) {
6768
} else {
6869
return new LibvirtStoragePoolDef(LibvirtStoragePoolDef.poolType.valueOf(type.toUpperCase()), poolName, uuid, host, port, pool, "");
6970
}
71+
/* Gluster is a sub-type of LibvirtStoragePoolDef.poolType.NETFS, need to check format */
72+
} else if (format != null && format.equalsIgnoreCase("glusterfs")) {
73+
/* libvirt does not return the default port, but requires it for a disk-definition */
74+
int port = 24007;
75+
76+
String path = getAttrValue("dir", "path", source);
77+
78+
Element target = (Element) rootElement.getElementsByTagName(
79+
"target").item(0);
80+
String targetPath = getTagValue("path", target);
81+
82+
String portValue = getAttrValue("host", "port", source);
83+
if (portValue != "")
84+
port = Integer.parseInt(portValue);
85+
86+
return new LibvirtStoragePoolDef(LibvirtStoragePoolDef.poolType.valueOf(format.toUpperCase()),
87+
poolName, uuid, host, port, path, targetPath);
7088
} else {
7189
String path = getAttrValue("dir", "path", source);
7290

‎plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -378,7 +378,7 @@ public String toString() {
378378
}
379379

380380
public enum diskProtocol {
381-
RBD("rbd"), SHEEPDOG("sheepdog");
381+
RBD("rbd"), SHEEPDOG("sheepdog"), GLUSTER("gluster");
382382
String _diskProtocol;
383383

384384
diskProtocol(String protocol) {
@@ -664,7 +664,13 @@ public String toString() {
664664
diskBuilder.append(" protocol='" + _diskProtocol + "'");
665665
diskBuilder.append(" name='" + _sourcePath + "'");
666666
diskBuilder.append(">\n");
667-
diskBuilder.append("<host name='" + _sourceHost + "' port='" + _sourcePort + "'/>\n");
667+
diskBuilder.append("<host name='");
668+
diskBuilder.append(_sourceHost);
669+
if (_sourcePort != 0) {
670+
diskBuilder.append("' port='");
671+
diskBuilder.append(_sourcePort);
672+
}
673+
diskBuilder.append("'/>\n");
668674
diskBuilder.append("</source>\n");
669675
if (_authUserName != null) {
670676
diskBuilder.append("<auth username='" + _authUserName + "'>\n");

‎plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -931,6 +931,12 @@ protected synchronized String attachOrDetachDisk(Connect conn, boolean attach, S
931931
if (attachingPool.getType() == StoragePoolType.RBD) {
932932
diskdef.defNetworkBasedDisk(attachingDisk.getPath(), attachingPool.getSourceHost(), attachingPool.getSourcePort(), attachingPool.getAuthUserName(),
933933
attachingPool.getUuid(), devId, DiskDef.diskBus.VIRTIO, diskProtocol.RBD);
934+
} else if (attachingPool.getType() == StoragePoolType.Gluster) {
935+
String mountpoint = attachingPool.getLocalPath();
936+
String path = attachingDisk.getPath();
937+
String glusterVolume = attachingPool.getSourceDir().replace("/", "");
938+
diskdef.defNetworkBasedDisk(glusterVolume + path.replace(mountpoint, ""), attachingPool.getSourceHost(), attachingPool.getSourcePort(), null,
939+
null, devId, DiskDef.diskBus.VIRTIO, diskProtocol.GLUSTER);
934940
} else if (attachingDisk.getFormat() == PhysicalDiskFormat.QCOW2) {
935941
diskdef.defFileBasedDisk(attachingDisk.getPath(), devId, DiskDef.diskBus.VIRTIO, DiskDef.diskFmtType.QCOW2);
936942
} else if (attachingDisk.getFormat() == PhysicalDiskFormat.RAW) {

‎plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java‎

Lines changed: 20 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -129,9 +129,9 @@ public void storagePoolRefresh(StoragePool pool) {
129129
}
130130
}
131131

132-
private StoragePool createNfsStoragePool(Connect conn, String uuid, String host, String path) throws LibvirtException {
132+
private StoragePool createNetfsStoragePool(poolType fsType, Connect conn, String uuid, String host, String path) throws LibvirtException {
133133
String targetPath = _mountPoint + File.separator + uuid;
134-
LibvirtStoragePoolDef spd = new LibvirtStoragePoolDef(poolType.NETFS, uuid, uuid, host, path, targetPath);
134+
LibvirtStoragePoolDef spd = new LibvirtStoragePoolDef(fsType, uuid, uuid, host, path, targetPath);
135135
_storageLayer.mkdir(targetPath);
136136
StoragePool sp = null;
137137
try {
@@ -170,7 +170,7 @@ private StoragePool createNfsStoragePool(Connect conn, String uuid, String host,
170170
}
171171
sp.free();
172172
} catch (LibvirtException l) {
173-
s_logger.debug("Failed to undefine nfs storage pool with: " + l.toString());
173+
s_logger.debug("Failed to undefine " + fsType.toString() + " storage pool with: " + l.toString());
174174
}
175175
}
176176
return null;
@@ -345,14 +345,19 @@ public KVMStoragePool getStoragePool(String uuid) {
345345
type = StoragePoolType.RBD;
346346
} else if (spd.getPoolType() == LibvirtStoragePoolDef.poolType.LOGICAL) {
347347
type = StoragePoolType.CLVM;
348+
} else if (spd.getPoolType() == LibvirtStoragePoolDef.poolType.GLUSTERFS) {
349+
type = StoragePoolType.Gluster;
348350
}
349351

350352
LibvirtStoragePool pool = new LibvirtStoragePool(uuid, storage.getName(), type, this, storage);
351353

352-
if (pool.getType() != StoragePoolType.RBD) {
354+
if (pool.getType() != StoragePoolType.RBD)
353355
pool.setLocalPath(spd.getTargetPath());
354-
} else {
356+
else
355357
pool.setLocalPath("");
358+
359+
if (pool.getType() == StoragePoolType.RBD
360+
|| pool.getType() == StoragePoolType.Gluster) {
356361
pool.setSourceHost(spd.getSourceHost());
357362
pool.setSourcePort(spd.getSourcePort());
358363
pool.setSourceDir(spd.getSourceDir());
@@ -484,9 +489,17 @@ public KVMStoragePool createStoragePool(String name, String host, int port, Stri
484489

485490
if (type == StoragePoolType.NetworkFilesystem) {
486491
try {
487-
sp = createNfsStoragePool(conn, name, host, path);
492+
sp = createNetfsStoragePool(poolType.NETFS, conn, name, host, path);
493+
} catch (LibvirtException e) {
494+
s_logger.error("Failed to create netfs mount: " + host + ":" + path , e);
495+
s_logger.error(e.getStackTrace());
496+
throw new CloudRuntimeException(e.toString());
497+
}
498+
} else if (type == StoragePoolType.Gluster) {
499+
try {
500+
sp = createNetfsStoragePool(poolType.GLUSTERFS, conn, name, host, path);
488501
} catch (LibvirtException e) {
489-
s_logger.error("Failed to create mount");
502+
s_logger.error("Failed to create glusterfs mount: " + host + ":" + path , e);
490503
s_logger.error(e.getStackTrace());
491504
throw new CloudRuntimeException(e.toString());
492505
}

‎plugins/storage/volume/default/src/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,12 @@ public DataStore initialize(Map<String, Object> dsInfos) {
166166
if (uriPath == null) {
167167
throw new InvalidParameterValueException("host or path is null, should be rbd://hostname/pool");
168168
}
169+
} else if (uri.getScheme().equalsIgnoreCase("gluster")) {
170+
String uriHost = uri.getHost();
171+
String uriPath = uri.getPath();
172+
if (uriHost == null || uriPath == null || uriHost.trim().isEmpty() || uriPath.trim().isEmpty()) {
173+
throw new InvalidParameterValueException("host or path is null, should be gluster://hostname/volume");
174+
}
169175
}
170176
} catch (URISyntaxException e) {
171177
throw new InvalidParameterValueException(url + " is not a valid uri");
@@ -288,6 +294,14 @@ public DataStore initialize(Map<String, Object> dsInfos) {
288294
parameters.setHost("clustered");
289295
parameters.setPort(port);
290296
parameters.setPath(hostPath);
297+
} else if (scheme.equalsIgnoreCase("gluster")) {
298+
if (port == -1) {
299+
port = 24007;
300+
}
301+
parameters.setType(StoragePoolType.Gluster);
302+
parameters.setHost(storageHost);
303+
parameters.setPort(port);
304+
parameters.setPath(hostPath);
291305
} else {
292306
StoragePoolType type = Enum.valueOf(StoragePoolType.class, scheme);
293307

@@ -349,7 +363,8 @@ protected boolean createStoragePool(long hostId, StoragePool pool) {
349363
if (pool.getPoolType() != StoragePoolType.NetworkFilesystem && pool.getPoolType() != StoragePoolType.Filesystem &&
350364
pool.getPoolType() != StoragePoolType.IscsiLUN && pool.getPoolType() != StoragePoolType.Iscsi && pool.getPoolType() != StoragePoolType.VMFS &&
351365
pool.getPoolType() != StoragePoolType.SharedMountPoint && pool.getPoolType() != StoragePoolType.PreSetup && pool.getPoolType() != StoragePoolType.OCFS2 &&
352-
pool.getPoolType() != StoragePoolType.RBD && pool.getPoolType() != StoragePoolType.CLVM && pool.getPoolType() != StoragePoolType.SMB) {
366+
pool.getPoolType() != StoragePoolType.RBD && pool.getPoolType() != StoragePoolType.CLVM && pool.getPoolType() != StoragePoolType.SMB &&
367+
pool.getPoolType() != StoragePoolType.Gluster) {
353368
s_logger.warn(" Doesn't support storage pool type " + pool.getPoolType());
354369
return false;
355370
}

0 commit comments

Comments
 (0)