Skip to content

Add support to RBD erasure code pools #9808

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
JoaoJandre merged 6 commits into from
Apr 2, 2025
Merged

Add support to RBD erasure code pools #9808

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
4c74326
Readd filename string on qemuimg create
BryanMLima Oct 16, 2024
7c196d9
Remove empty object on the data pool details of storage pools with no…
BryanMLima Oct 16, 2024
83ef09e
Only use the method createPhysicalDiskByLibVirt with RBD when the poo…
BryanMLima Nov 25, 2024
fce6e14
Change literal '/' string to File.separator
BryanMLima Nov 25, 2024
62f8d27
Add support for erasure code pools
BryanMLima Oct 14, 2024
399b023
Fix null on putAll
BryanMLima Oct 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions api/src/main/java/org/apache/cloudstack/api/response/StoragePoolResponse.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,10 @@ public class StoragePoolResponse extends BaseResponseWithAnnotations {
@Param(description = "whether this pool is managed or not")
private Boolean managed;

@SerializedName(ApiConstants.DETAILS)
@Param(description = "the storage pool details")
private Map<String, String> details;

public Map<String, String> getCaps() {
return caps;
}
Expand Down Expand Up @@ -407,4 +411,12 @@ public Boolean getManaged() {
public void setManaged(Boolean managed) {
this.managed = managed;
}

public Map<String, String> getDetails() {
return details;
}

public void setDetails(Map<String, String> details) {
this.details = details;
}
}
8 changes: 5 additions & 3 deletions ...e/src/main/java/org/apache/cloudstack/storage/datastore/provider/DefaultHostListener.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,6 @@
import com.cloud.storage.StoragePoolHostVO;
import com.cloud.storage.StorageService;
import com.cloud.storage.dao.StoragePoolHostDao;
import com.cloud.utils.Pair;
import com.cloud.utils.exception.CloudRuntimeException;

import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
Expand All @@ -60,6 +59,7 @@

import java.util.List;
import java.util.Map;
import java.util.Optional;

public class DefaultHostListener implements HypervisorHostListener {
protected Logger logger = LogManager.getLogger(getClass());
Expand Down Expand Up @@ -133,9 +133,11 @@ private NicTO createNicTOFromNetworkAndOffering(NetworkVO networkVO, NetworkOffe
@Override
public boolean hostConnect(long hostId, long poolId) throws StorageConflictException {
StoragePool pool = (StoragePool) this.dataStoreMgr.getDataStore(poolId, DataStoreRole.Primary);
Pair<Map<String, String>, Boolean> nfsMountOpts = storageManager.getStoragePoolNFSMountOpts(pool, null);
Map<String, String> detailsMap = storagePoolDetailsDao.listDetailsKeyPairs(poolId);
Map<String, String> nfsMountOpts = storageManager.getStoragePoolNFSMountOpts(pool, null).first();

ModifyStoragePoolCommand cmd = new ModifyStoragePoolCommand(true, pool, nfsMountOpts.first());
Optional.ofNullable(nfsMountOpts).ifPresent(detailsMap::putAll);
ModifyStoragePoolCommand cmd = new ModifyStoragePoolCommand(true, pool, detailsMap);
cmd.setWait(modifyStoragePoolCommandWait);
HostVO host = hostDao.findById(hostId);
logger.debug("Sending modify storage pool command to agent: {} for storage pool: {} with timeout {} seconds", host, pool, cmd.getWait());
Expand Down
4 changes: 1 addition & 3 deletions ...hypervisor/kvm/resource/wrapper/LibvirtCreatePrivateTemplateFromVolumeCommandWrapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,9 +108,7 @@ public Answer execute(final CreatePrivateTemplateFromVolumeCommand command, fina
} else {
logger.debug("Converting RBD disk " + disk.getPath() + " into template " + command.getUniqueName());

final QemuImgFile srcFile =
new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(primary.getSourceHost(), primary.getSourcePort(), primary.getAuthUserName(),
primary.getAuthSecret(), disk.getPath()));
final QemuImgFile srcFile = new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(primary, disk.getPath()));
srcFile.setFormat(PhysicalDiskFormat.RAW);

final QemuImgFile destFile = new QemuImgFile(tmpltPath + "/" + command.getUniqueName() + ".qcow2");
Expand Down
6 changes: 1 addition & 5 deletions ...a/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -161,11 +161,7 @@ private Map<String, String> getDiskFileInfo(KVMStoragePool pool, KVMPhysicalDisk
QemuImg qemu = new QemuImg(0);
QemuImgFile qemuFile = new QemuImgFile(disk.getPath(), disk.getFormat());
if (StoragePoolType.RBD.equals(pool.getType())) {
String rbdDestFile = KVMPhysicalDisk.RBDStringBuilder(pool.getSourceHost(),
pool.getSourcePort(),
pool.getAuthUserName(),
pool.getAuthSecret(),
disk.getPath());
String rbdDestFile = KVMPhysicalDisk.RBDStringBuilder(pool, disk.getPath());
qemuFile = new QemuImgFile(rbdDestFile, disk.getFormat());
}
return qemu.info(qemuFile, secure);
Expand Down
4 changes: 1 addition & 3 deletions ...ypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/IscsiAdmStorageAdaptor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -410,9 +410,7 @@ public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk srcDisk, String destVolu
KVMStoragePool srcPool = srcDisk.getPool();

if (srcPool.getType() == StoragePoolType.RBD) {
srcFile = new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(srcPool.getSourceHost(), srcPool.getSourcePort(),
srcPool.getAuthUserName(), srcPool.getAuthSecret(),
srcDisk.getPath()),srcDisk.getFormat());
srcFile = new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(srcPool, srcDisk.getPath()), srcDisk.getFormat());
} else {
srcFile = new QemuImgFile(srcDisk.getPath(), srcDisk.getFormat());
}
Expand Down
18 changes: 15 additions & 3 deletions plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMPhysicalDisk.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@

import java.util.ArrayList;
import java.util.List;
import java.util.Map;

public class KVMPhysicalDisk {
private String path;
Expand All @@ -32,10 +33,17 @@ public class KVMPhysicalDisk {
private String vmName;
private boolean useAsTemplate;

public static String RBDStringBuilder(String monHost, int monPort, String authUserName, String authSecret, String image) {
String rbdOpts;
public static final String RBD_DEFAULT_DATA_POOL = "rbd_default_data_pool";

rbdOpts = "rbd:" + image;
public static String RBDStringBuilder(KVMStoragePool storagePool, String image) {
String monHost = storagePool.getSourceHost();
int monPort = storagePool.getSourcePort();
String authUserName = storagePool.getAuthUserName();
String authSecret = storagePool.getAuthSecret();
Map<String, String> details = storagePool.getDetails();
String dataPool = (details == null) ? null : details.get(RBD_DEFAULT_DATA_POOL);

String rbdOpts = "rbd:" + image;
rbdOpts += ":mon_host=" + composeOptionForMonHosts(monHost, monPort);

if (authUserName == null) {
Expand All @@ -46,6 +54,10 @@ public static String RBDStringBuilder(String monHost, int monPort, String authUs
rbdOpts += ":key=" + authSecret;
}

if (dataPool != null) {
rbdOpts += String.format(":rbd_default_data_pool=%s", dataPool);
}

rbdOpts += ":rbd_default_format=2";
rbdOpts += ":client_mount_timeout=30";

Expand Down
43 changes: 20 additions & 23 deletions ...hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,28 +53,6 @@
public class KVMStoragePoolManager {
protected Logger logger = LogManager.getLogger(getClass());

private class StoragePoolInformation {
String name;
String host;
int port;
String path;
String userInfo;
boolean type;
StoragePoolType poolType;
Map<String, String> details;

public StoragePoolInformation(String name, String host, int port, String path, String userInfo, StoragePoolType poolType, Map<String, String> details, boolean type) {
this.name = name;
this.host = host;
this.port = port;
this.path = path;
this.userInfo = userInfo;
this.type = type;
this.poolType = poolType;
this.details = details;
}
}

private KVMHAMonitor _haMonitor;
private final Map<String, StoragePoolInformation> _storagePools = new ConcurrentHashMap<String, StoragePoolInformation>();
private final Map<String, StorageAdaptor> _storageMapper = new HashMap<String, StorageAdaptor>();
Expand Down Expand Up @@ -303,14 +281,33 @@ public KVMStoragePool getStoragePool(StoragePoolType type, String uuid, boolean
} catch (Exception e) {
StoragePoolInformation info = _storagePools.get(uuid);
if (info != null) {
pool = createStoragePool(info.name, info.host, info.port, info.path, info.userInfo, info.poolType, info.details, info.type);
pool = createStoragePool(info.getName(), info.getHost(), info.getPort(), info.getPath(), info.getUserInfo(), info.getPoolType(), info.getDetails(), info.isType());
} else {
throw new CloudRuntimeException("Could not fetch storage pool " + uuid + " from libvirt due to " + e.getMessage());
}
}

if (pool instanceof LibvirtStoragePool) {
addPoolDetails(uuid, (LibvirtStoragePool) pool);
}

return pool;
}

/**
* As the class {@link LibvirtStoragePool} is constrained to the {@link org.libvirt.StoragePool} class, there is no way of saving a generic parameter such as the details, hence,
* this method was created to always make available the details of libvirt primary storages for when they are needed.
*/
private void addPoolDetails(String uuid, LibvirtStoragePool pool) {
StoragePoolInformation storagePoolInformation = _storagePools.get(uuid);
Map<String, String> details = storagePoolInformation.getDetails();

if (MapUtils.isNotEmpty(details)) {
logger.trace("Adding the details {} to the pool with UUID {}.", details, uuid);
pool.setDetails(details);
}
}

public KVMStoragePool getStoragePoolByURI(String uri) {
URI storageUri = null;

Expand Down
8 changes: 2 additions & 6 deletions ...s/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -661,9 +661,7 @@ public Answer createTemplateFromVolume(final CopyCommand cmd) {
} else {
logger.debug("Converting RBD disk " + disk.getPath() + " into template " + templateName);

final QemuImgFile srcFile =
new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(primary.getSourceHost(), primary.getSourcePort(), primary.getAuthUserName(),
primary.getAuthSecret(), disk.getPath()));
final QemuImgFile srcFile = new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(primary, disk.getPath()));
srcFile.setFormat(PhysicalDiskFormat.RAW);

final QemuImgFile destFile = new QemuImgFile(tmpltPath + "/" + templateName + ".qcow2");
Expand Down Expand Up @@ -1010,9 +1008,7 @@ public Answer backupSnapshot(final CopyCommand cmd) {
logger.debug("Attempting to create " + snapDir.getAbsolutePath() + " recursively for snapshot storage");
FileUtils.forceMkdir(snapDir);

final QemuImgFile srcFile =
new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(primaryPool.getSourceHost(), primaryPool.getSourcePort(), primaryPool.getAuthUserName(),
primaryPool.getAuthSecret(), rbdSnapshot));
final QemuImgFile srcFile = new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(primaryPool, rbdSnapshot));
srcFile.setFormat(snapshotDisk.getFormat());

final QemuImgFile destFile = new QemuImgFile(snapshotFile);
Expand Down
81 changes: 58 additions & 23 deletions ...hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -960,17 +960,55 @@ public boolean deleteStoragePool(String uuid) {
}
}

/**
* Creates a physical disk depending on the {@link StoragePoolType}:
* <ul>
* <li>
* <b>{@link StoragePoolType#RBD}</b>
* <ul>
* <li>
* If it is an erasure code pool, utilizes QemuImg to create the physical disk through the method
* {@link LibvirtStorageAdaptor#createPhysicalDiskByQemuImg(String, KVMStoragePool, PhysicalDiskFormat, Storage.ProvisioningType, long, byte[])}
* </li>
* <li>
* Otherwise, utilize Libvirt to create the physical disk through the method
* {@link LibvirtStorageAdaptor#createPhysicalDiskByLibVirt(String, KVMStoragePool, PhysicalDiskFormat, Storage.ProvisioningType, long)}
* </li>
* </ul>
* </li>
* <li>
* {@link StoragePoolType#NetworkFilesystem} and {@link StoragePoolType#Filesystem}
* <ul>
* <li>
* If the format is {@link PhysicalDiskFormat#QCOW2} or {@link PhysicalDiskFormat#RAW}, utilizes QemuImg to create the physical disk through the method
* {@link LibvirtStorageAdaptor#createPhysicalDiskByQemuImg(String, KVMStoragePool, PhysicalDiskFormat, Storage.ProvisioningType, long, byte[])}
* </li>
* <li>
* If the format is {@link PhysicalDiskFormat#DIR} or {@link PhysicalDiskFormat#TAR}, utilize Libvirt to create the physical disk through the method
* {@link LibvirtStorageAdaptor#createPhysicalDiskByLibVirt(String, KVMStoragePool, PhysicalDiskFormat, Storage.ProvisioningType, long)}
* </li>
* </ul>
* </li>
* <li>
* For the rest of the {@link StoragePoolType} types, utilizes the Libvirt method
* {@link LibvirtStorageAdaptor#createPhysicalDiskByLibVirt(String, KVMStoragePool, PhysicalDiskFormat, Storage.ProvisioningType, long)}
* </li>
* </ul>
*/
@Override
public KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool pool,
PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, byte[] passphrase) {

logger.info("Attempting to create volume " + name + " (" + pool.getType().toString() + ") in pool "
+ pool.getUuid() + " with size " + toHumanReadableSize(size));
logger.info("Attempting to create volume {} ({}) in pool {} with size {}", name, pool.getType().toString(), pool.getUuid(), toHumanReadableSize(size));

StoragePoolType poolType = pool.getType();
if (poolType.equals(StoragePoolType.RBD)) {
return createPhysicalDiskByLibVirt(name, pool, PhysicalDiskFormat.RAW, provisioningType, size);
} else if (poolType.equals(StoragePoolType.NetworkFilesystem) || poolType.equals(StoragePoolType.Filesystem)) {
if (StoragePoolType.RBD.equals(poolType)) {
Map<String, String> details = pool.getDetails();
String dataPool = (details == null) ? null : details.get(KVMPhysicalDisk.RBD_DEFAULT_DATA_POOL);

return (dataPool == null) ? createPhysicalDiskByLibVirt(name, pool, PhysicalDiskFormat.RAW, provisioningType, size) :
createPhysicalDiskByQemuImg(name, pool, PhysicalDiskFormat.RAW, provisioningType, size, passphrase);
} else if (StoragePoolType.NetworkFilesystem.equals(poolType) || StoragePoolType.Filesystem.equals(poolType)) {
switch (format) {
case QCOW2:
case RAW:
Expand Down Expand Up @@ -1018,18 +1056,25 @@ private KVMPhysicalDisk createPhysicalDiskByLibVirt(String name, KVMStoragePool
}


private KVMPhysicalDisk createPhysicalDiskByQemuImg(String name, KVMStoragePool pool,
PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, byte[] passphrase) {
String volPath = pool.getLocalPath() + "/" + name;
private KVMPhysicalDisk createPhysicalDiskByQemuImg(String name, KVMStoragePool pool, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size,
byte[] passphrase) {
String volPath;
String volName = name;
long virtualSize = 0;
long actualSize = 0;
QemuObject.EncryptFormat encryptFormat = null;
List<QemuObject> passphraseObjects = new ArrayList<>();

final int timeout = 0;
QemuImgFile destFile;

if (StoragePoolType.RBD.equals(pool.getType())) {
volPath = pool.getSourceDir() + File.separator + name;
destFile = new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(pool, volPath));
} else {
volPath = pool.getLocalPath() + File.separator + name;
destFile = new QemuImgFile(volPath);
}

QemuImgFile destFile = new QemuImgFile(volPath);
destFile.setFormat(format);
destFile.setSize(size);
Map<String, String> options = new HashMap<String, String>();
Expand Down Expand Up @@ -1312,11 +1357,7 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template,


QemuImgFile srcFile;
QemuImgFile destFile = new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(destPool.getSourceHost(),
destPool.getSourcePort(),
destPool.getAuthUserName(),
destPool.getAuthSecret(),
disk.getPath()));
QemuImgFile destFile = new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(destPool, disk.getPath()));
destFile.setFormat(format);

if (srcPool.getType() != StoragePoolType.RBD) {
Expand Down Expand Up @@ -1591,11 +1632,7 @@ to support snapshots(backuped) as qcow2 files. */
try {
srcFile = new QemuImgFile(sourcePath, sourceFormat);
String rbdDestPath = destPool.getSourceDir() + "/" + name;
String rbdDestFile = KVMPhysicalDisk.RBDStringBuilder(destPool.getSourceHost(),
destPool.getSourcePort(),
destPool.getAuthUserName(),
destPool.getAuthSecret(),
rbdDestPath);
String rbdDestFile = KVMPhysicalDisk.RBDStringBuilder(destPool, rbdDestPath);
destFile = new QemuImgFile(rbdDestFile, destFormat);

logger.debug("Starting copy from source image " + srcFile.getFileName() + " to RBD image " + rbdDestPath);
Expand Down Expand Up @@ -1638,9 +1675,7 @@ to support snapshots(backuped) as qcow2 files. */
We let Qemu-Img do the work here. Although we could work with librbd and have that do the cloning
it doesn't benefit us. It's better to keep the current code in place which works
*/
srcFile =
new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(srcPool.getSourceHost(), srcPool.getSourcePort(), srcPool.getAuthUserName(), srcPool.getAuthSecret(),
sourcePath));
srcFile = new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(srcPool, sourcePath));
srcFile.setFormat(sourceFormat);
destFile = new QemuImgFile(destPath);
destFile.setFormat(destFormat);
Expand Down
8 changes: 6 additions & 2 deletions ...ns/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,8 +56,8 @@ public class LibvirtStoragePool implements KVMStoragePool {
protected String authSecret;
protected String sourceHost;
protected int sourcePort;

protected String sourceDir;
protected Map<String, String> details;

public LibvirtStoragePool(String uuid, String name, StoragePoolType type, StorageAdaptor adaptor, StoragePool pool) {
this.uuid = uuid;
Expand Down Expand Up @@ -311,7 +311,11 @@ public boolean supportsConfigDriveIso() {

@Override
public Map<String, String> getDetails() {
return null;
return this.details;
}

public void setDetails(Map<String, String> details) {
this.details = details;
}

@Override
Expand Down
Loading