diff --git a/api/src/main/java/com/cloud/agent/api/StoragePoolInfo.java b/api/src/main/java/com/cloud/agent/api/StoragePoolInfo.java index 7d1e2389070e..d923694a854d 100644 --- a/api/src/main/java/com/cloud/agent/api/StoragePoolInfo.java +++ b/api/src/main/java/com/cloud/agent/api/StoragePoolInfo.java @@ -28,6 +28,7 @@ public class StoragePoolInfo { StoragePoolType poolType; long capacityBytes; long availableBytes; + String name; Map details; protected StoragePoolInfo() { @@ -67,14 +68,34 @@ public String getHost() { return host; } + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public void setLocalPath(String localPath) { + this.localPath = localPath; + } + public String getLocalPath() { return localPath; } + public void setHostPath(String hostPath) { + this.hostPath = hostPath; + } + public String getHostPath() { return hostPath; } + public void setPoolType(StoragePoolType poolType) { + this.poolType = poolType; + } + public StoragePoolType getPoolType() { return poolType; } diff --git a/api/src/main/java/com/cloud/agent/api/storage/OVFHelper.java b/api/src/main/java/com/cloud/agent/api/storage/OVFHelper.java index 15d63587490a..45eff89b0c91 100644 --- a/api/src/main/java/com/cloud/agent/api/storage/OVFHelper.java +++ b/api/src/main/java/com/cloud/agent/api/storage/OVFHelper.java @@ -22,8 +22,15 @@ import java.io.StringReader; import java.io.StringWriter; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedList; import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; import javax.xml.transform.Transformer; @@ -32,15 +39,27 @@ import javax.xml.transform.dom.DOMSource; import javax.xml.transform.stream.StreamResult; +import com.cloud.agent.api.to.deployasis.OVFConfigurationTO; +import com.cloud.agent.api.to.deployasis.OVFEulaSectionTO; +import com.cloud.agent.api.to.deployasis.OVFPropertyTO; +import com.cloud.agent.api.to.deployasis.OVFVirtualHardwareItemTO; +import com.cloud.agent.api.to.deployasis.OVFVirtualHardwareSectionTO; import com.cloud.configuration.Resource.ResourceType; import com.cloud.exception.InternalErrorException; +import com.cloud.utils.compression.CompressionUtil; +import com.cloud.agent.api.to.deployasis.OVFNetworkTO; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.math.NumberUtils; import org.apache.log4j.Logger; import org.w3c.dom.Document; import org.w3c.dom.Element; +import org.w3c.dom.NamedNodeMap; import org.w3c.dom.Node; import org.w3c.dom.NodeList; +import org.w3c.dom.traversal.DocumentTraversal; +import org.w3c.dom.traversal.NodeFilter; +import org.w3c.dom.traversal.NodeIterator; import org.xml.sax.InputSource; import org.xml.sax.SAXException; @@ -73,10 +92,11 @@ public static Long getDiskVirtualSize(Long capacity, String allocationUnits, Str } /** - * Get the text value of a node's child with name "childNodeName", null if not present + * Get the text value of a node's child with name or suffix "childNodeName", null if not present * Example: * * Text value + * Text value * */ private String getChildNodeValue(Node node, String childNodeName) { @@ -84,7 +104,9 @@ private String getChildNodeValue(Node node, String childNodeName) { NodeList childNodes = node.getChildNodes(); for (int i = 0; i < childNodes.getLength(); i++) { Node value = childNodes.item(i); - if (value != null && value.getNodeName().equals(childNodeName)) { + // Also match if the child's name has a suffix: + // Example: + if (value != null && (value.getNodeName().equals(childNodeName)) || value.getNodeName().endsWith(":" + childNodeName)) { return value.getTextContent(); } } @@ -92,45 +114,94 @@ private String getChildNodeValue(Node node, String childNodeName) { return null; } + /** + * Check if the attribute is present on the element, otherwise check preprending ':' + */ + private String getNodeAttribute(Element element, String prefix, String attr) { + if (element == null) { + return null; + } + if (element.hasAttribute(prefix + ":" + attr)) { + return element.getAttribute(prefix + ":" + attr); + } + else if (element.hasAttribute(attr)) { + return element.getAttribute(attr); + } + + NamedNodeMap attributes = element.getAttributes(); + if (attributes == null || attributes.getLength() == 0) { + return null; + } + for (int i = 0; i < attributes.getLength(); i++) { + Node node = attributes.item(i); + if (node != null && node.getNodeName().endsWith(":" + attr)) { + return node.getTextContent(); + } + } + return null; + } + /** * Create OVFProperty class from the parsed node. Note that some fields may not be present. * The key attribute is required */ - protected OVFPropertyTO createOVFPropertyFromNode(Node node) { - Element property = (Element) node; - String key = property.getAttribute("ovf:key"); + protected OVFPropertyTO createOVFPropertyFromNode(Node node, int index, String category) { + Element element = (Element) node; + String key = getNodeAttribute(element, "ovf","key"); if (StringUtils.isBlank(key)) { return null; } - String value = property.getAttribute("ovf:value"); - String type = property.getAttribute("ovf:type"); - String qualifiers = property.getAttribute("ovf:qualifiers"); - String userConfigurableStr = property.getAttribute("ovf:userConfigurable"); + String value = getNodeAttribute(element, "ovf","value"); + String type = getNodeAttribute(element, "ovf","type"); + String qualifiers = getNodeAttribute(element, "ovf","qualifiers"); + String userConfigurableStr = getNodeAttribute(element, "ovf","userConfigurable"); boolean userConfigurable = StringUtils.isNotBlank(userConfigurableStr) && userConfigurableStr.equalsIgnoreCase("true"); - String passStr = property.getAttribute("ovf:password"); + String passStr = getNodeAttribute(element, "ovf","password"); boolean password = StringUtils.isNotBlank(passStr) && passStr.equalsIgnoreCase("true"); String label = getChildNodeValue(node, "Label"); String description = getChildNodeValue(node, "Description"); - return new OVFPropertyTO(key, type, value, qualifiers, userConfigurable, label, description, password); + s_logger.debug("Creating OVF property index " + index + (category == null ? "" : " for category " + category) + + " with key = " + key); + return new OVFPropertyTO(key, type, value, qualifiers, userConfigurable, + label, description, password, index, category); } /** - * Retrieve OVF properties from a parsed OVF file, with attribute 'ovf:userConfigurable' set to true + * Retrieve OVF properties from a parsed OVF file including its category (if available) and in-order, + * with attribute 'ovf:userConfigurable' set to true. */ - private List getConfigurableOVFPropertiesFromDocument(Document doc) { + public List getConfigurableOVFPropertiesFromDocument(Document doc) { List props = new ArrayList<>(); - NodeList properties = doc.getElementsByTagName("Property"); - if (properties != null) { - for (int i = 0; i < properties.getLength(); i++) { - Node node = properties.item(i); + if (doc == null) { + return props; + } + int propertyIndex = 0; + NodeList productSections = doc.getElementsByTagName("ProductSection"); + if (productSections != null) { + String lastCategoryFound = null; + for (int i = 0; i < productSections.getLength(); i++) { + Node node = productSections.item(i); if (node == null) { continue; } - OVFPropertyTO prop = createOVFPropertyFromNode(node); - if (prop != null && prop.isUserConfigurable()) { - props.add(prop); + NodeList childNodes = node.getChildNodes(); + for (int j = 0; j < childNodes.getLength(); j++) { + Node child = childNodes.item(j); + if (child == null) { + continue; + } + if (child.getNodeName().equalsIgnoreCase("Category")) { + lastCategoryFound = child.getTextContent(); + s_logger.info("Category found " + lastCategoryFound); + } else if (child.getNodeName().equalsIgnoreCase("Property")) { + OVFPropertyTO prop = createOVFPropertyFromNode(child, propertyIndex, lastCategoryFound); + if (prop != null && prop.isUserConfigurable()) { + props.add(prop); + propertyIndex++; + } + } } } } @@ -138,128 +209,225 @@ private List getConfigurableOVFPropertiesFromDocument(Document do } /** - * Get properties from OVF file located on ovfFilePath + * Get properties from OVF XML string */ - public List getOVFPropertiesFromFile(String ovfFilePath) throws ParserConfigurationException, IOException, SAXException { - if (StringUtils.isBlank(ovfFilePath)) { - return new ArrayList<>(); - } - File ovfFile = new File(ovfFilePath); - final Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(ovfFile); + protected List getOVFPropertiesFromXmlString(final String ovfString) throws ParserConfigurationException, IOException, SAXException { + InputSource is = new InputSource(new StringReader(ovfString)); + final Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(is); return getConfigurableOVFPropertiesFromDocument(doc); } - /** - * Get properties from OVF XML string - */ - protected List getOVFPropertiesXmlString(final String ovfFilePath) throws ParserConfigurationException, IOException, SAXException { - InputSource is = new InputSource(new StringReader(ovfFilePath)); + protected List getOVFDeploymentOptionsFromXmlString(final String ovfString) throws ParserConfigurationException, IOException, SAXException { + InputSource is = new InputSource(new StringReader(ovfString)); final Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(is); - return getConfigurableOVFPropertiesFromDocument(doc); + return getDeploymentOptionsFromDocumentTree(doc); } - public List getOVFVolumeInfo(final String ovfFilePath) { + protected List getOVFVirtualHardwareSectionFromXmlString(final String ovfString) throws ParserConfigurationException, IOException, SAXException { + InputSource is = new InputSource(new StringReader(ovfString)); + final Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(is); + return getVirtualHardwareItemsFromDocumentTree(doc); + } + + protected List getOVFEulaSectionFromXmlString(final String ovfString) throws ParserConfigurationException, IOException, SAXException { + InputSource is = new InputSource(new StringReader(ovfString)); + final Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(is); + return getEulaSectionsFromDocument(doc); + } + + public List getOVFVolumeInfoFromFile(final String ovfFilePath, final String configurationId) throws InternalErrorException { if (StringUtils.isBlank(ovfFilePath)) { - return new ArrayList(); + return new ArrayList<>(); + } + Document doc = getDocumentFromFile(ovfFilePath); + + return getOVFVolumeInfoFromFile(ovfFilePath, doc, configurationId); + } + + public List getOVFVolumeInfoFromFile(String ovfFilePath, Document doc, String configurationId) throws InternalErrorException { + if (org.apache.commons.lang.StringUtils.isBlank(ovfFilePath)) { + return null; } - ArrayList vf = new ArrayList(); - ArrayList vd = new ArrayList(); File ovfFile = new File(ovfFilePath); - try { - final Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(new File(ovfFilePath)); - NodeList disks = doc.getElementsByTagName("Disk"); - NodeList files = doc.getElementsByTagName("File"); - NodeList items = doc.getElementsByTagName("Item"); - boolean toggle = true; - for (int j = 0; j < files.getLength(); j++) { - Element file = (Element)files.item(j); - OVFFile of = new OVFFile(); - of._href = file.getAttribute("ovf:href"); - if (of._href.endsWith("vmdk") || of._href.endsWith("iso")) { - of._id = file.getAttribute("ovf:id"); - String size = file.getAttribute("ovf:size"); - if (StringUtils.isNotBlank(size)) { - of._size = Long.parseLong(size); - } else { - String dataDiskPath = ovfFile.getParent() + File.separator + of._href; - File this_file = new File(dataDiskPath); - of._size = this_file.length(); - } - of.isIso = of._href.endsWith("iso"); - if (toggle && !of.isIso) { - of._bootable = true; - toggle = !toggle; - } - vf.add(of); - } + List hardwareItems = getVirtualHardwareItemsFromDocumentTree(doc); + List files = extractFilesFromOvfDocumentTree(ovfFile, doc); + List disks = extractDisksFromOvfDocumentTree(doc); + + List diskHardwareItems = hardwareItems.stream() + .filter(x -> x.getResourceType() == OVFVirtualHardwareItemTO.HardwareResourceType.DiskDrive && + hardwareItemContainsConfiguration(x, configurationId)) + .collect(Collectors.toList()); + List diskTOs = matchHardwareItemsToDiskAndFilesInformation(diskHardwareItems, files, disks, ovfFile.getParent()); + return diskTOs; + } + + private String extractDiskIdFromDiskHostResource(String hostResource) { + if (hostResource.startsWith("ovf:/disk/")) { + return hostResource.replace("ovf:/disk/", ""); + } + String[] resourceParts = hostResource.split("/"); + return resourceParts[resourceParts.length - 1]; + } + + private OVFDisk getDiskDefinitionFromDiskId(String diskId, List disks) { + for (OVFDisk disk : disks) { + if (disk._diskId.equalsIgnoreCase(diskId)) { + return disk; } - for (int i = 0; i < disks.getLength(); i++) { - Element disk = (Element)disks.item(i); - OVFDisk od = new OVFDisk(); - String virtualSize = disk.getAttribute("ovf:capacity"); - od._capacity = NumberUtils.toLong(virtualSize, 0L); - String allocationUnits = disk.getAttribute("ovf:capacityAllocationUnits"); - od._diskId = disk.getAttribute("ovf:diskId"); - od._fileRef = disk.getAttribute("ovf:fileRef"); - od._populatedSize = NumberUtils.toLong(disk.getAttribute("ovf:populatedSize")); - - if ((od._capacity != 0) && (allocationUnits != null)) { - - long units = 1; - if (allocationUnits.equalsIgnoreCase("KB") || allocationUnits.equalsIgnoreCase("KiloBytes") || allocationUnits.equalsIgnoreCase("byte * 2^10")) { - units = ResourceType.bytesToKiB; - } else if (allocationUnits.equalsIgnoreCase("MB") || allocationUnits.equalsIgnoreCase("MegaBytes") || allocationUnits.equalsIgnoreCase("byte * 2^20")) { - units = ResourceType.bytesToMiB; - } else if (allocationUnits.equalsIgnoreCase("GB") || allocationUnits.equalsIgnoreCase("GigaBytes") || allocationUnits.equalsIgnoreCase("byte * 2^30")) { - units = ResourceType.bytesToGiB; - } - od._capacity = od._capacity * units; - } - od._controller = getControllerType(items, od._diskId); - vd.add(od); + } + return null; + } + + private List matchHardwareItemsToDiskAndFilesInformation(List diskHardwareItems, + List files, List disks, + String ovfParentPath) throws InternalErrorException { + List diskTOs = new LinkedList<>(); + int diskNumber = 0; + for (OVFVirtualHardwareItemTO diskItem : diskHardwareItems) { + if (StringUtils.isBlank(diskItem.getHostResource())) { + s_logger.error("Missing disk information for hardware item " + diskItem.getElementName() + " " + diskItem.getInstanceId()); + continue; + } + String diskId = extractDiskIdFromDiskHostResource(diskItem.getHostResource()); + OVFDisk diskDefinition = getDiskDefinitionFromDiskId(diskId, disks); + if (diskDefinition == null) { + s_logger.error("Missing disk definition for disk ID " + diskId); } + OVFFile fileDefinition = getFileDefinitionFromDiskDefinition(diskDefinition._fileRef, files); + DatadiskTO datadiskTO = generateDiskTO(fileDefinition, diskDefinition, ovfParentPath, diskNumber, diskItem); + diskTOs.add(datadiskTO); + diskNumber++; + } + List isoFiles = files.stream().filter(x -> x.isIso).collect(Collectors.toList()); + for (OVFFile isoFile : isoFiles) { + DatadiskTO isoTO = generateDiskTO(isoFile, null, ovfParentPath, diskNumber, null); + diskTOs.add(isoTO); + diskNumber++; + } + return diskTOs; + } - } catch (SAXException | IOException | ParserConfigurationException e) { - s_logger.error("Unexpected exception caught while parsing ovf file:" + ovfFilePath, e); - throw new CloudRuntimeException(e); + private DatadiskTO generateDiskTO(OVFFile file, OVFDisk disk, String ovfParentPath, int diskNumber, + OVFVirtualHardwareItemTO diskItem) throws InternalErrorException { + String path = file != null ? ovfParentPath + File.separator + file._href : null; + if (StringUtils.isNotBlank(path)) { + File f = new File(path); + if (!f.exists() || f.isDirectory()) { + s_logger.error("One of the attached disk or iso does not exists " + path); + throw new InternalErrorException("One of the attached disk or iso as stated on OVF does not exists " + path); + } + } + Long capacity = disk != null ? disk._capacity : file._size; + Long fileSize = file != null ? file._size : 0L; + + String controller = ""; + String controllerSubType = ""; + if (disk != null) { + OVFDiskController cDiskController = disk._controller; + controller = cDiskController == null ? "" : disk._controller._name; + controllerSubType = cDiskController == null ? "" : disk._controller._subType; } - List disksTO = new ArrayList(); - for (OVFFile of : vf) { - if (StringUtils.isBlank(of._id)){ - s_logger.error("The ovf file info is incomplete file info"); - throw new CloudRuntimeException("The ovf file info has incomplete file info"); + boolean isIso = file != null && file.isIso; + boolean bootable = file != null && file._bootable; + String diskId = disk == null ? file._id : disk._diskId; + String configuration = diskItem != null ? diskItem.getConfigurationIds() : null; + return new DatadiskTO(path, capacity, fileSize, diskId, + isIso, bootable, controller, controllerSubType, diskNumber, configuration); + } + + protected List extractDisksFromOvfDocumentTree(Document doc) { + NodeList disks = doc.getElementsByTagName("Disk"); + NodeList ovfDisks = doc.getElementsByTagName("ovf:Disk"); + NodeList items = doc.getElementsByTagName("Item"); + + int totalDisksLength = disks.getLength() + ovfDisks.getLength(); + ArrayList vd = new ArrayList<>(); + for (int i = 0; i < totalDisksLength; i++) { + Element disk; + if (i >= disks.getLength()) { + int pos = i - disks.getLength(); + disk = (Element) ovfDisks.item(pos); + } else { + disk = (Element) disks.item(i); } - OVFDisk cdisk = getDisk(of._id, vd); - if (cdisk == null && !of.isIso){ - s_logger.error("The ovf file info has incomplete disk info"); - throw new CloudRuntimeException("The ovf file info has incomplete disk info"); + + if (disk == null) { + continue; } - Long capacity = cdisk == null ? of._size : cdisk._capacity; - String controller = ""; - String controllerSubType = ""; - if (cdisk != null) { - OVFDiskController cDiskController = cdisk._controller; - controller = cDiskController == null ? "" : cdisk._controller._name; - controllerSubType = cDiskController == null ? "" : cdisk._controller._subType; + OVFDisk od = new OVFDisk(); + String virtualSize = getNodeAttribute(disk, "ovf", "capacity"); + od._capacity = NumberUtils.toLong(virtualSize, 0L); + String allocationUnits = getNodeAttribute(disk,"ovf","capacityAllocationUnits"); + od._diskId = getNodeAttribute(disk,"ovf","diskId"); + od._fileRef = getNodeAttribute(disk,"ovf","fileRef"); + od._populatedSize = NumberUtils.toLong(getNodeAttribute(disk,"ovf","populatedSize")); + + if ((od._capacity != 0) && (allocationUnits != null)) { + long units = 1; + if (allocationUnits.equalsIgnoreCase("KB") || allocationUnits.equalsIgnoreCase("KiloBytes") || allocationUnits.equalsIgnoreCase("byte * 2^10")) { + units = ResourceType.bytesToKiB; + } else if (allocationUnits.equalsIgnoreCase("MB") || allocationUnits.equalsIgnoreCase("MegaBytes") || allocationUnits.equalsIgnoreCase("byte * 2^20")) { + units = ResourceType.bytesToMiB; + } else if (allocationUnits.equalsIgnoreCase("GB") || allocationUnits.equalsIgnoreCase("GigaBytes") || allocationUnits.equalsIgnoreCase("byte * 2^30")) { + units = ResourceType.bytesToGiB; + } + od._capacity = od._capacity * units; } + od._controller = getControllerType(items, od._diskId); + vd.add(od); + } + if (s_logger.isTraceEnabled()) { + s_logger.trace(String.format("found %d disk definitions",vd.size())); + } + return vd; + } - String dataDiskPath = ovfFile.getParent() + File.separator + of._href; - File f = new File(dataDiskPath); - if (!f.exists() || f.isDirectory()) { - s_logger.error("One of the attached disk or iso does not exists " + dataDiskPath); - throw new CloudRuntimeException("One of the attached disk or iso as stated on OVF does not exists " + dataDiskPath); + protected List extractFilesFromOvfDocumentTree(File ovfFile, Document doc) { + NodeList files = doc.getElementsByTagName("File"); + ArrayList vf = new ArrayList<>(); + boolean toggle = true; + for (int j = 0; j < files.getLength(); j++) { + Element file = (Element)files.item(j); + OVFFile of = new OVFFile(); + of._href = getNodeAttribute(file,"ovf","href"); + if (of._href.endsWith("vmdk") || of._href.endsWith("iso")) { + of._id = getNodeAttribute(file,"ovf","id"); + String size = getNodeAttribute(file,"ovf", "size"); + if (StringUtils.isNotBlank(size)) { + of._size = Long.parseLong(size); + } else { + String dataDiskPath = ovfFile.getParent() + File.separator + of._href; + File this_file = new File(dataDiskPath); + of._size = this_file.length(); + } + of.isIso = of._href.endsWith("iso"); + if (toggle && !of.isIso) { + of._bootable = true; + toggle = !toggle; + } + vf.add(of); } - disksTO.add(new DatadiskTO(dataDiskPath, capacity, of._size, of._id, of.isIso, of._bootable, controller, controllerSubType)); } - //check if first disk is an iso move it to the end - DatadiskTO fd = disksTO.get(0); - if (fd.isIso()) { - disksTO.remove(0); - disksTO.add(fd); + if (s_logger.isTraceEnabled()) { + s_logger.trace(String.format("found %d file definitions in %s",vf.size(), ovfFile.getPath())); + } + return vf; + } + + public Document getDocumentFromFile(String ovfFilePath) { + if (org.apache.commons.lang.StringUtils.isBlank(ovfFilePath)) { + return null; + } + DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newDefaultInstance(); + try { + DocumentBuilder builder = documentBuilderFactory.newDocumentBuilder(); + return builder.parse(new File(ovfFilePath)); + } catch (SAXException | IOException | ParserConfigurationException e) { + s_logger.error("Unexpected exception caught while parsing ovf file:" + ovfFilePath, e); + throw new CloudRuntimeException(e); } - return disksTO; } private OVFDiskController getControllerType(final NodeList itemList, final String diskId) { @@ -330,55 +498,61 @@ private OVFDiskController getController(Element controllerItem) { return dc; } - public void rewriteOVFFile(final String origOvfFilePath, final String newOvfFilePath, final String diskName) { - try { - final Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(new File(origOvfFilePath)); - NodeList disks = doc.getElementsByTagName("Disk"); - NodeList files = doc.getElementsByTagName("File"); - NodeList items = doc.getElementsByTagName("Item"); - String keepfile = null; - List toremove = new ArrayList(); - for (int j = 0; j < files.getLength(); j++) { - Element file = (Element)files.item(j); - String href = file.getAttribute("ovf:href"); - if (diskName.equals(href)) { - keepfile = file.getAttribute("ovf:id"); - } else { - toremove.add(file); - } + public void rewriteOVFFileForSingleDisk(final String origOvfFilePath, final String newOvfFilePath, final String diskName) { + final Document doc = getDocumentFromFile(origOvfFilePath); + + NodeList disks = doc.getElementsByTagName("Disk"); + NodeList files = doc.getElementsByTagName("File"); + NodeList items = doc.getElementsByTagName("Item"); + String keepfile = null; + List toremove = new ArrayList<>(); + for (int j = 0; j < files.getLength(); j++) { + Element file = (Element)files.item(j); + String href = getNodeAttribute(file,"ovf", "href"); + if (diskName.equals(href)) { + keepfile = getNodeAttribute(file,"ovf","id"); + } else { + toremove.add(file); } - String keepdisk = null; - for (int i = 0; i < disks.getLength(); i++) { - Element disk = (Element)disks.item(i); - String fileRef = disk.getAttribute("ovf:fileRef"); - if (keepfile == null) { - s_logger.info("FATAL: OVA format error"); - } else if (keepfile.equals(fileRef)) { - keepdisk = disk.getAttribute("ovf:diskId"); - } else { - toremove.add(disk); - } + } + String keepdisk = null; + for (int i = 0; i < disks.getLength(); i++) { + Element disk = (Element)disks.item(i); + String fileRef = getNodeAttribute(disk,"ovf","fileRef"); + if (keepfile == null) { + s_logger.info("FATAL: OVA format error"); + } else if (keepfile.equals(fileRef)) { + keepdisk = getNodeAttribute(disk,"ovf","diskId"); + } else { + toremove.add(disk); } - for (int k = 0; k < items.getLength(); k++) { - Element item = (Element)items.item(k); - NodeList cn = item.getChildNodes(); - for (int l = 0; l < cn.getLength(); l++) { - if (cn.item(l) instanceof Element) { - Element el = (Element)cn.item(l); - if ("rasd:HostResource".equals(el.getNodeName()) - && !(el.getTextContent().contains("ovf:/file/" + keepdisk) || el.getTextContent().contains("ovf:/disk/" + keepdisk))) { - toremove.add(item); - break; - } + } + for (int k = 0; k < items.getLength(); k++) { + Element item = (Element) items.item(k); + NodeList cn = item.getChildNodes(); + for (int l = 0; l < cn.getLength(); l++) { + if (cn.item(l) instanceof Element) { + Element el = (Element) cn.item(l); + if ("rasd:HostResource".equals(el.getNodeName()) + && !(el.getTextContent().contains("ovf:/file/" + keepdisk) || el.getTextContent().contains("ovf:/disk/" + keepdisk))) { + toremove.add(item); + break; } } } + } - for (Element rme : toremove) { - if (rme.getParentNode() != null) { - rme.getParentNode().removeChild(rme); - } + for (Element rme : toremove) { + if (rme.getParentNode() != null) { + rme.getParentNode().removeChild(rme); } + } + + writeDocumentToFile(newOvfFilePath, doc); + } + + private void writeDocumentToFile(String newOvfFilePath, Document doc) { + try { final StringWriter writer = new StringWriter(); final StreamResult result = new StreamResult(writer); @@ -389,21 +563,312 @@ public void rewriteOVFFile(final String origOvfFilePath, final String newOvfFile PrintWriter outfile = new PrintWriter(newOvfFilePath); outfile.write(writer.toString()); outfile.close(); - } catch (SAXException | IOException | ParserConfigurationException | TransformerException e) { - s_logger.info("Unexpected exception caught while removing network elements from OVF:" + e.getMessage(), e); + } catch (IOException | TransformerException e) { + s_logger.info("Unexpected exception caught while rewriting OVF:" + e.getMessage(), e); throw new CloudRuntimeException(e); } } - OVFDisk getDisk(String fileRef, List disks) { - for (OVFDisk disk : disks) { - if (disk._fileRef.equals(fileRef)) { - return disk; + OVFFile getFileDefinitionFromDiskDefinition(String fileRef, List files) { + for (OVFFile file : files) { + if (file._id.equals(fileRef)) { + return file; } } return null; } + public List getNetPrerequisitesFromDocument(Document doc) throws InternalErrorException { + if (doc == null) { + if (s_logger.isTraceEnabled()) { + s_logger.trace("no document to parse; returning no prerequiste networks"); + } + return Collections.emptyList(); + } + + Map nets = getNetworksFromDocumentTree(doc); + + checkForOnlyOneSystemNode(doc); + + matchNicsToNets(nets, doc); + + return new ArrayList<>(nets.values()); + } + + private void matchNicsToNets(Map nets, Node systemElement) { + final DocumentTraversal traversal = (DocumentTraversal) systemElement; + final NodeIterator iterator = traversal.createNodeIterator(systemElement, NodeFilter.SHOW_ELEMENT, null, true); + if (s_logger.isTraceEnabled()) { + s_logger.trace(String.format("starting out with %d network-prerequisites, parsing hardware",nets.size())); + } + int nicCount = 0; + for (Node n = iterator.nextNode(); n != null; n = iterator.nextNode()) { + final Element e = (Element) n; + if ("rasd:Connection".equals(e.getTagName())) { + nicCount++; + String name = e.getTextContent(); // should be in our nets + if(nets.get(name) == null) { + if(s_logger.isInfoEnabled()) { + s_logger.info(String.format("found a nic definition without a network definition byname %s, adding it to the list.", name)); + } + nets.put(name, new OVFNetworkTO()); + } + OVFNetworkTO thisNet = nets.get(name); + if (e.getParentNode() != null) { + fillNicPrerequisites(thisNet,e.getParentNode()); + } + } + } + if (s_logger.isTraceEnabled()) { + s_logger.trace(String.format("ending up with %d network-prerequisites, parsed %d nics", nets.size(), nicCount)); + } + } + + /** + * get all the stuff from parent node + * + * @param nic the object to carry through the system + * @param parentNode the xml container node for nic data + */ + private void fillNicPrerequisites(OVFNetworkTO nic, Node parentNode) { + String addressOnParentStr = getChildNodeValue(parentNode, "AddressOnParent"); + String automaticAllocationStr = getChildNodeValue(parentNode, "AutomaticAllocation"); + String description = getChildNodeValue(parentNode, "Description"); + String elementName = getChildNodeValue(parentNode, "ElementName"); + String instanceIdStr = getChildNodeValue(parentNode, "InstanceID"); + String resourceSubType = getChildNodeValue(parentNode, "ResourceSubType"); + String resourceType = getChildNodeValue(parentNode, "ResourceType"); + + try { + int addressOnParent = Integer.parseInt(addressOnParentStr); + nic.setAddressOnParent(addressOnParent); + } catch (NumberFormatException e) { + s_logger.warn("Encountered element of type \"AddressOnParent\", that could not be parse to an integer number: " + addressOnParentStr); + } + + boolean automaticAllocation = StringUtils.isNotBlank(automaticAllocationStr) && Boolean.parseBoolean(automaticAllocationStr); + nic.setAutomaticAllocation(automaticAllocation); + nic.setNicDescription(description); + nic.setElementName(elementName); + + try { + int instanceId = Integer.parseInt(instanceIdStr); + nic.setInstanceID(instanceId); + } catch (NumberFormatException e) { + s_logger.warn("Encountered element of type \"InstanceID\", that could not be parse to an integer number: " + instanceIdStr); + } + + nic.setResourceSubType(resourceSubType); + nic.setResourceType(resourceType); + } + + private void checkForOnlyOneSystemNode(Document doc) throws InternalErrorException { + // get hardware VirtualSystem, for now we support only one of those + NodeList systemElements = doc.getElementsByTagName("VirtualSystem"); + if (systemElements.getLength() != 1) { + String msg = "found " + systemElements.getLength() + " system definitions in OVA, can only handle exactly one."; + s_logger.warn(msg); + throw new InternalErrorException(msg); + } + } + + private Map getNetworksFromDocumentTree(Document doc) { + NodeList networkElements = doc.getElementsByTagName("Network"); + Map nets = new HashMap<>(); + for (int i = 0; i < networkElements.getLength(); i++) { + + Element networkElement = (Element)networkElements.item(i); + String networkName = getNodeAttribute(networkElement,"ovf","name"); + + String description = getChildNodeValue(networkElement, "Description"); + + OVFNetworkTO network = new OVFNetworkTO(); + network.setName(networkName); + network.setNetworkDescription(description); + + nets.put(networkName,network); + } + if (s_logger.isTraceEnabled()) { + s_logger.trace(String.format("found %d networks in template", nets.size())); + } + return nets; + } + + private boolean hardwareItemContainsConfiguration(OVFVirtualHardwareItemTO item, String configurationId) { + if (StringUtils.isBlank(configurationId) || StringUtils.isBlank(item.getConfigurationIds())) { + return true; + } + String configurationIds = item.getConfigurationIds(); + if (StringUtils.isNotBlank(configurationIds)) { + String[] configurations = configurationIds.split(" "); + List confList = Arrays.asList(configurations); + return confList.contains(configurationId); + } + return false; + } + + /** + * Retrieve the virtual hardware section and its deployment options as configurations + */ + public OVFVirtualHardwareSectionTO getVirtualHardwareSectionFromDocument(Document doc) { + List configurations = getDeploymentOptionsFromDocumentTree(doc); + List items = getVirtualHardwareItemsFromDocumentTree(doc); + if (CollectionUtils.isNotEmpty(configurations)) { + for (OVFConfigurationTO configuration : configurations) { + List confItems = items.stream(). + filter(x -> StringUtils.isNotBlank(x.getConfigurationIds()) + && hardwareItemContainsConfiguration(x, configuration.getId())) + .collect(Collectors.toList()); + configuration.setHardwareItems(confItems); + } + } + List commonItems = null; + if (CollectionUtils.isNotEmpty(items)) { + commonItems = items.stream().filter(x -> StringUtils.isBlank(x.getConfigurationIds())).collect(Collectors.toList()); + } + return new OVFVirtualHardwareSectionTO(configurations, commonItems); + } + + private List getDeploymentOptionsFromDocumentTree(Document doc) { + List options = new ArrayList<>(); + if (doc == null) { + return options; + } + NodeList deploymentOptionSection = doc.getElementsByTagName("DeploymentOptionSection"); + if (deploymentOptionSection.getLength() == 0) { + return options; + } + Node hardwareSectionNode = deploymentOptionSection.item(0); + NodeList childNodes = hardwareSectionNode.getChildNodes(); + int index = 0; + for (int i = 0; i < childNodes.getLength(); i++) { + Node node = childNodes.item(i); + if (node != null && node.getNodeName().equals("Configuration")) { + Element configuration = (Element) node; + String configurationId = getNodeAttribute(configuration,"ovf","id"); + String description = getChildNodeValue(configuration, "Description"); + String label = getChildNodeValue(configuration, "Label"); + OVFConfigurationTO option = new OVFConfigurationTO(configurationId, label, description, index); + options.add(option); + index++; + } + } + return options; + } + + private List getVirtualHardwareItemsFromDocumentTree(Document doc) { + List items = new LinkedList<>(); + if (doc == null) { + return items; + } + NodeList hardwareSection = doc.getElementsByTagName("VirtualHardwareSection"); + if (hardwareSection.getLength() == 0) { + return items; + } + Node hardwareSectionNode = hardwareSection.item(0); + NodeList childNodes = hardwareSectionNode.getChildNodes(); + for (int i = 0; i < childNodes.getLength(); i++) { + Node node = childNodes.item(i); + if (node != null && node.getNodeName().equals("Item")) { + Element configuration = (Element) node; + String configurationIds = getNodeAttribute(configuration, "ovf", "configuration"); + String allocationUnits = getChildNodeValue(configuration, "AllocationUnits"); + String description = getChildNodeValue(configuration, "Description"); + String elementName = getChildNodeValue(configuration, "ElementName"); + String instanceID = getChildNodeValue(configuration, "InstanceID"); + String limit = getChildNodeValue(configuration, "Limit"); + String reservation = getChildNodeValue(configuration, "Reservation"); + String resourceType = getChildNodeValue(configuration, "ResourceType"); + String virtualQuantity = getChildNodeValue(configuration, "VirtualQuantity"); + String hostResource = getChildNodeValue(configuration, "HostResource"); + String addressOnParent = getChildNodeValue(configuration, "AddressOnParent"); + String parent = getChildNodeValue(configuration, "Parent"); + OVFVirtualHardwareItemTO item = new OVFVirtualHardwareItemTO(); + item.setConfigurationIds(configurationIds); + item.setAllocationUnits(allocationUnits); + item.setDescription(description); + item.setElementName(elementName); + item.setInstanceId(instanceID); + item.setLimit(getLongValueFromString(limit)); + item.setReservation(getLongValueFromString(reservation)); + Integer resType = getIntValueFromString(resourceType); + if (resType != null) { + item.setResourceType(OVFVirtualHardwareItemTO.getResourceTypeFromId(resType)); + } + item.setVirtualQuantity(getLongValueFromString(virtualQuantity)); + item.setHostResource(hostResource); + item.setAddressOnParent(addressOnParent); + item.setParent(parent); + items.add(item); + } + } + return items; + } + + private Long getLongValueFromString(String value) { + if (StringUtils.isNotBlank(value)) { + try { + return Long.parseLong(value); + } catch (NumberFormatException e) { + s_logger.debug("Could not parse the value: " + value + ", ignoring it"); + } + } + return null; + } + + private Integer getIntValueFromString(String value) { + if (StringUtils.isNotBlank(value)) { + try { + return Integer.parseInt(value); + } catch (NumberFormatException e) { + s_logger.debug("Could not parse the value: " + value + ", ignoring it"); + } + } + return null; + } + + protected byte[] compressOVFEula(String license) throws IOException { + CompressionUtil compressionUtil = new CompressionUtil(); + return compressionUtil.compressString(license); + } + + public List getEulaSectionsFromDocument(Document doc) { + List eulas = new LinkedList<>(); + if (doc == null) { + return eulas; + } + NodeList eulaSections = doc.getElementsByTagName("EulaSection"); + int eulaIndex = 0; + if (eulaSections.getLength() > 0) { + for (int index = 0; index < eulaSections.getLength(); index++) { + Node eulaNode = eulaSections.item(index); + NodeList eulaChildNodes = eulaNode.getChildNodes(); + String eulaInfo = null; + String eulaLicense = null; + for (int i = 0; i < eulaChildNodes.getLength(); i++) { + Node eulaItem = eulaChildNodes.item(i); + if (eulaItem.getNodeName().equalsIgnoreCase("Info")) { + eulaInfo = eulaItem.getTextContent(); + } else if (eulaItem.getNodeName().equalsIgnoreCase("License")) { + eulaLicense = eulaItem.getTextContent(); + } + } + byte[] compressedLicense = new byte[0]; + try { + compressedLicense = compressOVFEula(eulaLicense); + } catch (IOException e) { + s_logger.error("Could not compress the license for info " + eulaInfo); + continue; + } + OVFEulaSectionTO eula = new OVFEulaSectionTO(eulaInfo, compressedLicense, eulaIndex); + eulas.add(eula); + eulaIndex++; + } + } + + return eulas; + } + class OVFFile { // public String _href; @@ -417,7 +882,6 @@ class OVFDisk { // public Long _capacity; - public String _capacityUnit; public String _diskId; public String _fileRef; public Long _populatedSize; diff --git a/api/src/main/java/com/cloud/agent/api/to/DatadiskTO.java b/api/src/main/java/com/cloud/agent/api/to/DatadiskTO.java index 1d3f91e25dbe..31a02b0bec33 100644 --- a/api/src/main/java/com/cloud/agent/api/to/DatadiskTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/DatadiskTO.java @@ -27,18 +27,14 @@ public class DatadiskTO { private boolean isIso; private String diskController; private String diskControllerSubType; + private int diskNumber; + private String configuration; public DatadiskTO() { } - public DatadiskTO(String path, long virtualSize, long fileSize, boolean bootable) { - this.path = path; - this.virtualSize = virtualSize; - this.fileSize = fileSize; - this.bootable = bootable; - } - - public DatadiskTO(String path, long virtualSize, long fileSize, String diskId, boolean isIso, boolean bootable, String controller, String controllerSubType) { + public DatadiskTO(String path, long virtualSize, long fileSize, String diskId, boolean isIso, boolean bootable, + String controller, String controllerSubType, int diskNumber, String configuration) { this.path = path; this.virtualSize = virtualSize; this.fileSize = fileSize; @@ -47,6 +43,8 @@ public DatadiskTO(String path, long virtualSize, long fileSize, String diskId, b this.isIso = isIso; this.diskController = controller; this.diskControllerSubType = controllerSubType; + this.diskNumber = diskNumber; + this.configuration = configuration; } public String getPath() { @@ -105,4 +103,11 @@ public void setDiskControllerSubType(String diskControllerSubType) { this.diskControllerSubType = diskControllerSubType; } + public int getDiskNumber() { + return this.diskNumber; + } + + public String getConfiguration() { + return configuration; + } } \ No newline at end of file diff --git a/api/src/main/java/com/cloud/agent/api/to/DeployAsIsInfoTO.java b/api/src/main/java/com/cloud/agent/api/to/DeployAsIsInfoTO.java new file mode 100644 index 000000000000..d82a2975eaf3 --- /dev/null +++ b/api/src/main/java/com/cloud/agent/api/to/DeployAsIsInfoTO.java @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.api.to; + +import com.cloud.agent.api.LogLevel; + +import java.util.HashMap; +import java.util.Map; + +public class DeployAsIsInfoTO { + + private String templatePath; + private String destStoragePool; + @LogLevel(LogLevel.Log4jLevel.Off) + private Map properties = new HashMap<>(); + private Map nicAdapterMap = new HashMap(); + + public DeployAsIsInfoTO() { + } + + public DeployAsIsInfoTO(String templatePath, String destStoragePool, Map properties, + Map nicAdapterMap) { + this.templatePath = templatePath; + this.destStoragePool = destStoragePool; + this.properties = properties; + this.nicAdapterMap = nicAdapterMap; + } + + public String getTemplatePath() { + return templatePath; + } + + public Map getProperties() { + return properties; + } + + public String getDestStoragePool() { + return destStoragePool; + } + + public Map getNicAdapterMap() { + return nicAdapterMap; + } +} diff --git a/api/src/main/java/com/cloud/agent/api/to/VirtualMachineTO.java b/api/src/main/java/com/cloud/agent/api/to/VirtualMachineTO.java index dceacf0e65bc..efc735ccecf0 100644 --- a/api/src/main/java/com/cloud/agent/api/to/VirtualMachineTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/VirtualMachineTO.java @@ -20,10 +20,7 @@ import java.util.Map; import java.util.HashMap; -import com.cloud.agent.api.LogLevel; -import com.cloud.agent.api.storage.OVFPropertyTO; import com.cloud.template.VirtualMachineTemplate.BootloaderType; -import com.cloud.utils.Pair; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachine.Type; @@ -81,8 +78,7 @@ public class VirtualMachineTO { Map guestOsDetails = new HashMap(); Map extraConfig = new HashMap<>(); - @LogLevel(LogLevel.Log4jLevel.Off) - Pair> ovfProperties; + DeployAsIsInfoTO deployAsIsInfo; public VirtualMachineTO(long id, String instanceName, VirtualMachine.Type type, int cpus, Integer speed, long minRam, long maxRam, BootloaderType bootloader, String os, boolean enableHA, boolean limitCpuUse, String vncPassword) { @@ -376,13 +372,6 @@ public Map getExtraConfig() { return extraConfig; } - public Pair> getOvfProperties() { - return ovfProperties; - } - - public void setOvfProperties(Pair> ovfProperties) { - this.ovfProperties = ovfProperties; - } public String getBootType() { return bootType; } @@ -402,4 +391,12 @@ public boolean isEnterHardwareSetup() { public void setEnterHardwareSetup(boolean enterHardwareSetup) { this.enterHardwareSetup = enterHardwareSetup; } + + public DeployAsIsInfoTO getDeployAsIsInfo() { + return deployAsIsInfo; + } + + public void setDeployAsIsInfo(DeployAsIsInfoTO deployAsIsInfo) { + this.deployAsIsInfo = deployAsIsInfo; + } } diff --git a/api/src/main/java/com/cloud/agent/api/to/deployasis/OVFConfigurationTO.java b/api/src/main/java/com/cloud/agent/api/to/deployasis/OVFConfigurationTO.java new file mode 100644 index 000000000000..f3cb75089ab1 --- /dev/null +++ b/api/src/main/java/com/cloud/agent/api/to/deployasis/OVFConfigurationTO.java @@ -0,0 +1,61 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package com.cloud.agent.api.to.deployasis; + +import java.util.List; + +public class OVFConfigurationTO implements TemplateDeployAsIsInformationTO { + + private final String id; + private final String label; + private final String description; + private List hardwareItems; + private int index; + + public OVFConfigurationTO(String id, String label, String description, int index) { + this.id = id; + this.label = label; + this.description = description; + this.index = index; + } + + public String getId() { + return id; + } + + public String getLabel() { + return label; + } + + public String getDescription() { + return description; + } + + public void setHardwareItems(List items) { + this.hardwareItems = items; + } + + public List getHardwareItems() { + return hardwareItems; + } + + public int getIndex() { + return index; + } +} \ No newline at end of file diff --git a/api/src/main/java/com/cloud/agent/api/to/deployasis/OVFEulaSectionTO.java b/api/src/main/java/com/cloud/agent/api/to/deployasis/OVFEulaSectionTO.java new file mode 100644 index 000000000000..893661782f46 --- /dev/null +++ b/api/src/main/java/com/cloud/agent/api/to/deployasis/OVFEulaSectionTO.java @@ -0,0 +1,49 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package com.cloud.agent.api.to.deployasis; + +import com.cloud.agent.api.LogLevel; + +/** + * End-user licence agreement + */ +public class OVFEulaSectionTO implements TemplateDeployAsIsInformationTO { + private String info; + @LogLevel(LogLevel.Log4jLevel.Off) + private byte[] compressedLicense; + private int index; + + public OVFEulaSectionTO(String info, byte[] license, int eulaIndex) { + this.info = info; + this.compressedLicense = license; + this.index = eulaIndex; + } + + public String getInfo() { + return this.info; + } + + public byte[] getCompressedLicense() { + return this.compressedLicense; + } + + public int getIndex() { + return index; + } +} diff --git a/api/src/main/java/com/cloud/agent/api/to/deployasis/OVFNetworkTO.java b/api/src/main/java/com/cloud/agent/api/to/deployasis/OVFNetworkTO.java new file mode 100644 index 000000000000..9b05dbc18636 --- /dev/null +++ b/api/src/main/java/com/cloud/agent/api/to/deployasis/OVFNetworkTO.java @@ -0,0 +1,124 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.api.to.deployasis; + +/** + * container for the network prerequisites as found in the appliance template + * + * for OVA: + * {code} + * + * Management Network Interface + * + * {code} + * {code} + * + * 7 + * true + * Management0-0 + * E1000 Ethernet adapter on "Management Network" + * Network adapter 1 + * 6 + * E1000 + * 10 + * + * {code} + */ +public class OVFNetworkTO implements TemplateDeployAsIsInformationTO { + String name; + String networkDescription; + + int addressOnParent; + boolean automaticAllocation; + String nicDescription; + String elementName; + int InstanceID; + String resourceSubType; + String resourceType; + + public int getAddressOnParent() { + return addressOnParent; + } + + public void setAddressOnParent(int addressOnParent) { + this.addressOnParent = addressOnParent; + } + + public boolean isAutomaticAllocation() { + return automaticAllocation; + } + + public void setAutomaticAllocation(boolean automaticAllocation) { + this.automaticAllocation = automaticAllocation; + } + + public String getNicDescription() { + return nicDescription; + } + + public void setNicDescription(String nicDescription) { + this.nicDescription = nicDescription; + } + + public String getElementName() { + return elementName; + } + + public void setElementName(String elementName) { + this.elementName = elementName; + } + + public int getInstanceID() { + return InstanceID; + } + + public void setInstanceID(int instanceID) { + InstanceID = instanceID; + } + + public String getResourceSubType() { + return resourceSubType; + } + + public void setResourceSubType(String resourceSubType) { + this.resourceSubType = resourceSubType; + } + + public String getResourceType() { + return resourceType; + } + + public void setResourceType(String resourceType) { + this.resourceType = resourceType; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getNetworkDescription() { + return networkDescription; + } + + public void setNetworkDescription(String networkDescription) { + this.networkDescription = networkDescription; + } +} \ No newline at end of file diff --git a/api/src/main/java/com/cloud/agent/api/storage/OVFPropertyTO.java b/api/src/main/java/com/cloud/agent/api/to/deployasis/OVFPropertyTO.java similarity index 90% rename from api/src/main/java/com/cloud/agent/api/storage/OVFPropertyTO.java rename to api/src/main/java/com/cloud/agent/api/to/deployasis/OVFPropertyTO.java index abf743ae713d..32c625592626 100644 --- a/api/src/main/java/com/cloud/agent/api/storage/OVFPropertyTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/deployasis/OVFPropertyTO.java @@ -17,7 +17,7 @@ // under the License. // -package com.cloud.agent.api.storage; +package com.cloud.agent.api.to.deployasis; import com.cloud.agent.api.LogLevel; @@ -30,7 +30,7 @@ * Choose "Remote HTTP and SSH Client Routes" to route only traffic destined for the management client(s), when they are on remote networks. * */ -public class OVFPropertyTO implements OVFProperty { +public class OVFPropertyTO implements TemplateDeployAsIsInformationTO { private String key; private String type; @@ -41,18 +41,14 @@ public class OVFPropertyTO implements OVFProperty { private String label; private String description; private Boolean password; + private int index; + private String category; public OVFPropertyTO() { } - public OVFPropertyTO(String key, String value, boolean password) { - this.key = key; - this.value = value; - this.password = password; - } - public OVFPropertyTO(String key, String type, String value, String qualifiers, boolean userConfigurable, - String label, String description, boolean password) { + String label, String description, boolean password, int index, String category) { this.key = key; this.type = type; this.value = value; @@ -61,9 +57,10 @@ public OVFPropertyTO(String key, String type, String value, String qualifiers, b this.label = label; this.description = description; this.password = password; + this.index = index; + this.category = category; } - @Override public Long getTemplateId() { return null; } @@ -131,4 +128,12 @@ public Boolean isPassword() { public void setPassword(Boolean password) { this.password = password; } + + public String getCategory() { + return category; + } + + public int getIndex() { + return index; + } } diff --git a/api/src/main/java/com/cloud/agent/api/to/deployasis/OVFVirtualHardwareItemTO.java b/api/src/main/java/com/cloud/agent/api/to/deployasis/OVFVirtualHardwareItemTO.java new file mode 100644 index 000000000000..525775066536 --- /dev/null +++ b/api/src/main/java/com/cloud/agent/api/to/deployasis/OVFVirtualHardwareItemTO.java @@ -0,0 +1,365 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.api.to.deployasis; + +// From: https://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData.xsd +public class OVFVirtualHardwareItemTO implements TemplateDeployAsIsInformationTO{ + + //From: https://schemas.dmtf.org/wbem/cim-html/2/CIM_ResourceAllocationSettingData.html + public enum HardwareResourceType { + Other("Other", 1), + ComputerSystem ("Computer System", 2), + Processor("Processor", 3), + Memory("Memory", 4), + IDEController("IDE Controller", 5), + ParallelSCSIHBA("Parallel SCSI HBA", 6), + FC_HBA("FC HBA", 7), + iSCSI_HBA("iSCSI HBA", 8), + IB_HCA("IB HCA", 9), + EthernetAdapter("Ethernet Adaptor", 10), + OtherNetworkAdapter("Other Network Adaptor", 11), + IO_Slot("I/O Slot", 12), + IO_Device("I/O Device", 13), + FloppyDrive("Floppy Drive", 14), + CD_Drive("CD Drive", 15), + DVD_Drive("DVD Drive", 16), + DiskDrive("Disk Drive", 17), + TapeDrive("Tape Drive", 18), + StorageExtent("Storage Extent", 19), + OtherStorageDevice("Other Storage Device", 20), + SerialPort("Serial Port", 21), + ParallelPort("Parallel Port", 22), + USBController("USB Controller", 23), + GraphicsController("Graphics Controller", 24), + IEEE_1394_Controller("IEEE 1394 Controller", 25), + PartitionableUnit("Partitionable Unit", 26), + BasePartitionableUnit("base Partitionable Unit", 27), + PowerSupply("Power", 28), + CoolingCapacity("Cooling Capacity", 29), + EthernetSwitchPort("Ethernet Switch Port", 30), + LogicalDisk("Logical Disk", 31), + StorageVolume("Storage Volume", 32), + EthernetConnection("Ethernet Connection", 33), + DMTF_reserved("DMTF Reserved", 35), + VendorReserved("Vendor Reserved", 32768); + + private String name; + private int id; + + HardwareResourceType(String name, int id) { + this.name = name; + this.id = id; + } + + public String getName() { + return name; + } + } + + public static HardwareResourceType getResourceTypeFromId(int id) { + if (id <= 33) { + for (HardwareResourceType type : HardwareResourceType.values()) { + if (type.id == id) { + return type; + } + } + } else if (id <= 32767) { + return HardwareResourceType.DMTF_reserved; + } + return HardwareResourceType.VendorReserved; + } + + public enum CustomerVisibility { + Unknown, PassedThrough, Virtualized, NotRepresented, DMTFReserved, VendorReserved; + } + + public enum MappingBehavior { + Unknown, NotSupported, Dedicated, SoftAffinity, HardAffinity, DMTFReserved, VendorReserved; + } + + private String address; + private String addressOnParent; + private String allocationUnits; + private boolean automaticAllocation; + private boolean automaticDeallocation; + private String caption; + private String changeableType; + private String componentSetting; + private String configurationName; + private String connection; + private CustomerVisibility customerVisibility; + private String description; + private String elementName; + private Long generation; + private String hostResource; + private String instanceId; + private Long limit; + private MappingBehavior mappingBehavior; + private String otherResourceType; + private String parent; + private String poolId; + private Long reservation; + private String resourceSubtype; + private HardwareResourceType resourceType; + private String soId; + private String soOrgId; + private Long virtualQuantity; + private String virtualQuantityUnits; + private int weight; + + private String configurationIds; + + public String getConfigurationIds() { + return configurationIds; + } + + public void setConfigurationIds(String configurationIds) { + this.configurationIds = configurationIds; + } + + public String getAddress() { + return address; + } + + public void setAddress(String address) { + this.address = address; + } + + public String getAddressOnParent() { + return addressOnParent; + } + + public void setAddressOnParent(String addressOnParent) { + this.addressOnParent = addressOnParent; + } + + public String getAllocationUnits() { + return allocationUnits; + } + + public void setAllocationUnits(String allocationUnits) { + this.allocationUnits = allocationUnits; + } + + public boolean isAutomaticAllocation() { + return automaticAllocation; + } + + public void setAutomaticAllocation(boolean automaticAllocation) { + this.automaticAllocation = automaticAllocation; + } + + public boolean isAutomaticDeallocation() { + return automaticDeallocation; + } + + public void setAutomaticDeallocation(boolean automaticDeallocation) { + this.automaticDeallocation = automaticDeallocation; + } + + public String getCaption() { + return caption; + } + + public void setCaption(String caption) { + this.caption = caption; + } + + public String getChangeableType() { + return changeableType; + } + + public void setChangeableType(String changeableType) { + this.changeableType = changeableType; + } + + public String getComponentSetting() { + return componentSetting; + } + + public void setComponentSetting(String componentSetting) { + this.componentSetting = componentSetting; + } + + public String getConfigurationName() { + return configurationName; + } + + public void setConfigurationName(String configurationName) { + this.configurationName = configurationName; + } + + public String getConnection() { + return connection; + } + + public void setConnection(String connection) { + this.connection = connection; + } + + public CustomerVisibility getCustomerVisibility() { + return customerVisibility; + } + + public void setCustomerVisibility(CustomerVisibility customerVisibility) { + this.customerVisibility = customerVisibility; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getElementName() { + return elementName; + } + + public void setElementName(String elementName) { + this.elementName = elementName; + } + + public Long getGeneration() { + return generation; + } + + public void setGeneration(Long generation) { + this.generation = generation; + } + + public String getHostResource() { + return hostResource; + } + + public void setHostResource(String hostResource) { + this.hostResource = hostResource; + } + + public String getInstanceId() { + return instanceId; + } + + public void setInstanceId(String instanceId) { + this.instanceId = instanceId; + } + + public Long getLimit() { + return limit; + } + + public void setLimit(Long limit) { + this.limit = limit; + } + + public MappingBehavior getMappingBehavior() { + return mappingBehavior; + } + + public void setMappingBehavior(MappingBehavior mappingBehavior) { + this.mappingBehavior = mappingBehavior; + } + + public String getOtherResourceType() { + return otherResourceType; + } + + public void setOtherResourceType(String otherResourceType) { + this.otherResourceType = otherResourceType; + } + + public String getParent() { + return parent; + } + + public void setParent(String parent) { + this.parent = parent; + } + + public String getPoolId() { + return poolId; + } + + public void setPoolId(String poolId) { + this.poolId = poolId; + } + + public Long getReservation() { + return reservation; + } + + public void setReservation(Long reservation) { + this.reservation = reservation; + } + + public String getResourceSubtype() { + return resourceSubtype; + } + + public void setResourceSubtype(String resourceSubtype) { + this.resourceSubtype = resourceSubtype; + } + + public HardwareResourceType getResourceType() { + return resourceType; + } + + public void setResourceType(HardwareResourceType resourceType) { + this.resourceType = resourceType; + } + + public String getSoId() { + return soId; + } + + public void setSoId(String soId) { + this.soId = soId; + } + + public String getSoOrgId() { + return soOrgId; + } + + public void setSoOrgId(String soOrgId) { + this.soOrgId = soOrgId; + } + + public Long getVirtualQuantity() { + return virtualQuantity; + } + + public void setVirtualQuantity(Long virtualQuantity) { + this.virtualQuantity = virtualQuantity; + } + + public String getVirtualQuantityUnits() { + return virtualQuantityUnits; + } + + public void setVirtualQuantityUnits(String virtualQuantityUnits) { + this.virtualQuantityUnits = virtualQuantityUnits; + } + + public int getWeight() { + return weight; + } + + public void setWeight(int weight) { + this.weight = weight; + } +} \ No newline at end of file diff --git a/api/src/main/java/com/cloud/agent/api/to/deployasis/OVFVirtualHardwareSectionTO.java b/api/src/main/java/com/cloud/agent/api/to/deployasis/OVFVirtualHardwareSectionTO.java new file mode 100644 index 000000000000..2698f8c5e189 --- /dev/null +++ b/api/src/main/java/com/cloud/agent/api/to/deployasis/OVFVirtualHardwareSectionTO.java @@ -0,0 +1,43 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package com.cloud.agent.api.to.deployasis; + +import java.util.List; + +public class OVFVirtualHardwareSectionTO implements TemplateDeployAsIsInformationTO { + + public OVFVirtualHardwareSectionTO() { + } + + private List configurations; + private List commonHardwareItems; + + public OVFVirtualHardwareSectionTO(List configurations, List commonHardwareItems) { + this.configurations = configurations; + this.commonHardwareItems = commonHardwareItems; + } + + public List getConfigurations() { + return configurations; + } + + public List getCommonHardwareItems() { + return commonHardwareItems; + } +} \ No newline at end of file diff --git a/api/src/main/java/com/cloud/agent/api/to/deployasis/TemplateDeployAsIsInformationTO.java b/api/src/main/java/com/cloud/agent/api/to/deployasis/TemplateDeployAsIsInformationTO.java new file mode 100644 index 000000000000..9080b92f5946 --- /dev/null +++ b/api/src/main/java/com/cloud/agent/api/to/deployasis/TemplateDeployAsIsInformationTO.java @@ -0,0 +1,24 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package com.cloud.agent.api.to.deployasis; + +import java.io.Serializable; + +public interface TemplateDeployAsIsInformationTO extends Serializable { +} diff --git a/api/src/main/java/com/cloud/agent/api/storage/OVFProperty.java b/api/src/main/java/com/cloud/dc/VsphereStoragePolicy.java similarity index 74% rename from api/src/main/java/com/cloud/agent/api/storage/OVFProperty.java rename to api/src/main/java/com/cloud/dc/VsphereStoragePolicy.java index ac9ae7721b0f..ca0ed5447003 100644 --- a/api/src/main/java/com/cloud/agent/api/storage/OVFProperty.java +++ b/api/src/main/java/com/cloud/dc/VsphereStoragePolicy.java @@ -1,4 +1,3 @@ -// // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information @@ -15,19 +14,18 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -// +package com.cloud.dc; + +import org.apache.cloudstack.api.Identity; +import org.apache.cloudstack.api.InternalIdentity; + +public interface VsphereStoragePolicy extends Identity, InternalIdentity { + + long getZoneId(); -package com.cloud.agent.api.storage; + String getPolicyId(); -public interface OVFProperty { + String getName(); - Long getTemplateId(); - String getKey(); - String getType(); - String getValue(); - String getQualifiers(); - Boolean isUserConfigurable(); - String getLabel(); String getDescription(); - Boolean isPassword(); -} \ No newline at end of file +} diff --git a/api/src/main/java/com/cloud/deployasis/DeployAsIsConstants.java b/api/src/main/java/com/cloud/deployasis/DeployAsIsConstants.java new file mode 100644 index 000000000000..3dc961b9de85 --- /dev/null +++ b/api/src/main/java/com/cloud/deployasis/DeployAsIsConstants.java @@ -0,0 +1,27 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.deployasis; + +public interface DeployAsIsConstants { + + String PROPERTY_PREFIX = "property-"; + String NETWORK_PREFIX = "network-"; + String CONFIGURATION_PREFIX = "configuration-"; + String HARDWARE_ITEM_PREFIX = "hardware-item-"; + String EULA_PREFIX = "eula-"; + +} diff --git a/api/src/main/java/com/cloud/event/EventTypes.java b/api/src/main/java/com/cloud/event/EventTypes.java index d14719247c17..852198b91a7a 100644 --- a/api/src/main/java/com/cloud/event/EventTypes.java +++ b/api/src/main/java/com/cloud/event/EventTypes.java @@ -70,6 +70,7 @@ import com.cloud.server.ResourceTag; import com.cloud.storage.GuestOS; import com.cloud.storage.GuestOSHypervisor; +import com.cloud.storage.ImageStore; import com.cloud.storage.Snapshot; import com.cloud.storage.StoragePool; import com.cloud.storage.Volume; @@ -239,6 +240,7 @@ public class EventTypes { public static final String EVENT_TEMPLATE_EXTRACT = "TEMPLATE.EXTRACT"; public static final String EVENT_TEMPLATE_UPLOAD = "TEMPLATE.UPLOAD"; public static final String EVENT_TEMPLATE_CLEANUP = "TEMPLATE.CLEANUP"; + public static final String EVENT_FILE_MIGRATE = "FILE.MIGRATE"; // Volume Events public static final String EVENT_VOLUME_CREATE = "VOLUME.CREATE"; @@ -329,6 +331,8 @@ public class EventTypes { public static final String EVENT_STORAGE_IP_RANGE_DELETE = "STORAGE.IP.RANGE.DELETE"; public static final String EVENT_STORAGE_IP_RANGE_UPDATE = "STORAGE.IP.RANGE.UPDATE"; + public static final String EVENT_IMAGE_STORE_DATA_MIGRATE = "IMAGE.STORE.MIGRATE.DATA"; + // Configuration Table public static final String EVENT_CONFIGURATION_VALUE_EDIT = "CONFIGURATION.VALUE.EDIT"; @@ -615,6 +619,9 @@ public class EventTypes { public static final String EVENT_POD_ROLLING_MAINTENANCE = "POD.ROLLING.MAINTENANCE"; public static final String EVENT_ZONE_ROLLING_MAINTENANCE = "ZONE.ROLLING.MAINTENANCE"; + // Storage Policies + public static final String EVENT_IMPORT_VCENTER_STORAGE_POLICIES = "IMPORT.VCENTER.STORAGE.POLICIES"; + static { // TODO: need a way to force author adding event types to declare the entity details as well, with out braking @@ -1021,6 +1028,10 @@ public class EventTypes { entityEventDetails.put(EVENT_POD_ROLLING_MAINTENANCE, PodResponse.class); entityEventDetails.put(EVENT_CLUSTER_ROLLING_MAINTENANCE, ClusterResponse.class); entityEventDetails.put(EVENT_HOST_ROLLING_MAINTENANCE, HostResponse.class); + + entityEventDetails.put(EVENT_IMPORT_VCENTER_STORAGE_POLICIES, "StoragePolicies"); + + entityEventDetails.put(EVENT_IMAGE_STORE_DATA_MIGRATE, ImageStore.class); } public static String getEntityForEvent(String eventName) { diff --git a/api/src/main/java/com/cloud/storage/Storage.java b/api/src/main/java/com/cloud/storage/Storage.java index 82bc5f6d4e5a..7a229b676746 100644 --- a/api/src/main/java/com/cloud/storage/Storage.java +++ b/api/src/main/java/com/cloud/storage/Storage.java @@ -16,11 +16,11 @@ // under the License. package com.cloud.storage; +import org.apache.commons.lang.NotImplementedException; + import java.util.ArrayList; import java.util.List; -import org.apache.commons.lang.NotImplementedException; - public class Storage { public static enum ImageFormat { QCOW2(true, true, false, "qcow2"), @@ -135,7 +135,8 @@ public static enum StoragePoolType { OCFS2(true, false), SMB(true, false), Gluster(true, false), - ManagedNFS(true, false); + ManagedNFS(true, false), + DatastoreCluster(true, true); // for VMware, to abstract pool of clusters private final boolean shared; private final boolean overprovisioning; diff --git a/api/src/main/java/com/cloud/storage/StoragePool.java b/api/src/main/java/com/cloud/storage/StoragePool.java index 3a2d3bd8feec..6c65ca17ac92 100644 --- a/api/src/main/java/com/cloud/storage/StoragePool.java +++ b/api/src/main/java/com/cloud/storage/StoragePool.java @@ -106,4 +106,6 @@ public interface StoragePool extends Identity, InternalIdentity { Hypervisor.HypervisorType getHypervisor(); boolean isManaged(); + + Long getParent(); } diff --git a/api/src/main/java/com/cloud/storage/StorageService.java b/api/src/main/java/com/cloud/storage/StorageService.java index aebbbcd4bd04..207fc8f0cd7a 100644 --- a/api/src/main/java/com/cloud/storage/StorageService.java +++ b/api/src/main/java/com/cloud/storage/StorageService.java @@ -102,4 +102,6 @@ public interface StorageService { */ ImageStore migrateToObjectStore(String name, String url, String providerName, Map details) throws DiscoveryException; + ImageStore updateImageStoreStatus(Long id, Boolean readonly); + } diff --git a/api/src/main/java/com/cloud/storage/Volume.java b/api/src/main/java/com/cloud/storage/Volume.java index dde9d60a8482..5fd78efb307e 100644 --- a/api/src/main/java/com/cloud/storage/Volume.java +++ b/api/src/main/java/com/cloud/storage/Volume.java @@ -84,6 +84,8 @@ public String getDescription() { s_fsm.addTransition(new StateMachine2.Transition(Resizing, Event.OperationFailed, Ready, null)); s_fsm.addTransition(new StateMachine2.Transition(Allocated, Event.UploadRequested, UploadOp, null)); s_fsm.addTransition(new StateMachine2.Transition(Uploaded, Event.CopyRequested, Copying, null)); + s_fsm.addTransition(new StateMachine2.Transition(Ready, Event.OperationSucceeded, Ready, null)); + s_fsm.addTransition(new StateMachine2.Transition(Ready, Event.OperationFailed, Ready, null)); s_fsm.addTransition(new StateMachine2.Transition(Copying, Event.OperationSucceeded, Ready, Arrays.asList(new StateMachine2.Transition.Impact[]{StateMachine2.Transition.Impact.USAGE}))); s_fsm.addTransition(new StateMachine2.Transition(Copying, Event.OperationFailed, Uploaded, null)); s_fsm.addTransition(new StateMachine2.Transition(UploadOp, Event.DestroyRequested, Destroy, null)); diff --git a/api/src/main/java/com/cloud/template/VirtualMachineTemplate.java b/api/src/main/java/com/cloud/template/VirtualMachineTemplate.java index 5177e51d4015..95d1ebf0b87a 100644 --- a/api/src/main/java/com/cloud/template/VirtualMachineTemplate.java +++ b/api/src/main/java/com/cloud/template/VirtualMachineTemplate.java @@ -138,4 +138,6 @@ public enum TemplateFilter { void incrUpdatedCount(); Date getUpdated(); + + boolean isDeployAsIs(); } diff --git a/api/src/main/java/com/cloud/vm/DiskProfile.java b/api/src/main/java/com/cloud/vm/DiskProfile.java index 2b76c682ba35..175a92afaf9d 100644 --- a/api/src/main/java/com/cloud/vm/DiskProfile.java +++ b/api/src/main/java/com/cloud/vm/DiskProfile.java @@ -100,6 +100,10 @@ public String getName() { return name; } + public void setTags(String[] tags) { + this.tags = tags; + } + /** * @return tags for the disk. This can be used to match it to different storage pools. */ diff --git a/api/src/main/java/com/cloud/vm/NicProfile.java b/api/src/main/java/com/cloud/vm/NicProfile.java index 47021c85108c..cd2215cf715d 100644 --- a/api/src/main/java/com/cloud/vm/NicProfile.java +++ b/api/src/main/java/com/cloud/vm/NicProfile.java @@ -52,6 +52,8 @@ public class NicProfile implements InternalIdentity, Serializable { Integer networkRate; boolean isSecurityGroupEnabled; + Integer orderIndex; + // IPv4 String iPv4Address; String iPv4Netmask; @@ -381,6 +383,14 @@ public void setRequestedIPv6(String requestedIPv6) { this.requestedIPv6 = requestedIPv6; } + public Integer getOrderIndex() { + return orderIndex; + } + + public void setOrderIndex(Integer orderIndex) { + this.orderIndex = orderIndex; + } + // // OTHER METHODS // @@ -410,6 +420,8 @@ public void deallocate() { broadcastUri = null; isolationUri = null; + orderIndex = null; + } @Override diff --git a/api/src/main/java/com/cloud/vm/VmDetailConstants.java b/api/src/main/java/com/cloud/vm/VmDetailConstants.java index 3812aa211445..9991e1f35b4f 100644 --- a/api/src/main/java/com/cloud/vm/VmDetailConstants.java +++ b/api/src/main/java/com/cloud/vm/VmDetailConstants.java @@ -63,4 +63,6 @@ public interface VmDetailConstants { String IP6_ADDRESS = "ip6Address"; String DISK = "disk"; String DISK_OFFERING = "diskOffering"; + + String DEPLOY_AS_IS_CONFIGURATION = "configurationId"; } diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index df284552f5d4..ba1f17683954 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -114,6 +114,7 @@ public class ApiConstants { public static final String DISK_IO_WRITE = "diskiowrite"; public static final String DISK_IO_PSTOTAL = "diskiopstotal"; public static final String DISK_SIZE = "disksize"; + public static final String DOWNLOAD_DETAILS = "downloaddetails"; public static final String UTILIZATION = "utilization"; public static final String DRIVER = "driver"; public static final String ROOT_DISK_SIZE = "rootdisksize"; @@ -184,6 +185,7 @@ public class ApiConstants { public static final String ICMP_TYPE = "icmptype"; public static final String ID = "id"; public static final String IDS = "ids"; + public static final String INDEX = "index"; public static final String PREVIOUS_ACL_RULE_ID = "previousaclruleid"; public static final String NEXT_ACL_RULE_ID = "nextaclruleid"; public static final String MOVE_ACL_CONSISTENCY_HASH = "aclconsistencyhash"; @@ -235,6 +237,7 @@ public class ApiConstants { public static final String MAX_MEMORY = "maxmemory"; public static final String MIN_CPU_NUMBER = "mincpunumber"; public static final String MIN_MEMORY = "minmemory"; + public static final String MIGRATION_TYPE = "migrationtype"; public static final String MEMORY = "memory"; public static final String MODE = "mode"; public static final String KEEPALIVE_ENABLED = "keepaliveenabled"; @@ -260,7 +263,7 @@ public class ApiConstants { public static final String OUTOFBANDMANAGEMENT_POWERSTATE = "outofbandmanagementpowerstate"; public static final String OUTOFBANDMANAGEMENT_ENABLED = "outofbandmanagementenabled"; public static final String OUTPUT = "output"; - public static final String OVF_PROPERTIES = "ovfproperties"; + public static final String PROPERTIES = "properties"; public static final String PARAMS = "params"; public static final String PARENT_ID = "parentid"; public static final String PARENT_DOMAIN_ID = "parentdomainid"; @@ -355,6 +358,7 @@ public class ApiConstants { public static final String TARGET_IQN = "targetiqn"; public static final String TEMPLATE_FILTER = "templatefilter"; public static final String TEMPLATE_ID = "templateid"; + public static final String TEMPLATE_IDS = "templateids"; public static final String TEMPLATE_NAME = "templatename"; public static final String ISO_ID = "isoid"; public static final String TIMEOUT = "timeout"; @@ -789,6 +793,8 @@ public class ApiConstants { public static final String EXITCODE = "exitcode"; public static final String TARGET_ID = "targetid"; public static final String FILES = "files"; + public static final String SRC_POOL = "srcpool"; + public static final String DEST_POOLS = "destpools"; public static final String VOLUME_IDS = "volumeids"; public static final String ROUTER_ID = "routerid"; @@ -819,6 +825,11 @@ public class ApiConstants { public static final String BOOT_TYPE = "boottype"; public static final String BOOT_MODE = "bootmode"; public static final String BOOT_INTO_SETUP = "bootintosetup"; + public static final String DEPLOY_AS_IS = "deployasis"; + public static final String DEPLOY_AS_IS_DETAILS = "deployasisdetails"; + public static final String CROSS_ZONES = "crossZones"; + public static final String TEMPLATETYPE = "templatetype"; + public static final String SOURCETEMPLATEID = "sourcetemplateid"; public enum BootType { UEFI, BIOS; diff --git a/api/src/main/java/org/apache/cloudstack/api/BaseCmd.java b/api/src/main/java/org/apache/cloudstack/api/BaseCmd.java index 5b4c7aa3ef8b..c897aad4d4b4 100644 --- a/api/src/main/java/org/apache/cloudstack/api/BaseCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/BaseCmd.java @@ -40,6 +40,7 @@ import org.apache.cloudstack.network.lb.ApplicationLoadBalancerService; import org.apache.cloudstack.network.lb.InternalLoadBalancerVMService; import org.apache.cloudstack.query.QueryService; +import org.apache.cloudstack.storage.ImageStoreService; import org.apache.cloudstack.usage.UsageService; import org.apache.log4j.Logger; @@ -131,6 +132,8 @@ public static enum CommandType { @Inject public TemplateApiService _templateService; @Inject + public ImageStoreService _imageStoreService; + @Inject public SecurityGroupService _securityGroupService; @Inject public SnapshotApiService _snapshotService; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/CreateDiskOfferingCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/CreateDiskOfferingCmd.java index f0ca5fb851a1..a830777031f7 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/CreateDiskOfferingCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/CreateDiskOfferingCmd.java @@ -28,6 +28,7 @@ import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.response.DiskOfferingResponse; import org.apache.cloudstack.api.response.DomainResponse; +import org.apache.cloudstack.api.response.VsphereStoragePoliciesResponse; import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; @@ -151,6 +152,9 @@ public class CreateDiskOfferingCmd extends BaseCmd { since = "4.14") private String cacheMode; + @Parameter(name = ApiConstants.STORAGE_POLICY, type = CommandType.UUID, entityType = VsphereStoragePoliciesResponse.class,required = false, description = "Name of the storage policy defined at vCenter, this is applicable only for VMware", since = "4.15") + private Long storagePolicy; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -273,6 +277,9 @@ public String getCacheMode() { return cacheMode; } + public Long getStoragePolicy() { + return storagePolicy; + } ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/CreateServiceOfferingCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/CreateServiceOfferingCmd.java index 5015f7c51b87..745e6a0f3c6e 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/CreateServiceOfferingCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/CreateServiceOfferingCmd.java @@ -31,6 +31,7 @@ import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.response.DomainResponse; import org.apache.cloudstack.api.response.ServiceOfferingResponse; +import org.apache.cloudstack.api.response.VsphereStoragePoliciesResponse; import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.commons.collections.MapUtils; import org.apache.commons.collections.CollectionUtils; @@ -216,6 +217,9 @@ public class CreateServiceOfferingCmd extends BaseCmd { since = "4.13") private Integer minMemory; + @Parameter(name = ApiConstants.STORAGE_POLICY, type = CommandType.UUID, entityType = VsphereStoragePoliciesResponse.class,required = false, description = "Name of the storage policy defined at vCenter, this is applicable only for VMware", since = "4.15") + private Long storagePolicy; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -418,6 +422,10 @@ public Integer getMinMemory() { return minMemory; } + public Long getStoragePolicy() { + return storagePolicy; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/ListImageStoresCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/ListImageStoresCmd.java index 8c37c78c7632..4f7cf81f20db 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/ListImageStoresCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/ListImageStoresCmd.java @@ -16,8 +16,6 @@ // under the License. package org.apache.cloudstack.api.command.admin.storage; -import org.apache.log4j.Logger; - import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseListCmd; @@ -25,6 +23,7 @@ import org.apache.cloudstack.api.response.ImageStoreResponse; import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.api.response.ZoneResponse; +import org.apache.log4j.Logger; @APICommand(name = "listImageStores", description = "Lists image stores.", responseObject = ImageStoreResponse.class, since = "4.2.0", requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) @@ -52,6 +51,9 @@ public class ListImageStoresCmd extends BaseListCmd { @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = ImageStoreResponse.class, description = "the ID of the storage pool") private Long id; + @Parameter(name = ApiConstants.READ_ONLY, type = CommandType.BOOLEAN, entityType = ImageStoreResponse.class, description = "read-only status of the image store", since = "4.15.0") + private Boolean readonly; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -80,6 +82,10 @@ public void setProvider(String provider) { this.provider = provider; } + public Boolean getReadonly() { + return readonly; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/MigrateSecondaryStorageDataCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/MigrateSecondaryStorageDataCmd.java new file mode 100644 index 000000000000..9abbecfcd8e7 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/MigrateSecondaryStorageDataCmd.java @@ -0,0 +1,115 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.admin.storage; + +import java.util.List; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.response.ImageStoreResponse; +import org.apache.cloudstack.api.response.MigrationResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.log4j.Logger; + +import com.cloud.event.EventTypes; + +@APICommand(name = MigrateSecondaryStorageDataCmd.APINAME, + description = "migrates data objects from one secondary storage to destination image store(s)", + responseObject = MigrationResponse.class, + requestHasSensitiveInfo = false, + responseHasSensitiveInfo = false, + since = "4.15.0", + authorized = {RoleType.Admin}) +public class MigrateSecondaryStorageDataCmd extends BaseAsyncCmd { + + public static final Logger LOGGER = Logger.getLogger(MigrateSecondaryStorageDataCmd.class.getName()); + + public static final String APINAME = "migrateSecondaryStorageData"; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.SRC_POOL, + type = CommandType.UUID, + entityType = ImageStoreResponse.class, + description = "id of the image store from where the data is to be migrated", + required = true) + private Long id; + + @Parameter(name = ApiConstants.DEST_POOLS, + type = CommandType.LIST, + collectionType = CommandType.UUID, + entityType = ImageStoreResponse.class, + description = "id(s) of the destination secondary storage pool(s) to which the templates are to be migrated", + required = true) + private List migrateTo; + + @Parameter(name = ApiConstants.MIGRATION_TYPE, + type = CommandType.STRING, + description = "Balance: if you want data to be distributed evenly among the destination stores, " + + "Complete: If you want to migrate the entire data from source image store to the destination store(s). Default: Complete") + private String migrationType; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getId() { + return id; + } + + public List getMigrateTo() { + return migrateTo; + } + + public String getMigrationType() { + return migrationType; + } + + @Override + public String getEventType() { + return EventTypes.EVENT_IMAGE_STORE_DATA_MIGRATE; + } + + @Override + public String getEventDescription() { + return "Attempting to migrate files/data objects "; + } + + @Override + public void execute() { + MigrationResponse response = _imageStoreService.migrateData(this); + response.setObjectName("imagestore"); + this.setResponseObject(response); + CallContext.current().setEventDetails(response.getMessage()); + } + + @Override + public String getCommandName() { + return APINAME.toLowerCase() + BaseAsyncCmd.RESPONSE_SUFFIX; + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccountId(); + } +} \ No newline at end of file diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/UpdateImageStoreCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/UpdateImageStoreCmd.java new file mode 100644 index 000000000000..d7dca93b485a --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/UpdateImageStoreCmd.java @@ -0,0 +1,89 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.admin.storage; + +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.ImageStoreResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.log4j.Logger; + +import com.cloud.storage.ImageStore; + +@APICommand(name = UpdateImageStoreCmd.APINAME, description = "Updates image store read-only status", responseObject = ImageStoreResponse.class, entityType = {ImageStore.class}, + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, since = "4.15.0") +public class UpdateImageStoreCmd extends BaseCmd { + private static final Logger LOG = Logger.getLogger(UpdateImageStoreCmd.class.getName()); + public static final String APINAME = "updateImageStore"; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = ImageStoreResponse.class, required = true, description = "Image Store UUID") + private Long id; + + @Parameter(name = ApiConstants.READ_ONLY, type = CommandType.BOOLEAN, required = true, description = "If set to true, it designates the corresponding image store to read-only, " + + "hence not considering them during storage migration") + private Boolean readonly; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getId() { + return id; + } + + public Boolean getReadonly() { + return readonly; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() { + ImageStore result = _storageService.updateImageStoreStatus(getId(), getReadonly()); + ImageStoreResponse storeResponse = null; + if (result != null) { + storeResponse = _responseGenerator.createImageStoreResponse(result); + storeResponse.setResponseName(getCommandName()+"response"); + storeResponse.setObjectName("imagestore"); + setResponseObject(storeResponse); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to update Image store status"); + } + + } + + @Override + public String getCommandName() { + return APINAME; + } + + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccountId(); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportUnmanagedInstanceCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportUnmanagedInstanceCmd.java index 10321cc035d4..5f924f213a02 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportUnmanagedInstanceCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportUnmanagedInstanceCmd.java @@ -204,6 +204,9 @@ public Map getNicNetworkList() { for (Map entry : (Collection>)nicNetworkList.values()) { String nic = entry.get(VmDetailConstants.NIC); String networkUuid = entry.get(VmDetailConstants.NETWORK); + if (LOGGER.isTraceEnabled()) { + LOGGER.trace(String.format("nic, '%s', goes on net, '%s'", nic, networkUuid)); + } if (Strings.isNullOrEmpty(nic) || Strings.isNullOrEmpty(networkUuid) || _entityMgr.findByUuid(Network.class, networkUuid) == null) { throw new InvalidParameterValueException(String.format("Network ID: %s for NIC ID: %s is invalid", networkUuid, nic)); } @@ -219,11 +222,14 @@ public Map getNicIpAddressList() { for (Map entry : (Collection>)nicIpAddressList.values()) { String nic = entry.get(VmDetailConstants.NIC); String ipAddress = Strings.emptyToNull(entry.get(VmDetailConstants.IP4_ADDRESS)); + if (LOGGER.isTraceEnabled()) { + LOGGER.trace(String.format("nic, '%s', gets ip, '%s'", nic, ipAddress)); + } if (Strings.isNullOrEmpty(nic)) { throw new InvalidParameterValueException(String.format("NIC ID: '%s' is invalid for IP address mapping", nic)); } if (Strings.isNullOrEmpty(ipAddress)) { - throw new InvalidParameterValueException(String.format("IP address '%s' for NIC ID: %s is invalid", ipAddress, nic)); + throw new InvalidParameterValueException(String.format("Empty address for NIC ID: %s is invalid", nic)); } if (!Strings.isNullOrEmpty(ipAddress) && !ipAddress.equals("auto") && !NetUtils.isValidIp4(ipAddress)) { throw new InvalidParameterValueException(String.format("IP address '%s' for NIC ID: %s is invalid", ipAddress, nic)); @@ -239,12 +245,15 @@ public Map getDataDiskToDiskOfferingList() { Map dataDiskToDiskOfferingMap = new HashMap<>(); if (MapUtils.isNotEmpty(dataDiskToDiskOfferingList)) { for (Map entry : (Collection>)dataDiskToDiskOfferingList.values()) { - String nic = entry.get(VmDetailConstants.DISK); + String disk = entry.get(VmDetailConstants.DISK); String offeringUuid = entry.get(VmDetailConstants.DISK_OFFERING); - if (Strings.isNullOrEmpty(nic) || Strings.isNullOrEmpty(offeringUuid) || _entityMgr.findByUuid(DiskOffering.class, offeringUuid) == null) { - throw new InvalidParameterValueException(String.format("Disk offering ID: %s for disk ID: %s is invalid", offeringUuid, nic)); + if (LOGGER.isTraceEnabled()) { + LOGGER.trace(String.format("disk, '%s', gets offering, '%s'", disk, offeringUuid)); + } + if (Strings.isNullOrEmpty(disk) || Strings.isNullOrEmpty(offeringUuid) || _entityMgr.findByUuid(DiskOffering.class, offeringUuid) == null) { + throw new InvalidParameterValueException(String.format("Disk offering ID: %s for disk ID: %s is invalid", offeringUuid, disk)); } - dataDiskToDiskOfferingMap.put(nic, _entityMgr.findByUuid(DiskOffering.class, offeringUuid).getId()); + dataDiskToDiskOfferingMap.put(disk, _entityMgr.findByUuid(DiskOffering.class, offeringUuid).getId()); } } return dataDiskToDiskOfferingMap; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/offering/ListServiceOfferingsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/offering/ListServiceOfferingsCmd.java index dcb1730cec6c..91cac0937d49 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/offering/ListServiceOfferingsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/offering/ListServiceOfferingsCmd.java @@ -65,6 +65,24 @@ public class ListServiceOfferingsCmd extends BaseListDomainResourcesCmd { since = "4.13") private Long zoneId; + @Parameter(name = ApiConstants.CPU_NUMBER, + type = CommandType.INTEGER, + description = "the CPU number that listed offerings must support", + since = "4.15") + private Integer cpuNumber; + + @Parameter(name = ApiConstants.MEMORY, + type = CommandType.INTEGER, + description = "the RAM memory that listed offering must support", + since = "4.15") + private Integer memory; + + @Parameter(name = ApiConstants.CPU_SPEED, + type = CommandType.INTEGER, + description = "the CPU speed that listed offerings must support", + since = "4.15") + private Integer cpuSpeed; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -93,6 +111,18 @@ public Long getZoneId() { return zoneId; } + public Integer getCpuNumber() { + return cpuNumber; + } + + public Integer getMemory() { + return memory; + } + + public Integer getCpuSpeed() { + return cpuSpeed; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/template/ListTemplateOVFProperties.java b/api/src/main/java/org/apache/cloudstack/api/command/user/template/ListTemplateOVFProperties.java deleted file mode 100644 index 2a620c9abe5e..000000000000 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/template/ListTemplateOVFProperties.java +++ /dev/null @@ -1,68 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -package org.apache.cloudstack.api.command.user.template; - -import com.cloud.exception.ConcurrentOperationException; -import com.cloud.exception.InsufficientCapacityException; -import com.cloud.exception.NetworkRuleConflictException; -import com.cloud.exception.ResourceAllocationException; -import com.cloud.exception.ResourceUnavailableException; -import org.apache.cloudstack.acl.RoleType; -import org.apache.cloudstack.api.APICommand; -import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.api.BaseCmd; -import org.apache.cloudstack.api.BaseListCmd; -import org.apache.cloudstack.api.Parameter; -import org.apache.cloudstack.api.ServerApiException; -import org.apache.cloudstack.api.response.ListResponse; -import org.apache.cloudstack.api.response.TemplateOVFPropertyResponse; -import org.apache.cloudstack.api.response.TemplateResponse; -import org.apache.cloudstack.context.CallContext; - -@APICommand(name = ListTemplateOVFProperties.APINAME, - description = "List template OVF properties if available.", - responseObject = TemplateOVFPropertyResponse.class, - authorized = {RoleType.Admin, RoleType.DomainAdmin, RoleType.ResourceAdmin, RoleType.User}) -public class ListTemplateOVFProperties extends BaseListCmd { - - public static final String APINAME = "listTemplateOvfProperties"; - - @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = TemplateResponse.class, - description = "the template ID", required = true) - private Long templateId; - - public Long getTemplateId() { - return templateId; - } - - @Override - public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { - ListResponse response = _queryService.listTemplateOVFProperties(this); - response.setResponseName(getCommandName()); - setResponseObject(response); - } - - @Override - public String getCommandName() { - return APINAME.toLowerCase() + BaseCmd.RESPONSE_SUFFIX; - } - - @Override - public long getEntityOwnerId() { - return CallContext.current().getCallingAccount().getId(); - } -} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/template/ListTemplatesCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/template/ListTemplatesCmd.java index 5630d7fba42a..157118285fc0 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/template/ListTemplatesCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/template/ListTemplatesCmd.java @@ -16,8 +16,12 @@ // under the License. package org.apache.cloudstack.api.command.user.template; +import com.cloud.exception.InvalidParameterValueException; +import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; +import java.util.ArrayList; +import java.util.EnumSet; import java.util.List; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiCommandJobType; @@ -82,10 +86,35 @@ public class ListTemplatesCmd extends BaseListTaggedResourcesCmd implements User @Parameter(name = ApiConstants.PARENT_TEMPLATE_ID, type = CommandType.UUID, entityType = TemplateResponse.class, description = "list datadisk templates by parent template id", since = "4.4") private Long parentTemplateId; + @Parameter(name = ApiConstants.DETAILS, + type = CommandType.LIST, + collectionType = CommandType.STRING, + description = "comma separated list of template details requested, value can be a list of [ all, resource, min]") + private List viewDetails; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// + public EnumSet getDetails() throws InvalidParameterValueException { + EnumSet dv; + if (CollectionUtils.isEmpty(viewDetails)) { + dv = EnumSet.of(ApiConstants.DomainDetails.all); + } else { + try { + ArrayList dc = new ArrayList<>(); + for (String detail : viewDetails) { + dc.add(ApiConstants.DomainDetails.valueOf(detail)); + } + dv = EnumSet.copyOf(dc); + } catch (IllegalArgumentException e) { + throw new InvalidParameterValueException("The details parameter contains a non permitted value. The allowed values are " + + EnumSet.allOf(ApiConstants.DomainDetails.class)); + } + } + return dv; + } + public String getHypervisor() { return hypervisor; } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java index 7e0002dbe996..ca08c2a5dedd 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java @@ -162,6 +162,13 @@ public class RegisterTemplateCmd extends BaseCmd implements UserCmd { description = "true if template should bypass Secondary Storage and be downloaded to Primary Storage on deployment") private Boolean directDownload; + @Parameter(name= ApiConstants.DEPLOY_AS_IS, + type = CommandType.BOOLEAN, + description = "VMware only: true if template should not strip and define disks and networks but leave those to the template definition", + since = "4.15" + ) + private Boolean deployAsIs; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -274,6 +281,10 @@ public boolean isDirectDownload() { return directDownload == null ? false : directDownload; } + public Boolean isDeployAsIs() { + return deployAsIs == null ? false : deployAsIs; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java index b4514b1c759b..09455d45dff7 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java @@ -52,6 +52,7 @@ import org.apache.cloudstack.api.response.UserVmResponse; import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.cloudstack.context.CallContext; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.log4j.Logger; @@ -72,6 +73,8 @@ import com.cloud.utils.net.Dhcp; import com.cloud.utils.net.NetUtils; import com.cloud.vm.VirtualMachine; +import com.cloud.vm.VmDetailConstants; +import com.google.common.base.Strings; @APICommand(name = "deployVirtualMachine", description = "Creates and automatically starts a virtual machine based on a service offering, disk offering, and template.", responseObject = UserVmResponse.class, responseView = ResponseView.Restricted, entityType = {VirtualMachine.class}, requestHasSensitiveInfo = false, responseHasSensitiveInfo = true) @@ -221,10 +224,16 @@ public class DeployVMCmd extends BaseAsyncCreateCustomIdCmd implements SecurityG @Parameter(name = ApiConstants.COPY_IMAGE_TAGS, type = CommandType.BOOLEAN, since = "4.13", description = "if true the image tags (if any) will be copied to the VM, default value is false") private Boolean copyImageTags; - @Parameter(name = ApiConstants.OVF_PROPERTIES, type = CommandType.MAP, since = "4.13", - description = "used to specify the OVF properties.") + @Parameter(name = ApiConstants.PROPERTIES, type = CommandType.MAP, since = "4.15", + description = "used to specify the vApp properties.") @LogLevel(LogLevel.Log4jLevel.Off) - private Map vmOvfProperties; + private Map vAppProperties; + + @Parameter(name = ApiConstants.NIC_NETWORK_LIST, type = CommandType.MAP, since = "4.15", + description = "VMware only: used to specify network mapping of a vApp VMware template registered \"as-is\"." + + " Example nicnetworklist[0].ip=Nic-101&nicnetworklist[0].network=uuid") + @LogLevel(LogLevel.Log4jLevel.Off) + private Map vAppNetworks; ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// @@ -311,11 +320,10 @@ public ApiConstants.BootMode getBootMode() { return null; } - - public Map getVmOVFProperties() { + public Map getVmProperties() { Map map = new HashMap<>(); - if (MapUtils.isNotEmpty(vmOvfProperties)) { - Collection parameterCollection = vmOvfProperties.values(); + if (MapUtils.isNotEmpty(vAppProperties)) { + Collection parameterCollection = vAppProperties.values(); Iterator iterator = parameterCollection.iterator(); while (iterator.hasNext()) { HashMap entry = (HashMap)iterator.next(); @@ -325,6 +333,32 @@ public Map getVmOVFProperties() { return map; } + public Map getVmNetworkMap() { + Map map = new HashMap<>(); + if (MapUtils.isNotEmpty(vAppNetworks)) { + Collection parameterCollection = vAppNetworks.values(); + Iterator iterator = parameterCollection.iterator(); + while (iterator.hasNext()) { + HashMap entry = (HashMap) iterator.next(); + Integer nic; + try { + nic = Integer.valueOf(entry.get(VmDetailConstants.NIC)); + } catch (NumberFormatException nfe) { + nic = null; + } + String networkUuid = entry.get(VmDetailConstants.NETWORK); + if (s_logger.isTraceEnabled()) { + s_logger.trace(String.format("nic, '%s', goes on net, '%s'", nic, networkUuid)); + } + if (nic == null || Strings.isNullOrEmpty(networkUuid) || _entityMgr.findByUuid(Network.class, networkUuid) == null) { + throw new InvalidParameterValueException(String.format("Network ID: %s for NIC ID: %s is invalid", networkUuid, nic)); + } + map.put(nic, _entityMgr.findByUuid(Network.class, networkUuid).getId()); + } + } + return map; + } + public String getGroup() { return group; } @@ -374,6 +408,13 @@ public Long getZoneId() { } public List getNetworkIds() { + if (MapUtils.isNotEmpty(vAppNetworks)) { + if (CollectionUtils.isNotEmpty(networkIds) || ipAddress != null || getIp6Address() != null || MapUtils.isNotEmpty(ipToNetworkList)) { + throw new InvalidParameterValueException(String.format("%s can't be specified along with %s, %s, %s", ApiConstants.NIC_NETWORK_LIST, ApiConstants.NETWORK_IDS, ApiConstants.IP_ADDRESS, ApiConstants.IP_NETWORK_LIST)); + } else { + return new ArrayList<>(); + } + } if (ipToNetworkList != null && !ipToNetworkList.isEmpty()) { if ((networkIds != null && !networkIds.isEmpty()) || ipAddress != null || getIp6Address() != null) { throw new InvalidParameterValueException("ipToNetworkMap can't be specified along with networkIds or ipAddress"); diff --git a/api/src/main/java/org/apache/cloudstack/api/response/ImageStoreResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/ImageStoreResponse.java index fba9b2d946a7..190181e67a99 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/ImageStoreResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/ImageStoreResponse.java @@ -16,8 +16,6 @@ // under the License. package org.apache.cloudstack.api.response; -import com.google.gson.annotations.SerializedName; - import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseResponse; import org.apache.cloudstack.api.EntityReference; @@ -25,6 +23,7 @@ import com.cloud.serializer.Param; import com.cloud.storage.ImageStore; import com.cloud.storage.ScopeType; +import com.google.gson.annotations.SerializedName; @EntityReference(value = ImageStore.class) public class ImageStoreResponse extends BaseResponse { @@ -60,6 +59,10 @@ public class ImageStoreResponse extends BaseResponse { @Param(description = "the scope of the image store") private ScopeType scope; + @SerializedName("readonly") + @Param(description = "defines if store is read-only") + private Boolean readonly; + @SerializedName("disksizetotal") @Param(description = "the total disk size of the host") private Long diskSizeTotal; @@ -140,6 +143,12 @@ public void setProtocol(String protocol) { this.protocol = protocol; } + public Boolean getReadonly() { + return readonly; + } + + public void setReadonly(Boolean readonly) { this.readonly = readonly; } + public void setDiskSizeTotal(Long diskSizeTotal) { this.diskSizeTotal = diskSizeTotal; } @@ -147,5 +156,4 @@ public void setDiskSizeTotal(Long diskSizeTotal) { public void setDiskSizeUsed(Long diskSizeUsed) { this.diskSizeUsed = diskSizeUsed; } - -} +} \ No newline at end of file diff --git a/api/src/main/java/org/apache/cloudstack/api/response/MigrationResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/MigrationResponse.java new file mode 100644 index 000000000000..c67b1d2d13ee --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/response/MigrationResponse.java @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.response; + +import org.apache.cloudstack.api.BaseResponse; +import org.apache.cloudstack.api.EntityReference; + +import com.cloud.serializer.Param; +import com.cloud.storage.ImageStore; +import com.google.gson.annotations.SerializedName; + +@EntityReference(value = ImageStore.class) +public class MigrationResponse extends BaseResponse { + @SerializedName("message") + @Param(description = "Response message from migration of secondary storage data objects") + private String message; + + @SerializedName("migrationtype") + @Param(description = "Type of migration requested for") + private String migrationType; + + @SerializedName("success") + @Param(description = "true if operation is executed successfully") + private boolean success; + + MigrationResponse() { + } + + public MigrationResponse(String message, String migrationType, boolean success) { + this.message = message; + this.migrationType = migrationType; + this.success = success; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + public String getMigrationType() { + return migrationType; + } + + public void setMigrationType(String migrationType) { + this.migrationType = migrationType; + } + + public boolean isSuccess() { + return success; + } + + public void setSuccess(boolean success) { + this.success = success; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/response/NetworkACLItemResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/NetworkACLItemResponse.java index 09f78243facb..f63cbbf4cb51 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/NetworkACLItemResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/NetworkACLItemResponse.java @@ -73,6 +73,10 @@ public class NetworkACLItemResponse extends BaseResponse { @Param(description = "the ID of the ACL this item belongs to") private String aclId; + @SerializedName(ApiConstants.ACL_NAME) + @Param(description = "the name of the ACL this item belongs to") + private String aclName; + @SerializedName(ApiConstants.NUMBER) @Param(description = "Number of the ACL Item") private Integer number; @@ -133,6 +137,10 @@ public void setAclId(String aclId) { this.aclId = aclId; } + public void setAclName(String aclName) { + this.aclName = aclName; + } + public void setNumber(Integer number) { this.number = number; } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/NetworkResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/NetworkResponse.java index 4079ab31e662..1b6821248a39 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/NetworkResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/NetworkResponse.java @@ -222,6 +222,10 @@ public class NetworkResponse extends BaseResponse implements ControlledEntityRes @Param(description = "ACL Id associated with the VPC network") private String aclId; + @SerializedName(ApiConstants.ACL_NAME) + @Param(description = "ACL name associated with the VPC network") + private String aclName; + @SerializedName(ApiConstants.STRECHED_L2_SUBNET) @Param(description = "true if network can span multiple zones", since = "4.4") private Boolean strechedL2Subnet; @@ -238,10 +242,6 @@ public class NetworkResponse extends BaseResponse implements ControlledEntityRes @Param(description = "If the network has redundant routers enabled", since = "4.11.1") private Boolean redundantRouter; - @SerializedName(ApiConstants.ACL_NAME) - @Param(description = "ACL name associated with the VPC network", since = "4.15.0") - private String aclName; - public Boolean getDisplayNetwork() { return displayNetwork; } @@ -443,6 +443,14 @@ public void setAclId(String aclId) { this.aclId = aclId; } + public String getAclName() { + return aclName; + } + + public void setAclName(String aclName) { + this.aclName = aclName; + } + public void setStrechedL2Subnet(Boolean strechedL2Subnet) { this.strechedL2Subnet = strechedL2Subnet; } @@ -462,12 +470,4 @@ public Boolean getRedundantRouter() { public void setRedundantRouter(Boolean redundantRouter) { this.redundantRouter = redundantRouter; } - - public String getAclName() { - return aclName; - } - - public void setAclName(String aclName) { - this.aclName = aclName; - } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/PrivateGatewayResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/PrivateGatewayResponse.java index be2faa796d6a..381c7d163999 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/PrivateGatewayResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/PrivateGatewayResponse.java @@ -101,6 +101,10 @@ public class PrivateGatewayResponse extends BaseResponse implements ControlledEn @Param(description = "ACL Id set for private gateway") private String aclId; + @SerializedName(ApiConstants.ACL_NAME) + @Param(description = "ACL name set for private gateway") + private String aclName; + @Override public String getObjectId() { return this.id; @@ -183,4 +187,8 @@ public void setAclId(String aclId) { this.aclId = aclId; } + public void setAclName(String aclName) { + this.aclName = aclName; + } + } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/TemplateOVFPropertyResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/TemplateOVFPropertyResponse.java index 83455a3fe6e2..ebe0d1cfd0db 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/TemplateOVFPropertyResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/TemplateOVFPropertyResponse.java @@ -16,14 +16,14 @@ // under the License. package org.apache.cloudstack.api.response; -import com.cloud.agent.api.storage.OVFProperty; import com.cloud.serializer.Param; import com.google.gson.annotations.SerializedName; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseResponse; -import org.apache.cloudstack.api.EntityReference; -@EntityReference(value = OVFProperty.class) +/** + * the placeholder of parameters to fill for deployment + */ public class TemplateOVFPropertyResponse extends BaseResponse { @SerializedName(ApiConstants.KEY) @@ -58,6 +58,27 @@ public class TemplateOVFPropertyResponse extends BaseResponse { @Param(description = "the ovf property label") private String description; + @SerializedName(ApiConstants.INDEX) + @Param(description = "the ovf property index") + private Integer index; + + @SerializedName(ApiConstants.CATEGORY) + @Param(description = "the ovf property category") + private String category; + + @Override + public boolean equals(Object other) { + if (!(other instanceof TemplateOVFPropertyResponse)) { + return false; + } + return key != null && key.equals(((TemplateOVFPropertyResponse)other).key); + } + + @Override + public int hashCode() { + return key.hashCode(); + } + public String getKey() { return key; } @@ -121,4 +142,20 @@ public Boolean getPassword() { public void setPassword(Boolean password) { this.password = password; } + + public Integer getIndex() { + return index; + } + + public void setIndex(Integer index) { + this.index = index; + } + + public String getCategory() { + return category; + } + + public void setCategory(String category) { + this.category = category; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/TemplateResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/TemplateResponse.java index 81fc2f37b0d6..c733f37a9fbe 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/TemplateResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/TemplateResponse.java @@ -17,7 +17,9 @@ package org.apache.cloudstack.api.response; import java.util.Date; +import java.util.HashMap; import java.util.LinkedHashSet; +import java.util.List; import java.util.Map; import java.util.Set; @@ -54,7 +56,7 @@ public class TemplateResponse extends BaseResponseWithTagInformation implements @Param(description = "the date this template was created") private Date created; - @SerializedName("removed") + @SerializedName(ApiConstants.REMOVED) @Param(description = "the date this template was removed") private Date removed; @@ -79,7 +81,7 @@ public class TemplateResponse extends BaseResponseWithTagInformation implements @Param(description = "true if this template is a featured template, false otherwise") private boolean featured; - @SerializedName("crossZones") + @SerializedName(ApiConstants.CROSS_ZONES) @Param(description = "true if the template is managed across all Zones, false otherwise") private boolean crossZones; @@ -121,7 +123,7 @@ public class TemplateResponse extends BaseResponseWithTagInformation implements @Param(description = "the physical size of the template") private Long physicalSize; - @SerializedName("templatetype") + @SerializedName(ApiConstants.TEMPLATETYPE) @Param(description = "the type of the template") private String templateType; @@ -145,7 +147,7 @@ public class TemplateResponse extends BaseResponseWithTagInformation implements @Param(description = "checksum of the template") private String checksum; - @SerializedName("sourcetemplateid") + @SerializedName(ApiConstants.SOURCETEMPLATEID) @Param(description = "the template ID of the parent template if present") private String sourcetemplateId; @@ -153,7 +155,7 @@ public class TemplateResponse extends BaseResponseWithTagInformation implements @Param(description = "the ID of the secondary storage host for the template") private String hostId; - @SerializedName("hostname") + @SerializedName(ApiConstants.HOST_NAME) @Param(description = "the name of the secondary storage host for the template") private String hostName; @@ -171,7 +173,11 @@ public class TemplateResponse extends BaseResponseWithTagInformation implements @SerializedName(ApiConstants.DETAILS) @Param(description = "additional key/value details tied with template") - private Map details; + private Map details; + + @SerializedName(ApiConstants.DOWNLOAD_DETAILS) + @Param(description = "Lists the download progress of a template across all secondary storages") + private List> downloadDetails; @SerializedName(ApiConstants.BITS) @Param(description = "the processor bit size", since = "4.10") @@ -189,12 +195,22 @@ public class TemplateResponse extends BaseResponseWithTagInformation implements @Param(description = "KVM Only: true if template is directly downloaded to Primary Storage bypassing Secondary Storage") private Boolean directDownload; + @SerializedName(ApiConstants.DEPLOY_AS_IS) + @Param(description = "VMware only: true if template is deployed without orchestrating disks and networks but \"as-is\" defined in the template.") + private Boolean deployAsIs; + + @SerializedName(ApiConstants.DEPLOY_AS_IS_DETAILS) + @Param(description = "VMware only: additional key/value details tied with deploy-as-is template") + private Map deployAsIsDetails; + @SerializedName("parenttemplateid") @Param(description = "if Datadisk template, then id of the root disk template this template belongs to") + @Deprecated(since = "4.15") private String parentTemplateId; @SerializedName("childtemplates") @Param(description = "if root disk template, then ids of the datas disk templates this template owns") + @Deprecated(since = "4.15") private Set childTemplates; @SerializedName(ApiConstants.REQUIRES_HVM) @@ -255,6 +271,10 @@ public void setPublic(boolean isPublic) { this.isPublic = isPublic; } + public void setDownloadProgress(List> downloadDetails) { + this.downloadDetails = downloadDetails; + } + public void setCreated(Date created) { this.created = created; } @@ -351,14 +371,21 @@ public void setProjectName(String projectName) { this.projectName = projectName; } - public Map getDetails() { + public Map getDetails() { return this.details; } - public void setDetails(Map details) { + public void setDetails(Map details) { this.details = details; } + public void addDetail(String key, String value) { + if (this.details == null) { + setDetails(new HashMap<>()); + } + this.details.put(key,value); + } + public void setTags(Set tags) { this.tags = tags; } @@ -387,6 +414,10 @@ public Boolean getDirectDownload() { return directDownload; } + public void setDeployAsIs(Boolean deployAsIs) { + this.deployAsIs = deployAsIs; + } + public void setParentTemplateId(String parentTemplateId) { this.parentTemplateId = parentTemplateId; } @@ -402,4 +433,19 @@ public Boolean isRequiresHvm() { public void setRequiresHvm(Boolean requiresHvm) { this.requiresHvm = requiresHvm; } + + public Map getDeployAsIsDetails() { + return this.deployAsIsDetails; + } + + public void setDeployAsIsDetails(Map details) { + this.deployAsIsDetails = details; + } + + public void addDeployAsIsDetail(String key, String value) { + if (this.deployAsIsDetails == null) { + setDeployAsIsDetails(new HashMap<>()); + } + this.deployAsIsDetails.put(key,value); + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/VsphereStoragePoliciesResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/VsphereStoragePoliciesResponse.java new file mode 100644 index 000000000000..63c49f148790 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/response/VsphereStoragePoliciesResponse.java @@ -0,0 +1,89 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.response; + +import com.cloud.dc.VsphereStoragePolicy; +import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponse; +import org.apache.cloudstack.api.EntityReference; + + +@EntityReference(value = VsphereStoragePolicy.class) +public class VsphereStoragePoliciesResponse extends BaseResponse { + + @SerializedName(ApiConstants.ID) + @Param(description = "the ID of the Storage Policy") + private String id; + + @SerializedName(ApiConstants.ZONE_ID) + @Param(description = "the ID of the Zone") + private String zoneId; + + @SerializedName(ApiConstants.POLICY_ID) + @Param(description = "the identifier of the Storage Policy in vSphere DataCenter") + private String policyId; + + @SerializedName(ApiConstants.NAME) + @Param(description = "the name of the Storage Policy") + private String name; + + @SerializedName(ApiConstants.DESCRIPTION) + @Param(description = "the description of the Storage Policy") + private String description; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getZoneId() { + return zoneId; + } + + public void setZoneId(String zoneId) { + this.zoneId = zoneId; + } + + public String getPolicyId() { + return policyId; + } + + public void setPolicyId(String policyId) { + this.policyId = policyId; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/query/QueryService.java b/api/src/main/java/org/apache/cloudstack/query/QueryService.java index 0a400ed9ae3c..717af5df8e94 100644 --- a/api/src/main/java/org/apache/cloudstack/query/QueryService.java +++ b/api/src/main/java/org/apache/cloudstack/query/QueryService.java @@ -44,7 +44,6 @@ import org.apache.cloudstack.api.command.user.resource.ListDetailOptionsCmd; import org.apache.cloudstack.api.command.user.securitygroup.ListSecurityGroupsCmd; import org.apache.cloudstack.api.command.user.tag.ListTagsCmd; -import org.apache.cloudstack.api.command.user.template.ListTemplateOVFProperties; import org.apache.cloudstack.api.command.user.template.ListTemplatesCmd; import org.apache.cloudstack.api.command.user.vm.ListVMsCmd; import org.apache.cloudstack.api.command.user.vmgroup.ListVMGroupsCmd; @@ -74,7 +73,6 @@ import org.apache.cloudstack.api.response.ServiceOfferingResponse; import org.apache.cloudstack.api.response.StoragePoolResponse; import org.apache.cloudstack.api.response.StorageTagResponse; -import org.apache.cloudstack.api.response.TemplateOVFPropertyResponse; import org.apache.cloudstack.api.response.TemplateResponse; import org.apache.cloudstack.api.response.UserResponse; import org.apache.cloudstack.api.response.UserVmResponse; @@ -173,7 +171,5 @@ public interface QueryService { ListResponse listManagementServers(ListMgmtsCmd cmd); - ListResponse listTemplateOVFProperties(ListTemplateOVFProperties cmd); - List listRouterHealthChecks(GetRouterHealthCheckResultsCmd cmd); } diff --git a/api/src/main/java/org/apache/cloudstack/storage/ImageStoreService.java b/api/src/main/java/org/apache/cloudstack/storage/ImageStoreService.java new file mode 100644 index 000000000000..b8f14ad2bfaf --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/storage/ImageStoreService.java @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.storage; + +import org.apache.cloudstack.api.command.admin.storage.MigrateSecondaryStorageDataCmd; +import org.apache.cloudstack.api.response.MigrationResponse; + +public interface ImageStoreService { + + public static enum MigrationPolicy { + BALANCE, COMPLETE + } + MigrationResponse migrateData(MigrateSecondaryStorageDataCmd cmd); +} diff --git a/api/src/test/java/com/cloud/agent/api/storage/OVFHelperTest.java b/api/src/test/java/com/cloud/agent/api/storage/OVFHelperTest.java index 8aa9852fb257..e28680784d70 100644 --- a/api/src/test/java/com/cloud/agent/api/storage/OVFHelperTest.java +++ b/api/src/test/java/com/cloud/agent/api/storage/OVFHelperTest.java @@ -16,6 +16,10 @@ // under the License. package com.cloud.agent.api.storage; +import com.cloud.agent.api.to.deployasis.OVFConfigurationTO; +import com.cloud.agent.api.to.deployasis.OVFEulaSectionTO; +import com.cloud.agent.api.to.deployasis.OVFPropertyTO; +import com.cloud.agent.api.to.deployasis.OVFVirtualHardwareItemTO; import org.junit.Assert; import org.junit.Test; import org.xml.sax.SAXException; @@ -40,16 +44,689 @@ public class OVFHelperTest { "" + ""; + private String ovfFileDeploymentOptionsSection = + "\n" + + " Deployment Configuration information\n" + + " \n" + + " \n" + + " Use this option to deploy an ASAv with a maximum throughput of 100 Mbps (uses 1 vCPU and 2 GB of memory).\n" + + " \n" + + " \n" + + " \n" + + " Use this option to deploy an ASAv with a maximum throughput of 1 Gbps (uses 1 vCPU and 2 GB of memory).\n" + + " \n" + + " \n" + + " \n" + + " Use this option to deploy an ASAv with a maximum throughput of 2 Gbps (uses 4 vCPUs and 8 GB of memory).\n" + + " \n" + + " "; + + private String ovfFileVirtualHardwareSection = + "\n" + + " Virtual hardware requirements\n" + + " \n" + + " Virtual Hardware Family\n" + + " 0\n" + + " ASAv\n" + + " vmx-08,vmx-09\n" + + " \n" + + " \n" + + " hertz * 10^6\n" + + " Number of Virtual CPUs\n" + + " 1 virtual CPU(s)\n" + + " 1\n" + + " 5000\n" + + " 1000\n" + + " 3\n" + + " 1\n" + + " \n" + + " \n" + + " hertz * 10^6\n" + + " Number of Virtual CPUs\n" + + " 4 virtual CPU(s)\n" + + " 1\n" + + " 20000\n" + + " 1000\n" + + " 3\n" + + " 4\n" + + " \n" + + " \n" + + " byte * 2^20\n" + + " Memory Size\n" + + " 2048MB of memory\n" + + " 2\n" + + " 2048\n" + + " 2048\n" + + " 4\n" + + " 2048\n" + + " \n" + + " \n" + + " byte * 2^20\n" + + " Memory Size\n" + + " 8192MB of memory\n" + + " 2\n" + + " 8192\n" + + " 8192\n" + + " 4\n" + + " 8192\n" + + " \n" + + " \n" + + " 0\n" + + " SCSI Controller\n" + + " SCSI controller 0\n" + + " 3\n" + + " lsilogic\n" + + " 6\n" + + " \n" + + " \n" + + " 0\n" + + " IDE Controller\n" + + " IDE 0\n" + + " 4\n" + + " 5\n" + + " \n" + + " \n" + + " 0\n" + + " true\n" + + " CD/DVD Drive\n" + + " 5\n" + + " 4\n" + + " 15\n" + + " \n" + + " \n" + + " 1\n" + + " true\n" + + " CD/DVD Drive\n" + + " ovf:/file/file3\n" + + " 18\n" + + " 4\n" + + " 15\n" + + " \n" + + " \n" + + " 7\n" + + " true\n" + + " Management0-0\n" + + " E1000 Ethernet adapter on \"Management Network\"\n" + + " Network adapter 1\n" + + " 6\n" + + " E1000\n" + + " 10\n" + + " \n" + + " \n" + + " 0\n" + + " Hard disk 1\n" + + " ovf:/disk/vmdisk1\n" + + " 7\n" + + " 3\n" + + " 17\n" + + " \n" + + " \n" + + " 1\n" + + " Hard disk 2\n" + + " ovf:/disk/vmdisk2\n" + + " 8\n" + + " 3\n" + + " 17\n" + + " \n" + + " \n" + + " 8\n" + + " true\n" + + " GigabitEthernet0-0\n" + + " General purpose E1000 Ethernet adapter\n" + + " Network adapter 2\n" + + " 9\n" + + " E1000\n" + + " 10\n" + + " \n" + + " \n" + + " 9\n" + + " true\n" + + " GigabitEthernet0-1\n" + + " General purpose E1000 Ethernet adapter\n" + + " Network adapter 3\n" + + " 10\n" + + " E1000\n" + + " 10\n" + + " \n" + + " \n" + + " 10\n" + + " true\n" + + " GigabitEthernet0-2\n" + + " General purpose E1000 Ethernet adapter\n" + + " Network adapter 4\n" + + " 11\n" + + " E1000\n" + + " 10\n" + + " \n" + + " \n" + + " 11\n" + + " true\n" + + " GigabitEthernet0-3\n" + + " General purpose E1000 Ethernet adapter\n" + + " Network adapter 5\n" + + " 12\n" + + " E1000\n" + + " 10\n" + + " \n" + + " \n" + + " 12\n" + + " true\n" + + " GigabitEthernet0-4\n" + + " General purpose E1000 Ethernet adapter\n" + + " Network adapter 6\n" + + " 13\n" + + " E1000\n" + + " 10\n" + + " \n" + + " \n" + + " 13\n" + + " true\n" + + " GigabitEthernet0-5\n" + + " General purpose E1000 Ethernet adapter\n" + + " Network adapter 7\n" + + " 14\n" + + " E1000\n" + + " 10\n" + + " \n" + + " \n" + + " 14\n" + + " true\n" + + " GigabitEthernet0-6\n" + + " General purpose E1000 Ethernet adapter\n" + + " Network adapter 8\n" + + " 15\n" + + " E1000\n" + + " 10\n" + + " \n" + + " \n" + + " 15\n" + + " true\n" + + " GigabitEthernet0-7\n" + + " General purpose E1000 Ethernet adapter\n" + + " Network adapter 9\n" + + " 16\n" + + " E1000\n" + + " 10\n" + + " \n" + + " \n" + + " 16\n" + + " true\n" + + " GigabitEthernet0-8\n" + + " Default HA failover E1000 Ethernet adapter, or additional standalone general purpose adapter\n" + + " Network adapter 10\n" + + " 17\n" + + " E1000\n" + + " 10\n" + + " \n" + + " \n" + + " "; + + private String eulaSections = + "\n" + + "\n" + + " end-user license agreement\n" + + " END USER LICENSE AGREEMENT\n" + + "\n" + + "IMPORTANT: PLEASE READ THIS END USER LICENSE AGREEMENT CAREFULLY. IT IS VERY IMPORTANT THAT YOU CHECK THAT YOU ARE PURCHASING CISCO SOFTWARE OR EQUIPMENT FROM AN APPROVED SOURCE AND THAT YOU, OR THE ENTITY YOU REPRESENT (COLLECTIVELY, THE \"CUSTOMER\") HAVE BEEN REGISTERED AS THE END USER FOR THE PURPOSES OF THIS CISCO END USER LICENSE AGREEMENT. IF YOU ARE NOT REGISTERED AS THE END USER YOU HAVE NO LICENSE TO USE THE SOFTWARE AND THE LIMITED WARRANTY IN THIS END USER LICENSE AGREEMENT DOES NOT APPLY. ASSUMING YOU HAVE PURCHASED FROM AN APPROVED SOURCE, DOWNLOADING, INSTALLING OR USING CISCO OR CISCO-SUPPLIED SOFTWARE CONSTITUTES ACCEPTANCE OF THIS AGREEMENT.\n" + + "\n" + + "CISCO SYSTEMS, INC. OR ITS AFFILIATE LICENSING THE SOFTWARE (\"CISCO\") IS WILLING TO LICENSE THIS SOFTWARE TO YOU ONLY UPON THE CONDITION THAT YOU PURCHASED THE SOFTWARE FROM AN APPROVED SOURCE AND THAT YOU ACCEPT ALL OF THE TERMS CONTAINED IN THIS END USER LICENSE AGREEMENT PLUS ANY ADDITIONAL LIMITATIONS ON THE LICENSE SET FORTH IN A SUPPLEMENTAL LICENSE AGREEMENT ACCOMPANYING THE PRODUCT, MADE AVAILABLE AT THE TIME OF YOUR ORDER, OR POSTED ON THE CISCO WEBSITE AT www.cisco.com/go/terms (COLLECTIVELY THE \"AGREEMENT\"). TO THE EXTENT OF ANY CONFLICT BETWEEN THE TERMS OF THIS END USER LICENSE AGREEMENT AND ANY SUPPLEMENTAL LICENSE AGREEMENT, THE SUPPLEMENTAL LICENSE AGREEMENT SHALL APPLY. BY DOWNLOADING, INSTALLING, OR USING THE SOFTWARE, YOU ARE REPRESENTING THAT YOU PURCHASED THE SOFTWARE FROM AN APPROVED SOURCE AND BINDING YOURSELF TO THE AGREEMENT. IF YOU DO " + + "NOT AGREE TO ALL OF THE TERMS OF THE AGREEMENT, THEN CISCO IS UNWILLING TO LICENSE THE SOFTWARE TO YOU AND (A) YOU MAY NOT DOWNLOAD, INSTALL OR USE THE SOFTWARE, AND (B) YOU MAY RETURN THE SOFTWARE (INCLUDING ANY UNOPENED CD PACKAGE AND ANY WRITTEN MATERIALS) FOR A FULL REFUND, OR, IF THE SOFTWARE AND WRITTEN MATERIALS ARE SUPPLIED AS PART OF ANOTHER PRODUCT, YOU MAY RETURN THE ENTIRE PRODUCT FOR A FULL REFUND. YOUR RIGHT TO RETURN AND REFUND EXPIRES 30 DAYS AFTER PURCHASE FROM AN APPROVED SOURCE, AND APPLIES ONLY IF YOU ARE THE ORIGINAL AND REGISTERED END USER PURCHASER. FOR THE PURPOSES OF THIS END USER LICENSE AGREEMENT, AN \"APPROVED SOURCE\" MEANS (A) CISCO; OR (B) A DISTRIBUTOR OR SYSTEMS INTEGRATOR AUTHORIZED BY CISCO TO DISTRIBUTE / SELL CISCO EQUIPMENT, SOFTWARE AND SERVICES WITHIN YOUR TERRITORY TO END " + + "USERS; OR (C) A RESELLER AUTHORIZED BY ANY SUCH DISTRIBUTOR OR SYSTEMS INTEGRATOR IN ACCORDANCE WITH THE TERMS OF THE DISTRIBUTOR'S AGREEMENT WITH CISCO TO DISTRIBUTE / SELL THE CISCO EQUIPMENT, SOFTWARE AND SERVICES WITHIN YOUR TERRITORY TO END USERS.\n" + + "\n" + + "THE FOLLOWING TERMS OF THE AGREEMENT GOVERN CUSTOMER'S USE OF THE SOFTWARE (DEFINED BELOW), EXCEPT TO THE EXTENT: (A) THERE IS A SEPARATE SIGNED CONTRACT BETWEEN CUSTOMER AND CISCO GOVERNING CUSTOMER'S USE OF THE SOFTWARE, OR (B) THE SOFTWARE INCLUDES A SEPARATE \"CLICK-ACCEPT\" LICENSE AGREEMENT OR THIRD PARTY LICENSE AGREEMENT AS PART OF THE INSTALLATION OR DOWNLOAD PROCESS GOVERNING CUSTOMER'S USE OF THE SOFTWARE. TO THE EXTENT OF A CONFLICT BETWEEN THE PROVISIONS OF THE FOREGOING DOCUMENTS, THE ORDER OF PRECEDENCE SHALL BE (1)THE SIGNED CONTRACT, (2) THE CLICK-ACCEPT AGREEMENT OR THIRD PARTY LICENSE AGREEMENT, AND (3) THE AGREEMENT. FOR PURPOSES OF THE AGREEMENT, \"SOFTWARE\" SHALL MEAN COMPUTER PROGRAMS, INCLUDING FIRMWARE AND COMPUTER PROGRAMS EMBEDDED IN CISCO EQUIPMENT, AS PROVIDED TO CUSTOMER BY AN APPROVED SOURCE, AND ANY UPGRADES, UPDATES, BUG FIXES " + + "OR MODIFIED VERSIONS THERETO (COLLECTIVELY, \"UPGRADES\"), ANY OF THE SAME WHICH HAS BEEN RELICENSED UNDER THE CISCO SOFTWARE TRANSFER AND RE-LICENSING POLICY (AS MAY BE AMENDED BY CISCO FROM TIME TO TIME) OR BACKUP COPIES OF ANY OF THE FOREGOING.\n" + + "\n" + + "License. Conditioned upon compliance with the terms and conditions of the Agreement, Cisco grants to Customer a nonexclusive and nontransferable license to use for Customer's internal business purposes the Software and the Documentation for which Customer has paid the required license fees to an Approved Source. \"Documentation\" means written information (whether contained in user or technical manuals, training materials, specifications or otherwise) pertaining to the Software and made available by an Approved Source with the Software in any manner (including on CD-Rom, or on-line). In order to use the Software, Customer may be required to input a registration number or product authorization key and register Customer's copy of the Software online at Cisco's website to obtain the necessary license key or license file.\n" + + "\n" + + "Customer's license to use the Software shall be limited to, and Customer shall not use the Software in excess of, a single hardware chassis or card or such other limitations as are set forth in the applicable Supplemental License Agreement or in the applicable purchase order which has been accepted by an Approved Source and for which Customer has paid to an Approved Source the required license fee (the \"Purchase Order\").\n" + + "\n" + + "Unless otherwise expressly provided in the Documentation or any applicable Supplemental License Agreement, Customer shall use the Software solely as embedded in, for execution on, or (where the applicable Documentation permits installation on non-Cisco equipment) for communication with Cisco equipment owned or leased by Customer and used for Customer's internal business purposes. No other licenses are granted by implication, estoppel or otherwise.\n" + + "\n" + + "For evaluation or beta copies for which Cisco does not charge a license fee, the above requirement to pay license fees does not apply.\n" + + "\n" + + "General Limitations. This is a license, not a transfer of title, to the Software and Documentation, and Cisco retains ownership of all copies of the Software and Documentation. Customer acknowledges that the Software and Documentation contain trade secrets of Cisco or its suppliers or licensors, including but not limited to the specific internal design and structure of individual programs and associated interface information. Except as otherwise expressly provided under the Agreement, Customer shall only use the Software in connection with the use of Cisco equipment purchased by the Customer from an Approved Source and Customer shall have no right, and Customer specifically agrees not to:\n" + + "\n" + + "(i) transfer, assign or sublicense its license rights to any other person or entity (other than in compliance with any Cisco relicensing/transfer policy then in force), or use the Software on Cisco equipment not purchased by the Customer from an Approved Source or on secondhand Cisco equipment, and Customer acknowledges that any attempted transfer, assignment, sublicense or use shall be void;\n" + + "\n" + + "(ii) make error corrections to or otherwise modify or adapt the Software or create derivative works based upon the Software, or permit third parties to do the same;\n" + + "\n" + + "(iii) reverse engineer or decompile, decrypt, disassemble or otherwise reduce the Software to human-readable form, except to the extent otherwise expressly permitted under applicable law notwithstanding this restriction or except to the extent that Cisco is legally required to permit such specific activity pursuant to any applicable open source license;\n" + + "\n" + + "(iv) publish any results of benchmark tests run on the Software;\n" + + "\n" + + "(v) use or permit the Software to be used to perform services for third parties, whether on a service bureau or time sharing basis or otherwise, without the express written authorization of Cisco; or\n" + + "\n" + + "(vi) disclose, provide, or otherwise make available trade secrets contained within the Software and Documentation in any form to any third party without the prior written consent of Cisco. Customer shall implement reasonable security measures to protect such trade secrets.\n" + + "\n" + + "To the extent required by applicable law, and at Customer's written request, Cisco shall provide Customer with the interface information needed to achieve interoperability between the Software and another independently created program, on payment of Cisco's applicable fee, if any. Customer shall observe strict obligations of confidentiality with respect to such information and shall use such information in compliance with any applicable terms and conditions upon which Cisco makes such information available.\n" + + "\n" + + "Software, Upgrades and Additional Copies. NOTWITHSTANDING ANY OTHER PROVISION OF THE AGREEMENT: (1) CUSTOMER HAS NO LICENSE OR RIGHT TO MAKE OR USE ANY ADDITIONAL COPIES OR UPGRADES UNLESS CUSTOMER, AT THE TIME OF MAKING OR ACQUIRING SUCH COPY OR UPGRADE, ALREADY HOLDS A VALID LICENSE TO THE ORIGINAL SOFTWARE AND HAS PAID THE APPLICABLE FEE TO AN APPROVED SOURCE FOR THE UPGRADE OR ADDITIONAL COPIES; (2) USE OF UPGRADES IS LIMITED TO CISCO EQUIPMENT SUPPLIED BY AN APPROVED SOURCE FOR WHICH CUSTOMER IS THE ORIGINAL END USER PURCHASER OR LESSEE OR OTHERWISE HOLDS A VALID LICENSE TO USE THE SOFTWARE WHICH IS BEING UPGRADED; AND (3) THE MAKING AND USE OF ADDITIONAL COPIES IS LIMITED TO NECESSARY BACKUP PURPOSES ONLY.\n" + + "\n" + + "Proprietary Notices. Customer agrees to maintain and reproduce all copyright, proprietary, and other notices on all copies, in any form, of the Software in the same form and manner that such copyright and other proprietary notices are included on the Software. Except as expressly authorized in the Agreement, Customer shall not make any copies or duplicates of any Software without the prior written permission of Cisco.\n" + + "\n" + + "Term and Termination. The Agreement and the license granted herein shall remain effective until terminated. Customer may terminate the Agreement and the license at any time by destroying all copies of Software and any Documentation. Customer's rights under the Agreement will terminate immediately without notice from Cisco if Customer fails to comply with any provision of the Agreement. Upon termination, Customer shall destroy all copies of Software and Documentation in its possession or control. All confidentiality obligations of Customer, all restrictions and limitations imposed on the Customer under the section titled \"General Limitations\" and all limitations of liability and disclaimers and restrictions of warranty shall survive termination of this Agreement. In addition, the provisions of the sections titled \"U.S. Government End User Purchasers\" and \"General Terms Applicable to the Limited Warranty Statement " + + "and End User License Agreement\" shall survive termination of the Agreement.\n" + + "\n" + + "Customer Records. Customer grants to Cisco and its independent accountants the right to examine Customer's books, records and accounts during Customer's normal business hours to verify compliance with this Agreement. In the event such audit discloses non-compliance with this Agreement, Customer shall promptly pay to Cisco the appropriate license fees, plus the reasonable cost of conducting the audit.\n" + + "\n" + + "Export, Re-Export, Transfer and Use Controls. The Software, Documentation and technology or direct products thereof (hereafter referred to as Software and Technology), supplied by Cisco under the Agreement are subject to export controls under the laws and regulations of the United States (\"U.S.\") and any other applicable countries' laws and regulations. Customer shall comply with such laws and regulations governing export, re-export, import, transfer and use of Cisco Software and Technology and will obtain all required U.S. and local authorizations, permits, or licenses. Cisco and Customer each agree to provide the other information, support documents, and assistance as may reasonably be required by the other in connection with securing authorizations or licenses. Information regarding compliance with export, re-export, transfer and use may be located at the following URL: " + + "www.cisco.com/web/about/doing_business/legal/global_export_trade/general_export/contract_compliance.html\n" + + "\n" + + "U.S. Government End User Purchasers. The Software and Documentation qualify as \"commercial items,\" as that term is defined at Federal Acquisition Regulation (\"FAR\") (48 C.F.R.) 2.101, consisting of \"commercial computer software\" and \"commercial computer software documentation\" as such terms are used in FAR 12.212. Consistent with FAR 12.212 and DoD FAR Supp. 227.7202-1 through 227.7202-4, and notwithstanding any other FAR or other contractual clause to the contrary in any agreement into which the Agreement may be incorporated, Customer may provide to Government end user or, if the Agreement is direct, Government end user will acquire, the Software and Documentation with only those rights set forth in the Agreement. Use of either the Software or Documentation or both constitutes agreement by the Government that the Software and Documentation are \"commercial computer software\" and \"commercial computer " + + "software documentation,\" and constitutes acceptance of the rights and restrictions herein.\n" + + "\n" + + "Identified Components; Additional Terms. The Software may contain or be delivered with one or more components, which may include third-party components, identified by Cisco in the Documentation, readme.txt file, third-party click-accept or elsewhere (e.g. on www.cisco.com) (the \"Identified Component(s)\") as being subject to different license agreement terms, disclaimers of warranties, limited warranties or other terms and conditions (collectively, \"Additional Terms\") than those set forth herein. You agree to the applicable Additional Terms for any such Identified Component(s).\n" + + "\n" + + "Limited Warranty\n" + + "\n" + + "Subject to the limitations and conditions set forth herein, Cisco warrants that commencing from the date of shipment to Customer (but in case of resale by an Approved Source other than Cisco, commencing not more than ninety (90) days after original shipment by Cisco), and continuing for a period of the longer of (a) ninety (90) days or (b) the warranty period (if any) expressly set forth as applicable specifically to software in the warranty card accompanying the product of which the Software is a part (the \"Product\") (if any): (a) the media on which the Software is furnished will be free of defects in materials and workmanship under normal use; and (b) the Software substantially conforms to the Documentation. The date of shipment of a Product by Cisco is set forth on the packaging material in which the Product is shipped. Except for the foregoing, the Software is provided \"AS IS\". This limited warranty extends only to the " + + "Software purchased from an Approved Source by a Customer who is the first registered end user. Customer's sole and exclusive remedy and the entire liability of Cisco and its suppliers under this limited warranty will be (i) replacement of defective media and/or (ii) at Cisco's option, repair, replacement, or refund of the purchase price of the Software, in both cases subject to the condition that any error or defect constituting a breach of this limited warranty is reported to the Approved Source supplying the Software to Customer, within the warranty period. Cisco or the Approved Source supplying the Software to Customer may, at its option, require return of the Software and/or Documentation as a condition to the remedy. In no event does Cisco warrant that the Software is error free or that Customer will be able to operate the Software without problems or interruptions. In addition, due to the continual development of new " + + "techniques for intruding upon and attacking networks, Cisco does not warrant that the Software or any equipment, system or network on which the Software is used will be free of vulnerability to intrusion or attack.\n" + + "\n" + + "Restrictions. This warranty does not apply if the Software, Product or any other equipment upon which the Software is authorized to be used (a) has been altered, except by Cisco or its authorized representative, (b) has not been installed, operated, repaired, or maintained in accordance with instructions supplied by Cisco, (c) has been subjected to abnormal physical or electrical stress, abnormal environmental conditions, misuse, negligence, or accident; or (d) is licensed for beta, evaluation, testing or demonstration purposes. The Software warranty also does not apply to (e) any temporary Software modules; (f) any Software not posted on Cisco's Software Center; (g) any Software that Cisco expressly provides on an \"AS IS\" basis on Cisco's Software Center; (h) any Software for which an Approved Source does not receive a license fee; and (i) Software supplied by any third party which is not an Approved Source.\n" + + "\n" + + "DISCLAIMER OF WARRANTY\n" + + "\n" + + "EXCEPT AS SPECIFIED IN THIS WARRANTY SECTION, ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS, AND WARRANTIES INCLUDING, WITHOUT LIMITATION, ANY IMPLIED WARRANTY OR CONDITION OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, SATISFACTORY QUALITY, NON-INTERFERENCE, ACCURACY OF INFORMATIONAL CONTENT, OR ARISING FROM A COURSE OF DEALING, LAW, USAGE, OR TRADE PRACTICE, ARE HEREBY EXCLUDED TO THE EXTENT ALLOWED BY APPLICABLE LAW AND ARE EXPRESSLY DISCLAIMED BY CISCO, ITS SUPPLIERS AND LICENSORS. TO THE EXTENT THAT ANY OF THE SAME CANNOT BE EXCLUDED, SUCH IMPLIED CONDITION, REPRESENTATION AND/OR WARRANTY IS LIMITED IN DURATION TO THE EXPRESS WARRANTY PERIOD REFERRED TO IN THE \"LIMITED WARRANTY\" SECTION ABOVE. BECAUSE SOME STATES OR JURISDICTIONS DO NOT ALLOW LIMITATIONS ON HOW LONG AN IMPLIED WARRANTY LASTS, THE ABOVE LIMITATION MAY NOT APPLY IN SUCH STATES. THIS WARRANTY GIVES CUSTOMER SPECIFIC LEGAL RIGHTS, " + + "AND CUSTOMER MAY ALSO HAVE OTHER RIGHTS WHICH VARY FROM JURISDICTION TO JURISDICTION. This disclaimer and exclusion shall apply even if the express warranty set forth above fails of its essential purpose.\n" + + "\n" + + "Disclaimer of Liabilities-Limitation of Liability. IF YOU ACQUIRED THE SOFTWARE IN THE UNITED STATES, LATIN AMERICA, CANADA, JAPAN OR THE CARIBBEAN, NOTWITHSTANDING ANYTHING ELSE IN THE AGREEMENT TO THE CONTRARY, ALL LIABILITY OF CISCO, ITS AFFILIATES, OFFICERS, DIRECTORS, EMPLOYEES, AGENTS, SUPPLIERS AND LICENSORS COLLECTIVELY, TO CUSTOMER, WHETHER IN CONTRACT, TORT (INCLUDING NEGLIGENCE), BREACH OF WARRANTY OR OTHERWISE, SHALL NOT EXCEED THE PRICE PAID BY CUSTOMER TO ANY APPROVED SOURCE FOR THE SOFTWARE THAT GAVE RISE TO THE CLAIM OR IF THE SOFTWARE IS PART OF ANOTHER PRODUCT, THE PRICE PAID FOR SUCH OTHER PRODUCT. THIS LIMITATION OF LIABILITY FOR SOFTWARE IS CUMULATIVE AND NOT PER INCIDENT (I.E. THE EXISTENCE OF TWO OR MORE CLAIMS WILL NOT ENLARGE THIS LIMIT).\n" + + "\n" + + "IF YOU ACQUIRED THE SOFTWARE IN EUROPE, THE MIDDLE EAST, AFRICA, ASIA OR OCEANIA, NOTWITHSTANDING ANYTHING ELSE IN THE AGREEMENT TO THE CONTRARY, ALL LIABILITY OF CISCO, ITS AFFILIATES, OFFICERS, DIRECTORS, EMPLOYEES, AGENTS, SUPPLIERS AND LICENSORS COLLECTIVELY, TO CUSTOMER, WHETHER IN CONTRACT, TORT (INCLUDING NEGLIGENCE), BREACH OF WARRANTY OR OTHERWISE, SHALL NOT EXCEED THE PRICE PAID BY CUSTOMER TO CISCO FOR THE SOFTWARE THAT GAVE RISE TO THE CLAIM OR IF THE SOFTWARE IS PART OF ANOTHER PRODUCT, THE PRICE PAID FOR SUCH OTHER PRODUCT. THIS LIMITATION OF LIABILITY FOR SOFTWARE IS CUMULATIVE AND NOT PER INCIDENT (I.E. THE EXISTENCE OF TWO OR MORE CLAIMS WILL NOT ENLARGE THIS LIMIT). NOTHING IN THE AGREEMENT SHALL LIMIT (I) THE LIABILITY OF CISCO, ITS AFFILIATES, OFFICERS, DIRECTORS, EMPLOYEES, AGENTS, SUPPLIERS AND LICENSORS TO CUSTOMER FOR PERSONAL INJURY OR DEATH CAUSED BY THEIR NEGLIGENCE, (II) CISCO'S LIABILITY FOR FRAUDULENT" + + " MISREPRESENTATION, OR (III) ANY LIABILITY OF CISCO WHICH CANNOT BE EXCLUDED UNDER APPLICABLE LAW.\n" + + "\n" + + "Disclaimer of Liabilities-Waiver of Consequential Damages and Other Losses. IF YOU ACQUIRED THE SOFTWARE IN THE UNITED STATES, LATIN AMERICA, THE CARIBBEAN OR CANADA, REGARDLESS OF WHETHER ANY REMEDY SET FORTH HEREIN FAILS OF ITS ESSENTIAL PURPOSE OR OTHERWISE, IN NO EVENT WILL CISCO OR ITS SUPPLIERS BE LIABLE FOR ANY LOST REVENUE, PROFIT, OR LOST OR DAMAGED DATA, BUSINESS INTERRUPTION, LOSS OF CAPITAL, OR FOR SPECIAL, INDIRECT, CONSEQUENTIAL, INCIDENTAL, OR PUNITIVE DAMAGES HOWEVER CAUSED AND REGARDLESS OF THE THEORY OF LIABILITY OR WHETHER ARISING OUT OF THE USE OF OR INABILITY TO USE SOFTWARE OR OTHERWISE AND EVEN IF CISCO OR ITS SUPPLIERS OR LICENSORS HAVE BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. BECAUSE SOME STATES OR JURISDICTIONS DO NOT ALLOW LIMITATION OR EXCLUSION OF CONSEQUENTIAL OR INCIDENTAL DAMAGES, THE ABOVE LIMITATION MAY NOT APPLY TO YOU.\n" + + "\n" + + "IF YOU ACQUIRED THE SOFTWARE IN JAPAN, EXCEPT FOR LIABILITY ARISING OUT OF OR IN CONNECTION WITH DEATH OR PERSONAL INJURY, FRAUDULENT MISREPRESENTATION, AND REGARDLESS OF WHETHER ANY REMEDY SET FORTH HEREIN FAILS OF ITS ESSENTIAL PURPOSE OR OTHERWISE, IN NO EVENT WILL CISCO, ITS AFFILIATES, OFFICERS, DIRECTORS, EMPLOYEES, AGENTS, SUPPLIERS AND LICENSORS BE LIABLE FOR ANY LOST REVENUE, PROFIT, OR LOST OR DAMAGED DATA, BUSINESS INTERRUPTION, LOSS OF CAPITAL, OR FOR SPECIAL, INDIRECT, CONSEQUENTIAL, INCIDENTAL, OR PUNITIVE DAMAGES HOWEVER CAUSED AND REGARDLESS OF THE THEORY OF LIABILITY OR WHETHER ARISING OUT OF THE USE OF OR INABILITY TO USE SOFTWARE OR OTHERWISE AND EVEN IF CISCO OR ANY APPROVED SOURCE OR THEIR SUPPLIERS OR LICENSORS HAVE BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.\n" + + "\n" + + "IF YOU ACQUIRED THE SOFTWARE IN EUROPE, THE MIDDLE EAST, AFRICA, ASIA OR OCEANIA, IN NO EVENT WILL CISCO, ITS AFFILIATES, OFFICERS, DIRECTORS, EMPLOYEES, AGENTS, SUPPLIERS AND LICENSORS, BE LIABLE FOR ANY LOST REVENUE, LOST PROFIT, OR LOST OR DAMAGED DATA, BUSINESS INTERRUPTION, LOSS OF CAPITAL, OR FOR SPECIAL, INDIRECT, CONSEQUENTIAL, INCIDENTAL, OR PUNITIVE DAMAGES, HOWSOEVER ARISING, INCLUDING, WITHOUT LIMITATION, IN CONTRACT, TORT (INCLUDING NEGLIGENCE) OR WHETHER ARISING OUT OF THE USE OF OR INABILITY TO USE THE SOFTWARE, EVEN IF, IN EACH CASE, CISCO, ITS AFFILIATES, OFFICERS, DIRECTORS, EMPLOYEES, AGENTS, SUPPLIERS AND LICENSORS, HAVE BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. BECAUSE SOME STATES OR JURISDICTIONS DO NOT ALLOW LIMITATION OR EXCLUSION OF CONSEQUENTIAL OR INCIDENTAL DAMAGES, THE ABOVE LIMITATION MAY NOT FULLY APPLY TO YOU. THE FOREGOING EXCLUSION SHALL NOT APPLY TO ANY LIABILITY ARISING OUT OF OR IN " + + "CONNECTION WITH: (I) DEATH OR PERSONAL INJURY, (II) FRAUDULENT MISREPRESENTATION, OR (III) CISCO'S LIABILITY IN CONNECTION WITH ANY TERMS THAT CANNOT BE EXCLUDED UNDER APPLICABLE LAW.\n" + + "\n" + + "Customer acknowledges and agrees that Cisco has set its prices and entered into the Agreement in reliance upon the disclaimers of warranty and the limitations of liability set forth herein, that the same reflect an allocation of risk between the parties (including the risk that a contract remedy may fail of its essential purpose and cause consequential loss), and that the same form an essential basis of the bargain between the parties.\n" + + "\n" + + "Controlling Law, Jurisdiction. If you acquired, by reference to the address on the purchase order accepted by the Approved Source, the Software in the United States, Latin America, or the Caribbean, the Agreement and warranties (\"Warranties\") are controlled by and construed under the laws of the State of California, United States of America, notwithstanding any conflicts of law provisions; and the state and federal courts of California shall have exclusive jurisdiction over any claim arising under the Agreement or Warranties. If you acquired the Software in Canada, unless expressly prohibited by local law, the Agreement and Warranties are controlled by and construed under the laws of the Province of Ontario, Canada, notwithstanding any conflicts of law provisions; and the courts of the Province of Ontario shall have exclusive jurisdiction over any claim arising under the Agreement or Warranties. If you acquired the Software in " + + "Europe, the Middle East, Africa, Asia or Oceania (excluding Australia), unless expressly prohibited by local law, the Agreement and Warranties are controlled by and construed under the laws of England, notwithstanding any conflicts of law provisions; and the English courts shall have exclusive jurisdiction over any claim arising under the Agreement or Warranties. In addition, if the Agreement is controlled by the laws of England, no person who is not a party to the Agreement shall be entitled to enforce or take the benefit of any of its terms under the Contracts (Rights of Third Parties) Act 1999. If you acquired the Software in Japan, unless expressly prohibited by local law, the Agreement and Warranties are controlled by and construed under the laws of Japan, notwithstanding any conflicts of law provisions; and the Tokyo District Court of Japan shall have exclusive jurisdiction over any claim arising under the Agreement or Warranties. " + + "If you acquired the Software in Australia, unless expressly prohibited by local law, the Agreement and Warranties are controlled by and construed under the laws of the State of New South Wales, Australia, notwithstanding any conflicts of law provisions; and the State and federal courts of New South Wales shall have exclusive jurisdiction over any claim arising under the Agreement or Warranties. If you acquired the Software in any other country, unless expressly prohibited by local law, the Agreement and Warranties are controlled by and construed under the laws of the State of California, United States of America, notwithstanding any conflicts of law provisions; and the state and federal courts of California shall have exclusive jurisdiction over any claim arising under the Agreement or Warranties.\n" + + "\n" + + "For all countries referred to above, the parties specifically disclaim the application of the UN Convention on Contracts for the International Sale of Goods. Notwithstanding the foregoing, either party may seek interim injunctive relief in any court of appropriate jurisdiction with respect to any alleged breach of such party's intellectual property or proprietary rights. If any portion hereof is found to be void or unenforceable, the remaining provisions of the Agreement and Warranties shall remain in full force and effect. Except as expressly provided herein, the Agreement constitutes the entire agreement between the parties with respect to the license of the Software and Documentation and supersedes any conflicting or additional terms contained in any Purchase Order or elsewhere, all of which terms are excluded. The Agreement has been written in the English language, and the parties agree that the English version will govern.\n" + + "\n" + + "Product warranty terms and other information applicable to Cisco products are available at the following URL: www.cisco.com/go/warranty\n" + + "\n" + + "Cisco and the Cisco logo are trademarks or registered trademarks of Cisco and/or its affiliates in the U.S. and other countries. To view a list of Cisco trademarks, go to this URL: www.cisco.com/go/trademarks. Third-party trademarks mentioned are the property of their respective owners. The use of the word partner does not imply a partnership relationship between Cisco and any other company. (1110R)\n" + + "\n" + + "© 1998, 2001, 2003, 2008-2014 Cisco Systems, Inc. All rights reserved.\n" + + "\n" + + "\n" + + " supplemental end-user license agreement\n" + + " SUPPLEMENTAL END USER LICENSE AGREEMENT FOR VIRTUAL SOFTWARE PRODUCTS\n" + + "\n" + + "IMPORTANT: READ CAREFULLY\n" + + "\n" + + "This Supplemental End User License Agreement (\"SEULA\") contains additional terms and conditions for the Software licensed under the End User License Agreement (\"EULA\") between you and Cisco (collectively, the \"Agreement\"). Capitalized terms used in this SEULA but not defined will have the meanings assigned to them in the EULA. To the extent that there is a conflict between the terms and conditions of the EULA and this SEULA, the terms and conditions of this SEULA will take precedence. In addition to the limitations set forth in the EULA on your access and use of the Software, you agree to comply at all times with the terms and conditions provided in this SEULA.\n" + + "\n" + + "DOWNLOADING, INSTALLING, OR USING THE SOFTWARE CONSTITUTES ACCEPTANCE OF THE AGREEMENT, AND YOU ARE BINDING YOURSELF AND THE BUSINESS ENTITY THAT YOU REPRESENT (COLLECTIVELY, \"CUSTOMER\") TO THE AGREEMENT. IF YOU DO NOT AGREE TO ALL OF THE TERMS OF THE AGREEMENT, THEN CISCO IS UNWILLING TO LICENSE THE SOFTWARE TO YOU AND (A) YOU MAY NOT DOWNLOAD, INSTALL OR USE THE SOFTWARE, AND (B) YOU MAY RETURN THE SOFTWARE (INCLUDING ANY UNOPENED CD PACKAGE AND ANY WRITTEN MATERIALS) FOR A FULL REFUND, OR, IF THE SOFTWARE AND WRITTEN MATERIALS ARE SUPPLIED AS PART OF ANOTHER PRODUCT, YOU MAY RETURN THE ENTIRE PRODUCT FOR A FULL REFUND. YOUR RIGHT TO RETURN AND REFUND EXPIRES 30 DAYS AFTER PURCHASE FROM CISCO OR AN AUTHORIZED CISCO RESELLER, AND APPLIES ONLY IF YOU ARE THE ORIGINAL END USER PURCHASER.\n" + + "\n" + + "Definitions\n" + + "\"CPU\" means a central processing unit that encompasses part of a Server.\n" + + "\"Failover Pair\" means a primary Instance and a standby Instance with the same Software configuration where the standby Instance can take over in case of failure of the primary Instance.\n" + + "\"Instance\" means a single copy of the Software. Each copy of the Software loaded into memory is an Instance.\n" + + "\"Server\" means a single physical computer or device on a network that manages or provides network resources for multiple users.\n" + + "\"Service Provider\" means a company that provides information technology services to external end user customers.\n" + + "\"Software\" means Cisco's Adaptive Security Virtual Appliance (\"ASAv\"), Adaptive Security Appliance 1000V Cloud Firewall Software (\"ASA 1000V\"), Nexus 1000V series switch products, Virtual Security Gateway products, or other Cisco virtual software products that Cisco includes under this SEULA.\n" + + "\"vCPU\" means a virtual central processing resource assigned to the VM by the underlying virtualization technology.\n" + + "\"Virtual Machine\" or \"VM\" means a software container that can run its own operating system and execute applications like a Server.\n" + + "\n" + + "Additional License Terms and Conditions\n" + + "1. Cisco hereby grants Customer the right to install and use the Software on single or multiple Cisco or non-Cisco Servers or on Virtual Machines. In order to use the Software Customer may be required to input a registration number or product activation key and register each Instance online at Cisco's website in order to obtain the necessary entitlements.\n" + + "2. Customer shall pay a unit license fee to Cisco or an authorized Cisco reseller, as applicable, for each Instance installed on a Cisco or non-Cisco Server CPU, vCPU or Virtual Machine, as determined by Cisco.\n" + + "3. For the ASA 1000V, Customer is licensed the number of Instances equal to the number of CPUs covered by the unit license fee. If Customer deploys a Failover Pair, then the fee for the additional standby Instance is included in the fee for each primary Instance.\n" + + "4. If Customer is a Service Provider, Customer may use the Software under the terms of this Agreement for the purpose of delivering hosted information technology services to Customer's end user customers, subject to payment of the required license fee(s).\n" + + "5. Customer may also use the Software under the terms of this Agreement to deliver hosted information technology services to Customer affiliates, subject to payment of the required license fee(s).\n" + + "6. If the Software is subject to Cisco's Smart Licensing program, Cisco will be able to assess if Customer is using the Software within the limits and entitlements paid for by Customer. If the Smart Licensing program is applicable, Customer will be required to enter into a separate terms of service agreement relating to Smart Licensing.\n" + + "\n" + + ""; + + private String productSectionWithCategories = + "\n" + + "\n" + + " Appliance ISV branding information\n" + + " VMware vCenter Server Appliance\n" + + " VMware Inc.\n" + + " \n" + + "\n" + + "\n" + + " 6.7.0.44000\n" + + " 6.7.0.44000 build 16046470\n" + + " \n" + + " http://www.vmware.com\n" + + " https://${vami.ip0.VMware_vCenter_Server_Appliance}:5480/\n" + + " Application\n" + + " Networking Configuration\n" + + " \n" + + " \n" + + " Network IP address family (i.e., 'ipv4' or 'ipv6').\n" + + " \n" + + " \n" + + " \n" + + " Network mode (i.e., 'static', 'dhcp', or 'autoconf' (IPv6 only).\n" + + " \n" + + " \n" + + " \n" + + " Network IP address. Only provide this when mode is 'static'. Can be IPv4 or IPv6 based on specified address family.\n" + + " \n" + + " \n" + + " \n" + + " Network prefix length. Only provide this when mode is 'static'. 0-32 for IPv4. 0-128 for IPv6.\n" + + " \n" + + " \n" + + " \n" + + " IP address of default gateway. Can be 'default' when using IPv6.\n" + + " \n" + + " \n" + + " \n" + + " Comma separated list of IP addresses of DNS servers.\n" + + " \n" + + " \n" + + " \n" + + " Network identity (IP address or fully-qualified domain name) services should use when advertising themselves.\n" + + " \n" + + " \n" + + " \n" + + " A string encoding a JSON object mapping port names to port numbers.\n" + + " \n" + + " SSO Configuration\n" + + " \n" + + " \n" + + " For the first instance of the identity domain, this is the username with Administrator privileges. Otherwise, this is the username of the replication partner.\n" + + " \n" + + " \n" + + " \n" + + " For the first instance of the identity domain, this is the password given to the Administrator account. Otherwise, this is the password of the Administrator account of the replication partner.\n" + + " \n" + + " \n" + + " \n" + + " For the first instance of the identity domain, this is the name of the newly created domain.\n" + + " \n" + + " \n" + + " \n" + + " Name of site. Use 'Default-First-Site' to define a new site.\n" + + " \n" + + " \n" + + " \n" + + " If this parameter is set to True, the VMware directory instance is setup as the first instance of a new identity domain. Otherwise, the instance is setup as a replication partner.\n" + + " \n" + + " \n" + + " \n" + + " The hostname of the VMware directory replication partner. This value is ignored for the first instance of the identity domain.\n" + + " \n" + + " Database Configuration\n" + + " \n" + + " \n" + + " String indicating whether the database is 'embedded' or 'external'.\n" + + " \n" + + " \n" + + " \n" + + " String naming the account to use when connecting to external database (ignored when db.type is 'embedded').\n" + + " \n" + + " \n" + + " \n" + + " String providing the password to use when connecting to external database (ignored when db.type is 'embedded').\n" + + " \n" + + " \n" + + " \n" + + " String naming the the hostname of the server on which the external database is running (ignored when db.type is 'embedded').\n" + + " \n" + + " \n" + + " \n" + + " String describing the port on the host on which the external database is running (ignored when db.type is 'embedded').\n" + + " \n" + + " \n" + + " \n" + + " String describing the external database provider. The only supported value is 'oracle' (ignored when the db.type is 'embedded').\n" + + " \n" + + " \n" + + " \n" + + " String describing the external database instance. Values could be anything depending on what the database instance name the DBA creates in the external db. (ignored when the db.type is 'embedded').\n" + + " \n" + + " System Configuration\n" + + " \n" + + " \n" + + " Password to assign to root account. If blank, password can be set on the console.\n" + + " \n" + + " \n" + + " \n" + + " This property is not changeable.\n" + + " \n" + + " \n" + + " \n" + + " Set whether SSH-based remote login is enabled. This configuration can be changed after deployment.\n" + + " \n" + + " \n" + + " \n" + + " Set whether VMware tools based time synchronization should be used. This parameter is ignored if appliance.ntp.servers is not empty.\n" + + " \n" + + " \n" + + " \n" + + " A comma-seperated list of hostnames or IP addresses of NTP Servers\n" + + " \n" + + " \n" + + " \n" + + " Type of appliance to deploy (i.e. 'embedded', 'infrastructure' or 'management').\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " When deploying a vCenter Server Node, please provide the FQDN or IP address of a Platform Services Controller (leave blank otherwise). The choice of FQDN versus IP address is decided based on the Platform Services Controller's own notion of its network identity.\n" + + " \n" + + " \n" + + " \n" + + " When deploying a vCenter Server pointing to an external platform services controller, please provide the HTTPS port of the external platform services controller if a custom port number is being used. The default HTTPS port number is 443.\n" + + " \n" + + " Upgrade Configuration\n" + + " \n" + + " \n" + + " IP/hostname of the appliance to upgrade. Set only for upgrade.\n" + + " \n" + + " \n" + + " \n" + + " Port used by Migration Assistant on source vCenter Server.\n" + + " \n" + + " \n" + + " \n" + + " vCenter username for the appliance to upgrade. Set only for upgrade.\n" + + " \n" + + " \n" + + " \n" + + " vCenter password for the appliance to upgrade. Set only for upgrade.\n" + + " \n" + + " \n" + + " \n" + + " Username for the appliance operating system to upgrade. Usually root. Set only for upgrade.\n" + + " \n" + + " \n" + + " \n" + + " Password for the appliance operating system to upgrade. Set only for upgrade.\n" + + " \n" + + " \n" + + " \n" + + " URL that consists of the IP address or FQDN and https port of the vCenter Server instance or ESXi host that manages the appliance to upgrade. Https port is an optional parameter which by default is 443. Example: 10.10.10.10, //10.10.10.10:444, //[2001:db8:a0b:12f0::1]:444. Set only for upgrade.\n" + + " \n" + + " \n" + + " \n" + + " Username for the host that manages appliance to upgrade. Can be either vCenter or ESX host. Set only for upgrade.\n" + + " \n" + + " \n" + + " \n" + + " Password for the host that manages appliance to upgrade. Can be either vCenter or ESX host. Set only for upgrade.\n" + + " \n" + + " \n" + + " \n" + + " Thumbprint for the SSL certificate of the host that manages the appliance to upgrade. Set only for upgrade.\n" + + " \n" + + " \n" + + " \n" + + " Source host platform. Optional. Set only for upgrade\n" + + " \n" + + " \n" + + " \n" + + " Folder on the source appliance, where to store migrate data. Optional. Set only for upgrade\n" + + " \n" + + " \n" + + " \n" + + " Folder where exported source data will be stored in the appliance. Optional. Set only for upgrade\n" + + " \n" + + " \n" + + " \n" + + " Advanced upgrade settings specified in json format. Optional. Set only for upgrade\n" + + " \n" + + " \n" + + " \n" + + " Active Directory domain to join.\n" + + " \n" + + " \n" + + " \n" + + " Active Directory domain admin user. This username will be used to join the machine to the domain.\n" + + " \n" + + " \n" + + " \n" + + " Active Directory domain admin user password. This password will be used to join the machine to the domain.\n" + + " \n" + + " \n" + + " \n" + + " FQDN or IP address of the vCenter Server managing that target appliance. Used when upgrading a source appliance in VCHA cluster.\n" + + " \n" + + " \n" + + " \n" + + " Https port of the vCenter Server managing that target appliance. Used when upgrading a source appliance in VCHA cluster. If not specified, port 443 will be used by default.\n" + + " \n" + + " \n" + + " \n" + + " User able to authenticate in vCenter Server managing that target appliance. The user must have the privilege Global.VCServer. Used when upgrading a source appliance in VCHA cluster.\n" + + " \n" + + " \n" + + " \n" + + " Password for administrator user authenticating to the vCenter Server managing target appliance. Used when upgrading a source appliance in VCHA cluster.\n" + + " \n" + + " \n" + + " \n" + + " Thumbprint for the SSL certificate of the host that manages the appliance to upgrade. Used when upgrading a source appliance in VCHA cluster.\n" + + " \n" + + " \n" + + " \n" + + " Path to host/cluster/resource pool where target appliance will be deployed on management vCenter Server. Used when upgrading a source appliance in VCHA cluster. Example: /my_datacenter/my_folder/my_host_or_cluster/my_resource_pool\n" + + " \n" + + " Miscellaneous\n" + + " \n" + + " \n" + + " Set whether ESXi Dump Collector service is enabled. This configuration can be changed after deployment.\n" + + " \n" + + " \n" + + " \n" + + " If this parameter is set to True, no questions will be posted during install or upgrade. Otherwise, the install process will wait for a reply if there is a pending question.\n" + + " \n" + + " \n" + + " \n" + + " This parameter specifies the client locale. Supported locales are en, fr, ja, ko, zh_CN and zh_TW. English is assumed if locale is unknown.\n" + + " \n" + + " \n" + + " \n" + + " Specify feature switch states which need to be added or modified in feature switch state config file. Format: key1=value1, key2=value2\n" + + " \n" + + " \n" + + " \n" + + " VMware’s Customer Experience Improvement Program ("CEIP") provides VMware with information that enables VMware to improve its products and services, to fix problems, and to advise you on how best to deploy and use our products. As part of the CEIP, VMware collects technical information about your organization’s use of VMware products and services on a regular basis in association with your organization’s VMware license key(s). This information does not personally identify any individual. For more details about the Program and how VMware uses the information it collects through CEIP, please see the product documentation at http://www.vmware.com/info?id=1399. If you want to participate in VMware’s CEIP for this product, set this property to True. You may join or leave VMware’s CEIP for this product at any time.\n" + + " \n" + + " \n" + + " \n" + + " If this parameter is set to True, the appliance will be configured after deployment using the specified OVF configuration parameters. If set to False, the appliance should be configured post-deployment using the VMware Appliance Management Interface.\n" + + " \n" + + " \n" + + " \n" + + " If a valid MAC address prefix is provided, then all MAC addresses assigned by vCenter Server will begin with this prefix instead of the VMware OUI. This property cannot co-exist with mac-allocation-scheme.ranges\n" + + " \n" + + " \n" + + " \n" + + " This property is mandatory whenever a custom MAC prefix is provided.\n" + + " \n" + + " \n" + + " \n" + + " If valid MAC address range is provided, then vCenter Server will assign MAC addresses from this range instead of allocating VMware OUI based MAC address. The address range must be provided in the format "BeginAddress1-EndAddress1,...,BeginAddressN-EndAddressN". This property cannot co-exist with mac-allocation-scheme.prefix.\n" + + " \n" + + "\n" + + "\n" + + " VAMI Properties\n" + + " Networking Properties\n" + + " \n" + + " \n" + + " The domain name of this VM. Leave blank if DHCP is desired.\n" + + " \n" + + " \n" + + " \n" + + " The domain search path (comma or space separated domain names) for this VM. Leave blank if DHCP is desired.\n" + + " \n" + + "\n" + + "\n" + + " VM specific properties\n" + + " \n" + + "\n" + + ""; + private OVFHelper ovfHelper = new OVFHelper(); @Test public void testGetOVFPropertiesValidOVF() throws IOException, SAXException, ParserConfigurationException { - List props = ovfHelper.getOVFPropertiesXmlString(ovfFileProductSection); + List props = ovfHelper.getOVFPropertiesFromXmlString(ovfFileProductSection); Assert.assertEquals(2, props.size()); } @Test(expected = SAXParseException.class) public void testGetOVFPropertiesInvalidOVF() throws IOException, SAXException, ParserConfigurationException { - ovfHelper.getOVFPropertiesXmlString(ovfFileProductSection + "xxxxxxxxxxxxxxxxx"); + ovfHelper.getOVFPropertiesFromXmlString(ovfFileProductSection + "xxxxxxxxxxxxxxxxx"); + } + + @Test + public void testGetOVFDeploymentOptionsValidOVF() throws IOException, SAXException, ParserConfigurationException { + List options = ovfHelper.getOVFDeploymentOptionsFromXmlString(ovfFileDeploymentOptionsSection); + Assert.assertEquals(3, options.size()); + } + + @Test + public void testGetOVFVirtualHardwareSectionValidOVF() throws IOException, SAXException, ParserConfigurationException { + List items = ovfHelper.getOVFVirtualHardwareSectionFromXmlString(ovfFileVirtualHardwareSection); + Assert.assertEquals(20, items.size()); + } + + @Test + public void testGetOVFEulaSectionValidOVF() throws IOException, SAXException, ParserConfigurationException { + List eulas = ovfHelper.getOVFEulaSectionFromXmlString(eulaSections); + Assert.assertEquals(2, eulas.size()); + } + + @Test + public void testGetOVFPropertiesWithCategories() throws IOException, SAXException, ParserConfigurationException { + List props = ovfHelper.getOVFPropertiesFromXmlString(productSectionWithCategories); + Assert.assertEquals(18, props.size()); } } diff --git a/api/src/test/java/com/cloud/storage/StorageTest.java b/api/src/test/java/com/cloud/storage/StorageTest.java index 332a8060d08d..61909e72e96e 100644 --- a/api/src/test/java/com/cloud/storage/StorageTest.java +++ b/api/src/test/java/com/cloud/storage/StorageTest.java @@ -45,6 +45,7 @@ public void isSharedStoragePool() { Assert.assertTrue(StoragePoolType.SMB.isShared()); Assert.assertTrue(StoragePoolType.Gluster.isShared()); Assert.assertTrue(StoragePoolType.ManagedNFS.isShared()); + Assert.assertTrue(StoragePoolType.DatastoreCluster.isShared()); } @Test @@ -66,5 +67,6 @@ public void supportsOverprovisioningStoragePool() { Assert.assertFalse(StoragePoolType.SMB.supportsOverProvisioning()); Assert.assertFalse(StoragePoolType.Gluster.supportsOverProvisioning()); Assert.assertFalse(StoragePoolType.ManagedNFS.supportsOverProvisioning()); + Assert.assertTrue(StoragePoolType.DatastoreCluster.supportsOverProvisioning()); } } diff --git a/core/pom.xml b/core/pom.xml index ff63e50c4a0a..d33d6866f7f0 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -51,9 +51,5 @@ commons-codec commons-codec - - org.apache.commons - commons-compress - diff --git a/core/src/main/java/com/cloud/agent/api/ModifyStoragePoolAnswer.java b/core/src/main/java/com/cloud/agent/api/ModifyStoragePoolAnswer.java index 6e6dadc67f7a..be84cce152d5 100644 --- a/core/src/main/java/com/cloud/agent/api/ModifyStoragePoolAnswer.java +++ b/core/src/main/java/com/cloud/agent/api/ModifyStoragePoolAnswer.java @@ -19,6 +19,8 @@ package com.cloud.agent.api; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import com.cloud.storage.template.TemplateProp; @@ -27,6 +29,8 @@ public class ModifyStoragePoolAnswer extends Answer { private StoragePoolInfo poolInfo; private Map templateInfo; private String localDatastoreName; + private String poolType; + private List datastoreClusterChildren = new ArrayList<>();; public ModifyStoragePoolAnswer(ModifyStoragePoolCommand cmd, long capacityBytes, long availableBytes, Map tInfo) { super(cmd); @@ -61,4 +65,20 @@ public void setLocalDatastoreName(String localDatastoreName) { public String getLocalDatastoreName() { return localDatastoreName; } + + public String getPoolType() { + return poolType; + } + + public void setPoolType(String poolType) { + this.poolType = poolType; + } + + public List getDatastoreClusterChildren() { + return datastoreClusterChildren; + } + + public void setDatastoreClusterChildren(List datastoreClusterChildren) { + this.datastoreClusterChildren = datastoreClusterChildren; + } } diff --git a/core/src/main/java/com/cloud/agent/api/ValidateVcenterDetailsCommand.java b/core/src/main/java/com/cloud/agent/api/ValidateVcenterDetailsCommand.java new file mode 100644 index 000000000000..d1d1393efe47 --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/ValidateVcenterDetailsCommand.java @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.agent.api; + +public class ValidateVcenterDetailsCommand extends Command { + + String vCenterServerAddress; + + public ValidateVcenterDetailsCommand(String vCenterServerAddress) { + this.vCenterServerAddress = vCenterServerAddress; + } + + public String getvCenterServerAddress() { + return vCenterServerAddress; + } + + public void setvCenterServerAddress(String vCenterServerAddress) { + this.vCenterServerAddress = vCenterServerAddress; + } + + @Override + public boolean executeInSequence() { + return false; + } +} diff --git a/core/src/main/java/com/cloud/agent/api/storage/DownloadAnswer.java b/core/src/main/java/com/cloud/agent/api/storage/DownloadAnswer.java index 9859c3f83d01..9ae1d6dcca54 100644 --- a/core/src/main/java/com/cloud/agent/api/storage/DownloadAnswer.java +++ b/core/src/main/java/com/cloud/agent/api/storage/DownloadAnswer.java @@ -25,8 +25,13 @@ import com.cloud.agent.api.Answer; import com.cloud.agent.api.Command; import com.cloud.agent.api.LogLevel; +import com.cloud.agent.api.to.DatadiskTO; +import com.cloud.agent.api.to.deployasis.OVFEulaSectionTO; +import com.cloud.agent.api.to.deployasis.OVFPropertyTO; +import com.cloud.agent.api.to.deployasis.OVFVirtualHardwareSectionTO; import com.cloud.storage.VMTemplateStorageResourceAssoc; import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; +import com.cloud.agent.api.to.deployasis.OVFNetworkTO; public class DownloadAnswer extends Answer { private String jobId; @@ -38,8 +43,17 @@ public class DownloadAnswer extends Answer { private long templateSize = 0L; private long templatePhySicalSize = 0L; private String checkSum; + @LogLevel(LogLevel.Log4jLevel.Off) private List ovfProperties; + @LogLevel(LogLevel.Log4jLevel.Off) + private List networkRequirements; + @LogLevel(LogLevel.Log4jLevel.Off) + private List disks; + @LogLevel(LogLevel.Log4jLevel.Off) + private OVFVirtualHardwareSectionTO ovfHardwareSection; + @LogLevel(LogLevel.Log4jLevel.Off) + private List eulaSections; public String getCheckSum() { return checkSum; @@ -157,4 +171,36 @@ public List getOvfProperties() { public void setOvfProperties(List ovfProperties) { this.ovfProperties = ovfProperties; } + + public List getNetworkRequirements() { + return networkRequirements; + } + + public void setNetworkRequirements(List networkRequirements) { + this.networkRequirements = networkRequirements; + } + + public List getDisks() { + return disks; + } + + public void setDisks(List disks) { + this.disks = disks; + } + + public OVFVirtualHardwareSectionTO getOvfHardwareSection() { + return ovfHardwareSection; + } + + public void setOvfHardwareSection(OVFVirtualHardwareSectionTO ovfHardwareSection) { + this.ovfHardwareSection = ovfHardwareSection; + } + + public List getEulaSections() { + return eulaSections; + } + + public void setEulaSections(List eulaSections) { + this.eulaSections = eulaSections; + } } diff --git a/core/src/main/java/com/cloud/agent/api/storage/GetDatadisksCommand.java b/core/src/main/java/com/cloud/agent/api/storage/GetDatadisksCommand.java index 0e22ea25e780..a6dfbfef99a0 100644 --- a/core/src/main/java/com/cloud/agent/api/storage/GetDatadisksCommand.java +++ b/core/src/main/java/com/cloud/agent/api/storage/GetDatadisksCommand.java @@ -21,10 +21,12 @@ public final class GetDatadisksCommand extends Command { private DataTO data; + private String configurationId; - public GetDatadisksCommand(DataTO data) { + public GetDatadisksCommand(DataTO data, String configurationId) { super(); this.data = data; + this.configurationId = configurationId; } protected GetDatadisksCommand() { @@ -40,4 +42,7 @@ public DataTO getData() { return data; } + public String getConfigurationId() { + return configurationId; + } } \ No newline at end of file diff --git a/core/src/main/java/com/cloud/resource/ServerResource.java b/core/src/main/java/com/cloud/resource/ServerResource.java index 9030db72f00b..16ac00ed176b 100644 --- a/core/src/main/java/com/cloud/resource/ServerResource.java +++ b/core/src/main/java/com/cloud/resource/ServerResource.java @@ -31,6 +31,7 @@ * ServerResource is a generic container to execute commands sent */ public interface ServerResource extends Manager { + /** * @return Host.Type type of the computing server we have. */ diff --git a/core/src/main/java/com/cloud/storage/resource/StorageProcessor.java b/core/src/main/java/com/cloud/storage/resource/StorageProcessor.java index f940e22f45a1..d86a1a619a9b 100644 --- a/core/src/main/java/com/cloud/storage/resource/StorageProcessor.java +++ b/core/src/main/java/com/cloud/storage/resource/StorageProcessor.java @@ -21,6 +21,7 @@ import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand; import org.apache.cloudstack.storage.command.AttachCommand; +import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplainceCommand; import org.apache.cloudstack.storage.command.CopyCommand; import org.apache.cloudstack.storage.command.CreateObjectCommand; import org.apache.cloudstack.storage.command.DeleteCommand; @@ -33,6 +34,9 @@ import com.cloud.agent.api.Answer; public interface StorageProcessor { + + String REQUEST_TEMPLATE_RELOAD = "request template reload"; + public Answer copyTemplateToPrimaryStorage(CopyCommand cmd); public Answer cloneVolumeFromBaseTemplate(CopyCommand cmd); @@ -76,4 +80,6 @@ public interface StorageProcessor { public Answer handleDownloadTemplateToPrimaryStorage(DirectDownloadCommand cmd); Answer copyVolumeFromPrimaryToPrimary(CopyCommand cmd); + + public Answer CheckDataStoreStoragePolicyComplaince(CheckDataStoreStoragePolicyComplainceCommand cmd); } diff --git a/core/src/main/java/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java b/core/src/main/java/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java index 17b9b700d6c0..910eb3d87905 100644 --- a/core/src/main/java/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java +++ b/core/src/main/java/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java @@ -21,6 +21,7 @@ import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand; import org.apache.cloudstack.storage.to.VolumeObjectTO; +import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplainceCommand; import org.apache.log4j.Logger; import org.apache.cloudstack.storage.command.AttachCommand; @@ -71,6 +72,8 @@ public Answer handleStorageCommands(StorageSubSystemCommand command) { return processor.resignature((ResignatureCommand) command); } else if (command instanceof DirectDownloadCommand) { return processor.handleDownloadTemplateToPrimaryStorage((DirectDownloadCommand) command); + } else if (command instanceof CheckDataStoreStoragePolicyComplainceCommand) { + return processor.CheckDataStoreStoragePolicyComplaince((CheckDataStoreStoragePolicyComplainceCommand) command); } return new Answer((Command)command, false, "not implemented yet"); diff --git a/core/src/main/java/com/cloud/storage/template/OVAProcessor.java b/core/src/main/java/com/cloud/storage/template/OVAProcessor.java index d771c67acec6..e980293b750a 100644 --- a/core/src/main/java/com/cloud/storage/template/OVAProcessor.java +++ b/core/src/main/java/com/cloud/storage/template/OVAProcessor.java @@ -20,13 +20,20 @@ package com.cloud.storage.template; import java.io.File; +import java.io.IOException; import java.util.List; import java.util.Map; import javax.naming.ConfigurationException; import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; -import com.cloud.agent.api.storage.OVFPropertyTO; +import com.cloud.agent.api.to.deployasis.OVFConfigurationTO; +import com.cloud.agent.api.to.deployasis.OVFEulaSectionTO; +import com.cloud.agent.api.to.deployasis.OVFPropertyTO; +import com.cloud.agent.api.to.deployasis.OVFVirtualHardwareItemTO; +import com.cloud.agent.api.to.deployasis.OVFVirtualHardwareSectionTO; +import com.cloud.agent.api.to.deployasis.OVFNetworkTO; import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; import org.w3c.dom.Document; @@ -41,9 +48,13 @@ import com.cloud.utils.Pair; import com.cloud.utils.component.AdapterBase; import com.cloud.utils.script.Script; +import org.xml.sax.SAXException; +/** + * processes the content of an OVA for registration of a template + */ public class OVAProcessor extends AdapterBase implements Processor { - private static final Logger s_logger = Logger.getLogger(OVAProcessor.class); + private static final Logger LOGGER = Logger.getLogger(OVAProcessor.class); StorageLayer _storage; @Override @@ -53,73 +64,137 @@ public FormatInfo process(String templatePath, ImageFormat format, String templa @Override public FormatInfo process(String templatePath, ImageFormat format, String templateName, long processTimeout) throws InternalErrorException { - if (format != null) { - if (s_logger.isInfoEnabled()) { - s_logger.info("We currently don't handle conversion from " + format + " to OVA."); - } + if (! conversionChecks(format)){ return null; } - s_logger.info("Template processing. templatePath: " + templatePath + ", templateName: " + templateName); + LOGGER.info("Template processing. templatePath: " + templatePath + ", templateName: " + templateName); String templateFilePath = templatePath + File.separator + templateName + "." + ImageFormat.OVA.getFileExtension(); if (!_storage.exists(templateFilePath)) { - if (s_logger.isInfoEnabled()) { - s_logger.info("Unable to find the vmware template file: " + templateFilePath); + if (LOGGER.isInfoEnabled()) { + LOGGER.info("Unable to find the vmware template file: " + templateFilePath); } return null; } - s_logger.info("Template processing - untar OVA package. templatePath: " + templatePath + ", templateName: " + templateName); - String templateFileFullPath = templatePath + File.separator + templateName + "." + ImageFormat.OVA.getFileExtension(); - File templateFile = new File(templateFileFullPath); - Script command = new Script("tar", processTimeout, s_logger); - command.add("--no-same-owner"); - command.add("--no-same-permissions"); - command.add("-xf", templateFileFullPath); - command.setWorkDir(templateFile.getParent()); - String result = command.execute(); - if (result != null) { - s_logger.info("failed to untar OVA package due to " + result + ". templatePath: " + templatePath + ", templateName: " + templateName); - throw new InternalErrorException("failed to untar OVA package"); + String templateFileFullPath = unpackOva(templatePath, templateName, processTimeout); + + setFileSystemAccessRights(templatePath); + + FormatInfo info = createFormatInfo(templatePath, templateName, templateFilePath, templateFileFullPath); + + return info; + } + + private FormatInfo createFormatInfo(String templatePath, String templateName, String templateFilePath, String templateFileFullPath) throws InternalErrorException { + FormatInfo info = new FormatInfo(); + info.format = ImageFormat.OVA; + info.filename = templateName + "." + ImageFormat.OVA.getFileExtension(); + info.size = _storage.getSize(templateFilePath); + info.virtualSize = getTemplateVirtualSize(templatePath, info.filename); + validateOva(templateFileFullPath, info); + + return info; + } + + /** + * side effect; properties are added to the info + * + * @throws InternalErrorException on an invalid ova contents + */ + private void validateOva(String templateFileFullPath, FormatInfo info) throws InternalErrorException { + String ovfFilePath = getOVFFilePath(templateFileFullPath); + OVFHelper ovfHelper = new OVFHelper(); + Document doc = ovfHelper.getDocumentFromFile(ovfFilePath); + + List disks = ovfHelper.getOVFVolumeInfoFromFile(ovfFilePath, doc, null); + if (LOGGER.isTraceEnabled()) { + LOGGER.trace(String.format("Found %d disks in template %s", CollectionUtils.isNotEmpty(disks) ? disks.size() : 0, ovfFilePath)); + } + if (CollectionUtils.isNotEmpty(disks)) { + info.disks = disks; + } + + List nets = ovfHelper.getNetPrerequisitesFromDocument(doc); + if (CollectionUtils.isNotEmpty(nets)) { + LOGGER.info("Found " + nets.size() + " prerequisite networks"); + info.networks = nets; + } else if (LOGGER.isTraceEnabled()) { + LOGGER.trace(String.format("no net prerequisites found in template %s", ovfFilePath)); + } + + List ovfProperties = ovfHelper.getConfigurableOVFPropertiesFromDocument(doc); + if (CollectionUtils.isNotEmpty(ovfProperties)) { + LOGGER.info("Found " + ovfProperties.size() + " configurable OVF properties"); + info.ovfProperties = ovfProperties; + } else if (LOGGER.isTraceEnabled()) { + LOGGER.trace(String.format("no ovf properties found in template %s", ovfFilePath)); } - command = new Script("chmod", 0, s_logger); + OVFVirtualHardwareSectionTO hardwareSection = ovfHelper.getVirtualHardwareSectionFromDocument(doc); + List configurations = hardwareSection.getConfigurations(); + if (CollectionUtils.isNotEmpty(configurations)) { + LOGGER.info("Found " + configurations.size() + " deployment option configurations"); + } + List hardwareItems = hardwareSection.getCommonHardwareItems(); + if (CollectionUtils.isNotEmpty(hardwareItems)) { + LOGGER.info("Found " + hardwareItems.size() + " virtual hardware items"); + } + info.hardwareSection = hardwareSection; + List eulaSections = ovfHelper.getEulaSectionsFromDocument(doc); + if (CollectionUtils.isNotEmpty(eulaSections)) { + LOGGER.info("Found " + eulaSections.size() + " license agreements"); + info.eulaSections = eulaSections; + } + } + + private void setFileSystemAccessRights(String templatePath) { + Script command; + String result; + + command = new Script("chmod", 0, LOGGER); command.add("-R"); command.add("666", templatePath); result = command.execute(); if (result != null) { - s_logger.warn("Unable to set permissions for files in " + templatePath + " due to " + result); + LOGGER.warn("Unable to set permissions for files in " + templatePath + " due to " + result); } - command = new Script("chmod", 0, s_logger); + command = new Script("chmod", 0, LOGGER); command.add("777", templatePath); result = command.execute(); if (result != null) { - s_logger.warn("Unable to set permissions for " + templatePath + " due to " + result); + LOGGER.warn("Unable to set permissions for " + templatePath + " due to " + result); } + } - FormatInfo info = new FormatInfo(); - info.format = ImageFormat.OVA; - info.filename = templateName + "." + ImageFormat.OVA.getFileExtension(); - info.size = _storage.getSize(templateFilePath); - info.virtualSize = getTemplateVirtualSize(templatePath, info.filename); + private String unpackOva(String templatePath, String templateName, long processTimeout) throws InternalErrorException { + LOGGER.info("Template processing - untar OVA package. templatePath: " + templatePath + ", templateName: " + templateName); + String templateFileFullPath = templatePath + File.separator + templateName + "." + ImageFormat.OVA.getFileExtension(); + File templateFile = new File(templateFileFullPath); + Script command = new Script("tar", processTimeout, LOGGER); + command.add("--no-same-owner"); + command.add("--no-same-permissions"); + command.add("-xf", templateFileFullPath); + command.setWorkDir(templateFile.getParent()); + String result = command.execute(); + if (result != null) { + LOGGER.info("failed to untar OVA package due to " + result + ". templatePath: " + templatePath + ", templateName: " + templateName); + throw new InternalErrorException("failed to untar OVA package"); + } + return templateFileFullPath; + } - //vaidate ova - String ovfFile = getOVFFilePath(templateFileFullPath); - try { - OVFHelper ovfHelper = new OVFHelper(); - List disks = ovfHelper.getOVFVolumeInfo(ovfFile); - List ovfProperties = ovfHelper.getOVFPropertiesFromFile(ovfFile); - if (CollectionUtils.isNotEmpty(ovfProperties)) { - s_logger.info("Found " + ovfProperties.size() + " configurable OVF properties"); - info.ovfProperties = ovfProperties; + private boolean conversionChecks(ImageFormat format) { + if (format != null) { + if (LOGGER.isInfoEnabled()) { + LOGGER.info("We currently don't handle conversion from " + format + " to OVA."); } - } catch (Exception e) { - s_logger.info("The ovf file " + ovfFile + " is invalid ", e); - throw new InternalErrorException("OVA package has bad ovf file " + e.getMessage(), e); + return false; } - // delete original OVA file - // templateFile.delete(); - return info; + if (LOGGER.isTraceEnabled()) { + LOGGER.trace("We are handling format " + format + "."); + } + return true; } @Override @@ -128,34 +203,43 @@ public long getVirtualSize(File file) { long size = getTemplateVirtualSize(file.getParent(), file.getName()); return size; } catch (Exception e) { - s_logger.info("[ignored]" + LOGGER.info("[ignored]" + "failed to get virtual template size for ova: " + e.getLocalizedMessage()); } return file.length(); } + /** + * gets the virtual size from the OVF file meta data. + * + * @return the accumulative virtual size of the disk definitions in the OVF + * @throws InternalErrorException + */ public long getTemplateVirtualSize(String templatePath, String templateName) throws InternalErrorException { - // get the virtual size from the OVF file meta data long virtualSize = 0; String templateFileFullPath = templatePath.endsWith(File.separator) ? templatePath : templatePath + File.separator; templateFileFullPath += templateName.endsWith(ImageFormat.OVA.getFileExtension()) ? templateName : templateName + "." + ImageFormat.OVA.getFileExtension(); String ovfFileName = getOVFFilePath(templateFileFullPath); if (ovfFileName == null) { String msg = "Unable to locate OVF file in template package directory: " + templatePath; - s_logger.error(msg); + LOGGER.error(msg); throw new InternalErrorException(msg); } try { Document ovfDoc = null; ovfDoc = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(new File(ovfFileName)); - Element disk = (Element)ovfDoc.getElementsByTagName("Disk").item(0); - virtualSize = Long.parseLong(disk.getAttribute("ovf:capacity")); - String allocationUnits = disk.getAttribute("ovf:capacityAllocationUnits"); - virtualSize = OVFHelper.getDiskVirtualSize(virtualSize, allocationUnits, ovfFileName); + NodeList diskElements = ovfDoc.getElementsByTagName("Disk"); + for (int i = 0; i < diskElements.getLength(); i++) { + Element disk = (Element)diskElements.item(i); + long diskSize = Long.parseLong(disk.getAttribute("ovf:capacity")); + String allocationUnits = disk.getAttribute("ovf:capacityAllocationUnits"); + diskSize = OVFHelper.getDiskVirtualSize(diskSize, allocationUnits, ovfFileName); + virtualSize += diskSize; + } return virtualSize; - } catch (Exception e) { + } catch (InternalErrorException | IOException | NumberFormatException | ParserConfigurationException | SAXException e) { String msg = "getTemplateVirtualSize: Unable to parse OVF XML document " + templatePath + " to get the virtual disk " + templateName + " size due to " + e; - s_logger.error(msg); + LOGGER.error(msg); throw new InternalErrorException(msg); } } @@ -187,9 +271,9 @@ public Pair getDiskDetails(String ovfFilePath, String diskName) thro } } return new Pair(virtualSize, fileSize); - } catch (Exception e) { + } catch (InternalErrorException | IOException | NumberFormatException | ParserConfigurationException | SAXException e) { String msg = "getDiskDetails: Unable to parse OVF XML document " + ovfFilePath + " to get the virtual disk " + diskName + " size due to " + e; - s_logger.error(msg); + LOGGER.error(msg); throw new InternalErrorException(msg); } } @@ -218,4 +302,4 @@ public boolean configure(String name, Map params) throws Configu return true; } -} +} \ No newline at end of file diff --git a/core/src/main/java/com/cloud/storage/template/Processor.java b/core/src/main/java/com/cloud/storage/template/Processor.java index 4bb714a7ab9d..15a3dec0c6ee 100644 --- a/core/src/main/java/com/cloud/storage/template/Processor.java +++ b/core/src/main/java/com/cloud/storage/template/Processor.java @@ -23,10 +23,14 @@ import java.io.IOException; import java.util.List; -import com.cloud.agent.api.storage.OVFPropertyTO; +import com.cloud.agent.api.to.deployasis.OVFEulaSectionTO; +import com.cloud.agent.api.to.deployasis.OVFPropertyTO; +import com.cloud.agent.api.to.deployasis.OVFVirtualHardwareSectionTO; +import com.cloud.agent.api.to.DatadiskTO; import com.cloud.exception.InternalErrorException; import com.cloud.storage.Storage.ImageFormat; import com.cloud.utils.component.Adapter; +import com.cloud.agent.api.to.deployasis.OVFNetworkTO; /** * Generic interface to process different types of image formats @@ -48,13 +52,17 @@ public interface Processor extends Adapter { FormatInfo process(String templatePath, ImageFormat format, String templateName, long processTimeout) throws InternalErrorException; - public static class FormatInfo { + class FormatInfo { public ImageFormat format; public long size; public long virtualSize; public String filename; public boolean isCorrupted; public List ovfProperties; + public List networks; + public List disks; + public OVFVirtualHardwareSectionTO hardwareSection; + public List eulaSections; } long getVirtualSize(File file) throws IOException; diff --git a/core/src/main/java/org/apache/cloudstack/storage/command/CheckDataStoreStoragePolicyComplainceCommand.java b/core/src/main/java/org/apache/cloudstack/storage/command/CheckDataStoreStoragePolicyComplainceCommand.java new file mode 100644 index 000000000000..f9544b873ef2 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/storage/command/CheckDataStoreStoragePolicyComplainceCommand.java @@ -0,0 +1,61 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.storage.command; + +import com.cloud.agent.api.to.StorageFilerTO; + +public class CheckDataStoreStoragePolicyComplainceCommand extends StorageSubSystemCommand { + + String storagePolicyId; + private StorageFilerTO storagePool; + + public CheckDataStoreStoragePolicyComplainceCommand(String storagePolicyId, StorageFilerTO storagePool) { + super(); + + this.storagePolicyId = storagePolicyId; + this.storagePool = storagePool; + } + + @Override + public void setExecuteInSequence(boolean inSeq) { + } + + @Override + public boolean executeInSequence() { + return false; + } + + + public String getStoragePolicyId() { + return storagePolicyId; + } + + public void setStoragePolicyId(String storagePolicyId) { + this.storagePolicyId = storagePolicyId; + } + + public StorageFilerTO getStoragePool() { + return storagePool; + } + + public void setStoragePool(StorageFilerTO storagePool) { + this.storagePool = storagePool; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/storage/to/TemplateObjectTO.java b/core/src/main/java/org/apache/cloudstack/storage/to/TemplateObjectTO.java index cc2eaadea073..b184a74312b5 100644 --- a/core/src/main/java/org/apache/cloudstack/storage/to/TemplateObjectTO.java +++ b/core/src/main/java/org/apache/cloudstack/storage/to/TemplateObjectTO.java @@ -47,6 +47,8 @@ public class TemplateObjectTO implements DataTO { private boolean bootable; private String uniqueName; private boolean directDownload; + private boolean deployAsIs; + private String deployAsIsConfiguration; public TemplateObjectTO() { @@ -82,6 +84,8 @@ public TemplateObjectTO(TemplateInfo template) { this.imageDataStore = template.getDataStore().getTO(); } this.hypervisorType = template.getHypervisorType(); + this.deployAsIs = template.isDeployAsIs(); + this.deployAsIsConfiguration = template.getDeployAsIsConfiguration(); } @Override @@ -244,6 +248,18 @@ public void setDirectDownload(boolean directDownload) { this.directDownload = directDownload; } + public boolean isDeployAsIs() { + return deployAsIs; + } + + public String getDeployAsIsConfiguration() { + return deployAsIsConfiguration; + } + + public void setDeployAsIsConfiguration(String deployAsIsConfiguration) { + this.deployAsIsConfiguration = deployAsIsConfiguration; + } + @Override public String toString() { return new StringBuilder("TemplateTO[id=").append(id).append("|origUrl=").append(origUrl).append("|name").append(name).append("]").toString(); diff --git a/core/src/test/java/com/cloud/agent/api/storage/DownloadAnswerTest.java b/core/src/test/java/com/cloud/agent/api/storage/DownloadAnswerTest.java new file mode 100644 index 000000000000..c7dcc22572a4 --- /dev/null +++ b/core/src/test/java/com/cloud/agent/api/storage/DownloadAnswerTest.java @@ -0,0 +1,59 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.api.storage; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.to.deployasis.OVFPropertyTO; +import com.cloud.serializer.GsonHelper; +import com.cloud.storage.VMTemplateStorageResourceAssoc; +import com.google.gson.Gson; +import com.cloud.agent.api.to.deployasis.OVFNetworkTO; +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +public class DownloadAnswerTest { + Gson gson = GsonHelper.getGson(); + + VMTemplateStorageResourceAssoc.Status status = VMTemplateStorageResourceAssoc.Status.DOWNLOADED; + DownloadAnswer answer = new DownloadAnswer("nothin wrong", status); + + @Test + public void redeserialise () + { + String json = gson.toJson(answer); + DownloadAnswer received = gson.fromJson(json, DownloadAnswer.class); + Assert.assertEquals(received,answer); + } + @Test + public void properties () + { + List properties = new ArrayList<>(); + properties.add(new OVFPropertyTO()); + List networks = new ArrayList<>(); + networks.add(new OVFNetworkTO()); + + answer.setOvfProperties(properties); + answer.setNetworkRequirements(networks); + + String json = gson.toJson(answer); + Answer received = gson.fromJson(json, Answer.class); + Assert.assertEquals(received,answer); + } +} \ No newline at end of file diff --git a/core/src/test/java/org/apache/cloudstack/api/agent/test/BackupSnapshotCommandTest.java b/core/src/test/java/org/apache/cloudstack/api/agent/test/BackupSnapshotCommandTest.java index edc90aab770b..4bdef109af36 100644 --- a/core/src/test/java/org/apache/cloudstack/api/agent/test/BackupSnapshotCommandTest.java +++ b/core/src/test/java/org/apache/cloudstack/api/agent/test/BackupSnapshotCommandTest.java @@ -137,6 +137,11 @@ public int getPort() { @Override public boolean isManaged() { return false; } + @Override + public Long getParent() { + return 0L; + } + @Override public Long getPodId() { return 0L; diff --git a/core/src/test/java/org/apache/cloudstack/api/agent/test/CheckNetworkAnswerTest.java b/core/src/test/java/org/apache/cloudstack/api/agent/test/CheckNetworkAnswerTest.java index 4d49c99ee900..8b2550151f38 100644 --- a/core/src/test/java/org/apache/cloudstack/api/agent/test/CheckNetworkAnswerTest.java +++ b/core/src/test/java/org/apache/cloudstack/api/agent/test/CheckNetworkAnswerTest.java @@ -176,6 +176,11 @@ public int getPort() { @Override public boolean isManaged() { return false; } + @Override + public Long getParent() { + return 0L; + } + @Override public Long getPodId() { return 0L; diff --git a/core/src/test/java/org/apache/cloudstack/api/agent/test/SnapshotCommandTest.java b/core/src/test/java/org/apache/cloudstack/api/agent/test/SnapshotCommandTest.java index 576419ab652a..e134a94e1e5a 100644 --- a/core/src/test/java/org/apache/cloudstack/api/agent/test/SnapshotCommandTest.java +++ b/core/src/test/java/org/apache/cloudstack/api/agent/test/SnapshotCommandTest.java @@ -138,6 +138,11 @@ public int getPort() { @Override public boolean isManaged() { return false; } + @Override + public Long getParent() { + return 0L; + } + @Override public Long getPodId() { return 0L; diff --git a/deps/install-non-oss.sh b/deps/install-non-oss.sh index a387050d5390..01da6dab93f0 100755 --- a/deps/install-non-oss.sh +++ b/deps/install-non-oss.sh @@ -39,3 +39,9 @@ mvn install:install-file -Dfile=vim25_65.jar -DgroupId=com.cloud.com.vmwa # From https://my.vmware.com/group/vmware/details?downloadGroup=WEBCLIENTSDK67U2&productId=742 mvn install:install-file -Dfile=vim25_67.jar -DgroupId=com.cloud.com.vmware -DartifactId=vmware-vim25 -Dversion=6.7 -Dpackaging=jar + +# From https://my.vmware.com/group/vmware/get-download?downloadGroup=VS-MGMT-SDK65 +mvn install:install-file -Dfile=pbm_65.jar -DgroupId=com.cloud.com.vmware -DartifactId=vmware-pbm -Dversion=6.5 -Dpackaging=jar + +# From https://my.vmware.com/group/vmware/details?downloadGroup=WEBCLIENTSDK67U2&productId=742 +mvn install:install-file -Dfile=pbm_67.jar -DgroupId=com.cloud.com.vmware -DartifactId=vmware-pbm -Dversion=6.7 -Dpackaging=jar diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/StorageOrchestrationService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/StorageOrchestrationService.java new file mode 100644 index 000000000000..7bf845d3ec56 --- /dev/null +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/StorageOrchestrationService.java @@ -0,0 +1,27 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.engine.orchestration.service; + +import java.util.List; + +import org.apache.cloudstack.api.response.MigrationResponse; +import org.apache.cloudstack.storage.ImageStoreService.MigrationPolicy; + +public interface StorageOrchestrationService { + MigrationResponse migrateData(Long srcDataStoreId, List destDatastores, MigrationPolicy migrationPolicy); +} diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java index 9458de763538..9baea60f29fa 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java @@ -18,6 +18,7 @@ */ package org.apache.cloudstack.engine.orchestration.service; +import java.util.List; import java.util.Map; import java.util.Set; @@ -84,6 +85,8 @@ VolumeInfo moveVolume(VolumeInfo volume, long destPoolDcId, Long destPoolPodId, String getVmNameOnVolume(Volume volume); + StoragePool findChildDataStoreInDataStoreCluster(DataCenter dc, Pod pod, Long clusterId, Long hostId, VirtualMachine vm, Long datastoreClusterId); + VolumeInfo createVolumeFromSnapshot(Volume volume, Snapshot snapshot, UserVm vm) throws StorageUnavailableException; Volume migrateVolume(Volume volume, StoragePool destPool) throws StorageUnavailableException; @@ -117,8 +120,11 @@ DiskProfile allocateRawVolume(Type type, String name, DiskOffering offering, Lon boolean canVmRestartOnAnotherServer(long vmId); - DiskProfile allocateTemplatedVolume(Type type, String name, DiskOffering offering, Long rootDisksize, Long minIops, Long maxIops, VirtualMachineTemplate template, VirtualMachine vm, - Account owner); + /** + * Allocate a volume or multiple volumes in case of template is registered with the 'deploy-as-is' option, allowing multiple disks + */ + List allocateTemplatedVolumes(Type type, String name, DiskOffering offering, Long rootDisksize, Long minIops, Long maxIops, VirtualMachineTemplate template, VirtualMachine vm, + Account owner); String getVmNameFromVolumeId(long volumeId); diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/service/api/OrchestrationService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/service/api/OrchestrationService.java index 5a18b3cab9ee..82c3dd14cf3c 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/service/api/OrchestrationService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/service/api/OrchestrationService.java @@ -55,7 +55,7 @@ public interface OrchestrationService { * @param memory memory to allocate in bytes * @param computeTags tags for the compute * @param rootDiskTags tags for the root disk - * @param networks networks that this VM should join + * @param networkNicMap map networks to nic profiles that this VM should join * @param rootDiskSize size the root disk in case of templates. * @return VirtualMachineEntity */ @@ -65,7 +65,7 @@ VirtualMachineEntity createVirtualMachine(@QueryParam("id") String id, @QueryPar @QueryParam("host-name") String hostName, @QueryParam("display-name") String displayName, @QueryParam("hypervisor") String hypervisor, @QueryParam("cpu") int cpu, @QueryParam("speed") int speed, @QueryParam("ram") long memory, @QueryParam("disk-size") Long diskSize, @QueryParam("compute-tags") List computeTags, @QueryParam("root-disk-tags") List rootDiskTags, - @QueryParam("network-nic-map") Map networkNicMap, @QueryParam("deploymentplan") DeploymentPlan plan, + @QueryParam("network-nic-map") Map> networkNicMap, @QueryParam("deploymentplan") DeploymentPlan plan, @QueryParam("root-disk-size") Long rootDiskSize, @QueryParam("extra-dhcp-option-map") Map> extraDhcpOptionMap, @QueryParam("datadisktemplate-diskoffering-map") Map datadiskTemplateToDiskOfferingMap) throws InsufficientCapacityException; @@ -74,7 +74,7 @@ VirtualMachineEntity createVirtualMachineFromScratch(@QueryParam("id") String id @QueryParam("host-name") String hostName, @QueryParam("display-name") String displayName, @QueryParam("hypervisor") String hypervisor, @QueryParam("os") String os, @QueryParam("cpu") int cpu, @QueryParam("speed") int speed, @QueryParam("ram") long memory, @QueryParam("disk-size") Long diskSize, @QueryParam("compute-tags") List computeTags, @QueryParam("root-disk-tags") List rootDiskTags, - @QueryParam("network-nic-map") Map networkNicMap, @QueryParam("deploymentplan") DeploymentPlan plan, @QueryParam("extra-dhcp-option-map") Map> extraDhcpOptionMap) throws InsufficientCapacityException; + @QueryParam("network-nic-map") Map> networkNicMap, @QueryParam("deploymentplan") DeploymentPlan plan, @QueryParam("extra-dhcp-option-map") Map> extraDhcpOptionMap) throws InsufficientCapacityException; @POST NetworkEntity createNetwork(String id, String name, String domainName, String cidr, String gateway); diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreManager.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreManager.java index ad5b1622cd22..80e3ce11c759 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreManager.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreManager.java @@ -33,10 +33,16 @@ public interface DataStoreManager { List getImageStoresByScope(ZoneScope scope); + List getImageStoresByScopeExcludingReadOnly(ZoneScope scope); + DataStore getRandomImageStore(long zoneId); + DataStore getRandomUsableImageStore(long zoneId); + DataStore getImageStoreWithFreeCapacity(long zoneId); + DataStore getImageStoreWithFreeCapacity(List imageStores); + List listImageStoresWithFreeCapacity(long zoneId); List getImageCacheStores(Scope scope); diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/EndPointSelector.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/EndPointSelector.java index 0613a11572f7..ec2725019983 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/EndPointSelector.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/EndPointSelector.java @@ -33,6 +33,8 @@ public interface EndPointSelector { List selectAll(DataStore store); + List findAllEndpointsForScope(DataStore store); + EndPoint select(Scope scope, Long storeId); EndPoint select(DataStore store, String downloadUrl); diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/ObjectInDataStoreStateMachine.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/ObjectInDataStoreStateMachine.java index 204cab0bd74c..611d1247c49d 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/ObjectInDataStoreStateMachine.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/ObjectInDataStoreStateMachine.java @@ -29,6 +29,7 @@ enum State { Ready("Template downloading is accomplished"), Copying("The object is being coping"), Migrating("The object is being migrated"), + Migrated("The object has been migrated"), Destroying("Template is destroying"), Destroyed("Template is destroyed"), Failed("Failed to download template"); @@ -49,12 +50,16 @@ enum Event { DestroyRequested, OperationSuccessed, OperationFailed, + CopyRequested, CopyingRequested, MigrationRequested, + MigrationSucceeded, + MigrationFailed, MigrationCopyRequested, MigrationCopySucceeded, MigrationCopyFailed, ResizeRequested, - ExpungeRequested + ExpungeRequested, + MigrateDataRequested } } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStore.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStore.java index a399758217be..5546571ba6b4 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStore.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStore.java @@ -23,7 +23,9 @@ import org.apache.cloudstack.engine.subsystem.api.storage.disktype.DiskFormat; public interface PrimaryDataStore extends DataStore, PrimaryDataStoreInfo { - DataObject create(DataObject dataObject, boolean createEntryInTempSpoolRef); + DataObject create(DataObject dataObject, String configuration); + + DataObject create(DataObject dataObject, boolean createEntryInTempSpoolRef, String configuration); VolumeInfo getVolume(long id); @@ -31,7 +33,7 @@ public interface PrimaryDataStore extends DataStore, PrimaryDataStoreInfo { boolean exists(DataObject data); - TemplateInfo getTemplate(long templateId); + TemplateInfo getTemplate(long templateId, String configuration); SnapshotInfo getSnapshot(long snapshotId); diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreInfo.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreInfo.java index 7f2f4dc6b856..3e072e8f1bb4 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreInfo.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreInfo.java @@ -58,4 +58,6 @@ public interface PrimaryDataStoreInfo extends StoragePool { Map getDetails(); PrimaryDataStoreLifeCycle getLifeCycle(); + + Long getParent(); } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SecondaryStorageService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SecondaryStorageService.java new file mode 100644 index 000000000000..07828fda5ce7 --- /dev/null +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SecondaryStorageService.java @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.engine.subsystem.api.storage; + +import java.util.List; +import java.util.Map; + +import org.apache.cloudstack.framework.async.AsyncCallFuture; +import org.apache.cloudstack.storage.command.CommandResult; + +import com.cloud.utils.Pair; + +public interface SecondaryStorageService { + class DataObjectResult extends CommandResult { + private final DataObject data; + + public DataObjectResult(DataObject data) { + super(); + this.data = data; + } + + public DataObject getData() { + return this.data; + } + + } + AsyncCallFuture migrateData(DataObject srcDataObject, DataStore srcDatastore, DataStore destDatastore, Map, Long>> snapshotChain); +} diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotInfo.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotInfo.java index ef72afc5b777..58a82ac2c740 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotInfo.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotInfo.java @@ -16,6 +16,8 @@ // under the License. package org.apache.cloudstack.engine.subsystem.api.storage; +import java.util.List; + import com.cloud.storage.Snapshot; import com.cloud.utils.exception.CloudRuntimeException; @@ -26,6 +28,8 @@ public interface SnapshotInfo extends DataObject, Snapshot { SnapshotInfo getChild(); + List getChildren(); + VolumeInfo getBaseVolume(); void addPayload(Object data); diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/StoragePoolAllocator.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/StoragePoolAllocator.java index dfdbd8ab92c4..46f8f5e04292 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/StoragePoolAllocator.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/StoragePoolAllocator.java @@ -51,4 +51,6 @@ public interface StoragePoolAllocator extends Adapter { List allocateToPool(DiskProfile dskCh, VirtualMachineProfile vmProfile, DeploymentPlan plan, ExcludeList avoid, int returnUpTo); static int RETURN_UPTO_ALL = -1; + + List reorderPools(List pools, VirtualMachineProfile vmProfile, DeploymentPlan plan); } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateDataFactory.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateDataFactory.java index b213625efadd..4d258f3b6d0b 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateDataFactory.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateDataFactory.java @@ -27,7 +27,7 @@ public interface TemplateDataFactory { TemplateInfo getReadyTemplateOnImageStore(long templateId, Long zoneId); - TemplateInfo getTemplate(DataObject obj, DataStore store); + TemplateInfo getTemplate(DataObject obj, DataStore store, String configuration); TemplateInfo getTemplate(long templateId, DataStoreRole storeRole); @@ -40,4 +40,6 @@ public interface TemplateDataFactory { TemplateInfo getReadyBypassedTemplateOnPrimaryStore(long templateId, Long poolId, Long hostId); boolean isTemplateMarkedForDirectDownload(long templateId); + + TemplateInfo getTemplateOnPrimaryStorage(long templateId, DataStore store, String configuration); } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateInfo.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateInfo.java index 0f7cc6f9de56..1e4a1b7373a6 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateInfo.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateInfo.java @@ -27,4 +27,8 @@ public interface TemplateInfo extends DataObject, VirtualMachineTemplate { String getInstallPath(); boolean isDirectDownload(); + + boolean isDeployAsIs(); + + String getDeployAsIsConfiguration(); } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java index f70a7813ae09..df13f951a448 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java @@ -18,6 +18,7 @@ */ package org.apache.cloudstack.engine.subsystem.api.storage; +import com.cloud.agent.api.to.DatadiskTO; import org.apache.cloudstack.framework.async.AsyncCallFuture; import org.apache.cloudstack.framework.async.AsyncCompletionCallback; import org.apache.cloudstack.storage.command.CommandResult; @@ -25,6 +26,8 @@ import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.storage.StoragePool; +import java.util.List; + public interface TemplateService { class TemplateApiResult extends CommandResult { @@ -47,7 +50,7 @@ public TemplateInfo getTemplate() { AsyncCallFuture createTemplateFromVolumeAsync(VolumeInfo volume, TemplateInfo template, DataStore store); - boolean createOvaDataDiskTemplates(TemplateInfo parentTemplate); + boolean createOvaDataDiskTemplates(TemplateInfo parentTemplate, boolean deployAsIs); AsyncCallFuture deleteTemplateAsync(TemplateInfo template); @@ -72,4 +75,6 @@ public TemplateInfo getTemplate() { void associateCrosszoneTemplatesToZone(long dcId); AsyncCallFuture createDatadiskTemplateAsync(TemplateInfo parentTemplate, TemplateInfo dataDiskTemplate, String path, String diskId, long fileSize, boolean bootable); + + List getTemplateDatadisksOnImageStore(TemplateInfo templateInfo, String configurationId); } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeInfo.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeInfo.java index f4a738109013..06bda51a092d 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeInfo.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeInfo.java @@ -83,4 +83,8 @@ public interface VolumeInfo extends DataObject, Volume { boolean isDirectDownload(); void setDirectDownload(boolean directDownload); + + boolean isDeployAsIs(); + + String getDeployAsIsConfiguration(); } diff --git a/engine/api/src/main/java/org/apache/cloudstack/storage/image/datastore/ImageStoreEntity.java b/engine/api/src/main/java/org/apache/cloudstack/storage/image/datastore/ImageStoreEntity.java index 5a0be952b391..88d908d78741 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/storage/image/datastore/ImageStoreEntity.java +++ b/engine/api/src/main/java/org/apache/cloudstack/storage/image/datastore/ImageStoreEntity.java @@ -51,7 +51,7 @@ public interface ImageStoreEntity extends DataStore, ImageStore { void deleteExtractUrl(String installPath, String url, Upload.Type volume); - List getDataDiskTemplates(DataObject obj); + List getDataDiskTemplates(DataObject obj, String configurationId); Void createDataDiskTemplateAsync(TemplateInfo dataDiskTemplate, String path, String diskId, long fileSize, boolean bootable, AsyncCompletionCallback callback); } diff --git a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java index 62a241be4345..0e245920af34 100644 --- a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java +++ b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java @@ -112,6 +112,12 @@ public interface StorageManager extends StorageService { ConfigKey PRIMARY_STORAGE_DOWNLOAD_WAIT = new ConfigKey("Storage", Integer.class, "primary.storage.download.wait", "10800", "In second, timeout for download template to primary storage", false); + ConfigKey SecStorageMaxMigrateSessions = new ConfigKey("Advanced", Integer.class, "secstorage.max.migrate.sessions", "2", + "The max number of concurrent copy command execution sessions that an SSVM can handle", true, ConfigKey.Scope.Global); + + ConfigKey MaxDataMigrationWaitTime = new ConfigKey("Advanced", Integer.class, "max.data.migration.wait.time", "15", + "Maximum wait time for a data migration task before spawning a new SSVM", false, ConfigKey.Scope.Global); + /** * Returns a comma separated list of tags for the specified storage pool * @param poolId @@ -206,6 +212,8 @@ public interface StorageManager extends StorageService { boolean storagePoolHasEnoughSpaceForResize(StoragePool pool, long currentSize, long newSiz); + boolean isStoragePoolComplaintWithStoragePolicy(List volumes, StoragePool pool) throws StorageUnavailableException; + boolean registerHostListener(String providerUuid, HypervisorHostListener listener); void connectHostToSharedPool(long hostId, long poolId) throws StorageUnavailableException, StorageConflictException; @@ -230,4 +238,6 @@ public interface StorageManager extends StorageService { DiskTO getDiskWithThrottling(DataTO volTO, Volume.Type volumeType, long deviceId, String path, long offeringId, long diskOfferingId); + boolean isStoragePoolDatastoreClusterParent(StoragePool pool); + } diff --git a/engine/components-api/src/main/java/com/cloud/template/TemplateManager.java b/engine/components-api/src/main/java/com/cloud/template/TemplateManager.java index 2dc6296fc51f..35fed56ac8e5 100644 --- a/engine/components-api/src/main/java/com/cloud/template/TemplateManager.java +++ b/engine/components-api/src/main/java/com/cloud/template/TemplateManager.java @@ -18,7 +18,9 @@ import java.util.List; +import com.cloud.agent.api.to.DatadiskTO; import com.cloud.deploy.DeployDestination; +import com.cloud.storage.DataStoreRole; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; import org.apache.cloudstack.framework.config.ConfigKey; @@ -133,4 +135,5 @@ public interface TemplateManager { public static final String MESSAGE_REGISTER_PUBLIC_TEMPLATE_EVENT = "Message.RegisterPublicTemplate.Event"; public static final String MESSAGE_RESET_TEMPLATE_PERMISSION_EVENT = "Message.ResetTemplatePermission.Event"; + List getTemplateDisksOnImageStore(Long templateId, DataStoreRole role, String configurationId); } diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index b8342b4a60e3..225996865f7b 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -40,6 +40,7 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.deployasis.dao.UserVmDeployAsIsDetailsDao; import org.apache.cloudstack.affinity.dao.AffinityGroupVMMapDao; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.command.admin.vm.MigrateVMCmd; @@ -344,6 +345,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac private NetworkDetailsDao networkDetailsDao; @Inject private SecurityGroupManager _securityGroupManager; + @Inject + private UserVmDeployAsIsDetailsDao userVmDeployAsIsDetailsDao; VmWorkJobHandlerProxy _jobHandlerProxy = new VmWorkJobHandlerProxy(this); @@ -414,6 +417,8 @@ public void allocate(final String vmInstanceName, final VirtualMachineTemplate t final LinkedHashMap> auxiliaryNetworks, final DeploymentPlan plan, final HypervisorType hyperType, final Map> extraDhcpOptions, final Map datadiskTemplateToDiskOfferingMap) throws InsufficientCapacityException { + s_logger.info(String.format("allocating virtual machine from template:%s with hostname:%s and %d networks", template.getUuid(), vmInstanceName, auxiliaryNetworks.size())); + final VMInstanceVO vm = _vmDao.findVMByInstanceName(vmInstanceName); final Account owner = _entityMgr.findById(Account.class, vm.getAccountId()); @@ -455,7 +460,7 @@ public void doInTransactionWithoutResult(final TransactionStatus status) throws } else if (template.getFormat() == ImageFormat.BAREMETAL) { // Do nothing } else { - volumeMgr.allocateTemplatedVolume(Type.ROOT, "ROOT-" + vmFinal.getId(), rootDiskOfferingInfo.getDiskOffering(), rootDiskOfferingInfo.getSize(), + volumeMgr.allocateTemplatedVolumes(Type.ROOT, "ROOT-" + vmFinal.getId(), rootDiskOfferingInfo.getDiskOffering(), rootDiskOfferingInfo.getSize(), rootDiskOfferingInfo.getMinIops(), rootDiskOfferingInfo.getMaxIops(), template, vmFinal, owner); } @@ -592,6 +597,13 @@ protected void advanceExpunge(VMInstanceVO vm) throws ResourceUnavailableExcepti //remove the overcommit details from the uservm details userVmDetailsDao.removeDetails(vm.getId()); + // Remove details if VM deploy as-is + long templateId = vm.getTemplateId(); + VMTemplateVO template = _templateDao.findById(templateId); + if (template != null && template.isDeployAsIs()) { + userVmDeployAsIsDetailsDao.removeDetails(vm.getId()); + } + // send hypervisor-dependent commands before removing final List finalizeExpungeCommands = hvGuru.finalizeExpunge(vm); if (finalizeExpungeCommands != null && finalizeExpungeCommands.size() > 0) { @@ -1104,7 +1116,9 @@ public void orchestrateStart(final String vmUuid, final Map volumes = _volsDao.findCreatedByInstance(vm.getId()); for (final VolumeVO volume : volumes) { if (!_storagePoolDao.findById(volume.getPoolId()).getScope().equals(ScopeType.ZONE)) { @@ -2955,7 +2975,8 @@ private void orchestrateMigrateWithStorage(final String vmUuid, final long srcHo final String serviceOffering = _serviceOfferingDao.findByIdIncludingRemoved(vm.getId(), vm.getServiceOfferingId()).getDisplayText(); boolean isWindows = _guestOSCategoryDao.findById(_guestOSDao.findById(vm.getGuestOSId()).getCategoryId()).getName().equalsIgnoreCase("Windows"); vmData = _networkModel.generateVmData(userVm.getUserData(), serviceOffering, vm.getDataCenterId(), vm.getInstanceName(), vm.getHostName(), vm.getId(), - vm.getUuid(), defaultNic.getMacAddress(), userVm.getDetail("SSH.PublicKey"), (String) profile.getParameter(VirtualMachineProfile.Param.VmPassword), isWindows, VirtualMachineManager.getHypervisorHostname(destination.getHost().getName())); + vm.getUuid(), defaultNic.getMacAddress(), userVm.getDetail("SSH.PublicKey"), (String) profile.getParameter(VirtualMachineProfile.Param.VmPassword), isWindows, + VirtualMachineManager.getHypervisorHostname(destination.getHost() != null ? destination.getHost().getName() : "")); String vmName = vm.getInstanceName(); String configDriveIsoRootFolder = "/tmp"; String isoFile = configDriveIsoRootFolder + "/" + vmName + "/configDrive/" + vmName + ".iso"; diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/CloudOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/CloudOrchestrator.java index 91e9b6f57bdb..c6f8e418642c 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/CloudOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/CloudOrchestrator.java @@ -20,7 +20,6 @@ import java.net.URL; import java.util.ArrayList; -import java.util.Arrays; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -156,7 +155,7 @@ public void destroyVolume(String volumeEntity) { @Override public VirtualMachineEntity createVirtualMachine(String id, String owner, String templateId, String hostName, String displayName, String hypervisor, int cpu, - int speed, long memory, Long diskSize, List computeTags, List rootDiskTags, Map networkNicMap, DeploymentPlan plan, + int speed, long memory, Long diskSize, List computeTags, List rootDiskTags, Map> networkNicMap, DeploymentPlan plan, Long rootDiskSize, Map> extraDhcpOptionMap, Map dataDiskTemplateToDiskOfferingMap) throws InsufficientCapacityException { // VirtualMachineEntityImpl vmEntity = new VirtualMachineEntityImpl(id, owner, hostName, displayName, cpu, speed, memory, computeTags, rootDiskTags, networks, @@ -166,7 +165,7 @@ public VirtualMachineEntity createVirtualMachine(String id, String owner, String for (String uuid : networkNicMap.keySet()) { NetworkVO network = _networkDao.findByUuid(uuid); if(network != null){ - networkIpMap.put(network, new ArrayList(Arrays.asList(networkNicMap.get(uuid)))); + networkIpMap.put(network, networkNicMap.get(uuid)); } } @@ -255,7 +254,7 @@ public VirtualMachineEntity createVirtualMachine(String id, String owner, String @Override public VirtualMachineEntity createVirtualMachineFromScratch(String id, String owner, String isoId, String hostName, String displayName, String hypervisor, String os, - int cpu, int speed, long memory, Long diskSize, List computeTags, List rootDiskTags, Map networkNicMap, DeploymentPlan plan, Map> extraDhcpOptionMap) + int cpu, int speed, long memory, Long diskSize, List computeTags, List rootDiskTags, Map> networkNicMap, DeploymentPlan plan, Map> extraDhcpOptionMap) throws InsufficientCapacityException { // VirtualMachineEntityImpl vmEntity = new VirtualMachineEntityImpl(id, owner, hostName, displayName, cpu, speed, memory, computeTags, rootDiskTags, networks, vmEntityManager); @@ -307,7 +306,7 @@ public VirtualMachineEntity createVirtualMachineFromScratch(String id, String ow for (String uuid : networkNicMap.keySet()) { NetworkVO network = _networkDao.findByUuid(uuid); if(network != null){ - networkIpMap.put(network, new ArrayList(Arrays.asList(networkNicMap.get(uuid)))); + networkIpMap.put(network, networkNicMap.get(uuid)); } } diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/DataMigrationUtility.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/DataMigrationUtility.java new file mode 100644 index 000000000000..86b7350d138b --- /dev/null +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/DataMigrationUtility.java @@ -0,0 +1,258 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.engine.orchestration; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import javax.inject.Inject; + +import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; +import org.apache.cloudstack.storage.ImageStoreService; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; +import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO; + +import com.cloud.host.HostVO; +import com.cloud.host.Status; +import com.cloud.host.dao.HostDao; +import com.cloud.storage.DataStoreRole; +import com.cloud.storage.SnapshotVO; +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.dao.SnapshotDao; +import com.cloud.storage.dao.VMTemplateDao; +import com.cloud.utils.Pair; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.vm.SecondaryStorageVmVO; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.dao.SecondaryStorageVmDao; + +public class DataMigrationUtility { + @Inject + SecondaryStorageVmDao secStorageVmDao; + @Inject + TemplateDataStoreDao templateDataStoreDao; + @Inject + SnapshotDataStoreDao snapshotDataStoreDao; + @Inject + VolumeDataStoreDao volumeDataStoreDao; + @Inject + VMTemplateDao templateDao; + @Inject + VolumeDataFactory volumeFactory; + @Inject + TemplateDataFactory templateFactory; + @Inject + SnapshotDataFactory snapshotFactory; + @Inject + HostDao hostDao; + @Inject + SnapshotDao snapshotDao; + + /** + * This function verifies if the given image store contains data objects that are not in any of the following states: + * "Ready" "Allocated", "Destroying", "Destroyed", "Failed". If this is the case, and if the migration policy is complete, + * the migration is terminated. + */ + private boolean filesReadyToMigrate(Long srcDataStoreId) { + String[] validStates = new String[]{"Ready", "Allocated", "Destroying", "Destroyed", "Failed"}; + boolean isReady = true; + List templates = templateDataStoreDao.listByStoreId(srcDataStoreId); + for (TemplateDataStoreVO template : templates) { + isReady &= (Arrays.asList(validStates).contains(template.getState().toString())); + } + List snapshots = snapshotDataStoreDao.listByStoreId(srcDataStoreId, DataStoreRole.Image); + for (SnapshotDataStoreVO snapshot : snapshots) { + isReady &= (Arrays.asList(validStates).contains(snapshot.getState().toString())); + } + List volumes = volumeDataStoreDao.listByStoreId(srcDataStoreId); + for (VolumeDataStoreVO volume : volumes) { + isReady &= (Arrays.asList(validStates).contains(volume.getState().toString())); + } + return isReady; + } + + protected void checkIfCompleteMigrationPossible(ImageStoreService.MigrationPolicy policy, Long srcDataStoreId) { + if (policy == ImageStoreService.MigrationPolicy.COMPLETE) { + if (!filesReadyToMigrate(srcDataStoreId)) { + throw new CloudRuntimeException("Complete migration failed as there are data objects which are not Ready - i.e, they may be in Migrating, creating, copying, etc. states"); + } + } + return; + } + + protected Long getFileSize(DataObject file, Map, Long>> snapshotChain) { + Long size = file.getSize(); + Pair, Long> chain = snapshotChain.get(file); + if (file instanceof SnapshotInfo && chain.first() != null) { + size = chain.second(); + } + return size; + } + + /** + * Sorts the datastores in decreasing order of their free capacities, so as to make + * an informed decision of picking the datastore with maximum free capactiy for migration + */ + protected List sortDataStores(Map> storageCapacities) { + List>> list = + new LinkedList>>((storageCapacities.entrySet())); + + Collections.sort(list, new Comparator>>() { + @Override + public int compare(Map.Entry> e1, Map.Entry> e2) { + return e2.getValue().first() > e1.getValue().first() ? 1 : -1; + } + }); + HashMap> temp = new LinkedHashMap<>(); + for (Map.Entry> value : list) { + temp.put(value.getKey(), value.getValue()); + } + + return new ArrayList<>(temp.keySet()); + } + + protected List getSortedValidSourcesList(DataStore srcDataStore, Map, Long>> snapshotChains) { + List files = new ArrayList<>(); + files.addAll(getAllReadyTemplates(srcDataStore)); + files.addAll(getAllReadySnapshotsAndChains(srcDataStore, snapshotChains)); + files.addAll(getAllReadyVolumes(srcDataStore)); + + files = sortFilesOnSize(files, snapshotChains); + + return files; + } + + protected List sortFilesOnSize(List files, Map, Long>> snapshotChains) { + Collections.sort(files, new Comparator() { + @Override + public int compare(DataObject o1, DataObject o2) { + Long size1 = o1.getSize(); + Long size2 = o2.getSize(); + if (o1 instanceof SnapshotInfo) { + size1 = snapshotChains.get(o1).second(); + } + if (o2 instanceof SnapshotInfo) { + size2 = snapshotChains.get(o2).second(); + } + return size2 > size1 ? 1 : -1; + } + }); + return files; + } + + protected List getAllReadyTemplates(DataStore srcDataStore) { + + List files = new LinkedList<>(); + List templates = templateDataStoreDao.listByStoreId(srcDataStore.getId()); + for (TemplateDataStoreVO template : templates) { + VMTemplateVO templateVO = templateDao.findById(template.getTemplateId()); + if (template.getState() == ObjectInDataStoreStateMachine.State.Ready && !templateVO.isPublicTemplate()) { + files.add(templateFactory.getTemplate(template.getTemplateId(), srcDataStore)); + } + } + return files; + } + + /** Returns parent snapshots and snapshots that do not have any children; snapshotChains comprises of the snapshot chain info + * for each parent snapshot and the cumulative size of the chain - this is done to ensure that all the snapshots in a chain + * are migrated to the same datastore + */ + protected List getAllReadySnapshotsAndChains(DataStore srcDataStore, Map, Long>> snapshotChains) { + List files = new LinkedList<>(); + List snapshots = snapshotDataStoreDao.listByStoreId(srcDataStore.getId(), DataStoreRole.Image); + for (SnapshotDataStoreVO snapshot : snapshots) { + SnapshotVO snapshotVO = snapshotDao.findById(snapshot.getSnapshotId()); + if (snapshot.getState() == ObjectInDataStoreStateMachine.State.Ready && snapshot.getParentSnapshotId() == 0 ) { + SnapshotInfo snap = snapshotFactory.getSnapshot(snapshotVO.getSnapshotId(), DataStoreRole.Image); + files.add(snap); + } + } + + for (SnapshotInfo parent : files) { + List chain = new ArrayList<>(); + chain.add(parent); + for (int i =0; i< chain.size(); i++) { + SnapshotInfo child = chain.get(i); + List children = child.getChildren(); + if (children != null) { + chain.addAll(children); + } + } + snapshotChains.put(parent, new Pair, Long>(chain, getSizeForChain(chain))); + } + + return (List) (List) files; + } + + protected Long getSizeForChain(List chain) { + Long size = 0L; + for (SnapshotInfo snapshot : chain) { + size += snapshot.getSize(); + } + return size; + } + + + protected List getAllReadyVolumes(DataStore srcDataStore) { + List files = new LinkedList<>(); + List volumes = volumeDataStoreDao.listByStoreId(srcDataStore.getId()); + for (VolumeDataStoreVO volume : volumes) { + if (volume.getState() == ObjectInDataStoreStateMachine.State.Ready) { + files.add(volumeFactory.getVolume(volume.getVolumeId(), srcDataStore)); + } + } + return files; + } + + /** Returns the count of active SSVMs - SSVM with agents in connected state, so as to dynamically increase the thread pool + * size when SSVMs scale + */ + protected int activeSSVMCount(DataStore dataStore) { + long datacenterId = dataStore.getScope().getScopeId(); + List ssvms = + secStorageVmDao.getSecStorageVmListInStates(null, datacenterId, VirtualMachine.State.Running, VirtualMachine.State.Migrating); + int activeSSVMs = 0; + for (SecondaryStorageVmVO vm : ssvms) { + String name = "s-"+vm.getId()+"-VM"; + HostVO ssHost = hostDao.findByName(name); + if (ssHost != null) { + if (ssHost.getState() == Status.Up) { + activeSSVMs++; + } + } + } + return activeSSVMs; + } +} diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java index 892f77119082..c536074093ad 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java @@ -16,6 +16,8 @@ // under the License. package org.apache.cloudstack.engine.orchestration; +import static org.apache.commons.lang.StringUtils.isNotBlank; + import java.net.URI; import java.util.ArrayList; import java.util.Arrays; @@ -38,12 +40,10 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; -import com.cloud.event.EventTypes; -import com.cloud.event.UsageEventUtils; -import com.cloud.network.dao.NetworkDetailVO; -import com.cloud.network.dao.NetworkDetailsDao; +import com.cloud.deployasis.dao.TemplateDeployAsIsDetailsDao; import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.api.ApiConstants; +import com.cloud.agent.api.to.deployasis.OVFNetworkTO; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.cloud.entity.api.db.VMNetworkMapVO; import org.apache.cloudstack.engine.cloud.entity.api.db.dao.VMNetworkMapDao; @@ -88,6 +88,8 @@ import com.cloud.deploy.DeployDestination; import com.cloud.deploy.DeploymentPlan; import com.cloud.domain.Domain; +import com.cloud.event.EventTypes; +import com.cloud.event.UsageEventUtils; import com.cloud.exception.ConcurrentOperationException; import com.cloud.exception.ConnectionException; import com.cloud.exception.InsufficientAddressCapacityException; @@ -129,6 +131,8 @@ import com.cloud.network.dao.NetworkAccountDao; import com.cloud.network.dao.NetworkAccountVO; import com.cloud.network.dao.NetworkDao; +import com.cloud.network.dao.NetworkDetailVO; +import com.cloud.network.dao.NetworkDetailsDao; import com.cloud.network.dao.NetworkDomainDao; import com.cloud.network.dao.NetworkDomainVO; import com.cloud.network.dao.NetworkServiceMapDao; @@ -231,8 +235,6 @@ import com.cloud.vm.dao.VMInstanceDao; import com.google.common.base.Strings; -import static org.apache.commons.lang.StringUtils.isNotBlank; - /** * NetworkManagerImpl implements NetworkManager. */ @@ -301,6 +303,8 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra VpcVirtualNetworkApplianceService _routerService; @Inject UserVmManager _userVmMgr; + @Inject + TemplateDeployAsIsDetailsDao templateDeployAsIsDetailsDao; List networkGurus; @@ -645,8 +649,8 @@ public List setupNetwork(final Account owner, final NetworkOf @Override @DB public List setupNetwork(final Account owner, final NetworkOffering offering, final Network predefined, final DeploymentPlan plan, final String name, - final String displayText, final boolean errorIfAlreadySetup, final Long domainId, final ACLType aclType, final Boolean subdomainAccess, final Long vpcId, - final Boolean isDisplayNetworkEnabled) throws ConcurrentOperationException { + final String displayText, final boolean errorIfAlreadySetup, final Long domainId, final ACLType aclType, final Boolean subdomainAccess, final Long vpcId, + final Boolean isDisplayNetworkEnabled) throws ConcurrentOperationException { final Account locked = _accountDao.acquireInLockTable(owner.getId()); if (locked == null) { @@ -656,8 +660,8 @@ public List setupNetwork(final Account owner, final NetworkOf try { if (predefined == null || offering.getTrafficType() != TrafficType.Guest && predefined.getCidr() == null && predefined.getBroadcastUri() == null && !(predefined - .getBroadcastDomainType() == BroadcastDomainType.Vlan || predefined.getBroadcastDomainType() == BroadcastDomainType.Lswitch || predefined - .getBroadcastDomainType() == BroadcastDomainType.Vxlan)) { + .getBroadcastDomainType() == BroadcastDomainType.Vlan || predefined.getBroadcastDomainType() == BroadcastDomainType.Lswitch || predefined + .getBroadcastDomainType() == BroadcastDomainType.Vxlan)) { final List configs = _networksDao.listBy(owner.getId(), offering.getId(), plan.getDataCenterId()); if (configs.size() > 0) { if (s_logger.isDebugEnabled()) { @@ -747,30 +751,96 @@ public void doInTransactionWithoutResult(final TransactionStatus status) { @Override @DB public void allocate(final VirtualMachineProfile vm, final LinkedHashMap> networks, final Map> extraDhcpOptions) throws InsufficientCapacityException, - ConcurrentOperationException { + ConcurrentOperationException { Transaction.execute(new TransactionCallbackWithExceptionNoReturn() { @Override public void doInTransactionWithoutResult(final TransactionStatus status) throws InsufficientCapacityException { - int deviceId = 0; - int size = 0; - for (final Network ntwk : networks.keySet()) { - final List profiles = networks.get(ntwk); - if (profiles != null && !profiles.isEmpty()) { - size = size + profiles.size(); - } else { - size = size + 1; - } + if (s_logger.isTraceEnabled()) { + s_logger.trace(String.format("allocating networks for %s(template %s); %d networks",vm.getInstanceName(), vm.getTemplate().getUuid(), networks.size())); } + int deviceId = 0; + int size; + size = determineNumberOfNicsRequired(); final boolean[] deviceIds = new boolean[size]; Arrays.fill(deviceIds, false); + List> profilesList = getOrderedNetworkNicProfileMapping(networks); final List nics = new ArrayList(size); NicProfile defaultNic = null; + Network nextNetwork = null; + for (Pair networkNicPair : profilesList) { + nextNetwork = networkNicPair.first(); + Pair newDeviceInfo = addRequestedNicToNicListWithDeviceNumberAndRetrieveDefaultDevice(networkNicPair.second(), deviceIds, deviceId, nextNetwork, nics, defaultNic); + defaultNic = newDeviceInfo.first(); + deviceId = newDeviceInfo.second(); + } + createExtraNics(size, nics, nextNetwork); + + if (nics.size() == 1) { + nics.get(0).setDefaultNic(true); + } + } + + /** + * private transaction method to check and add devices to the nic list and update the info + */ + Pair addRequestedNicToNicListWithDeviceNumberAndRetrieveDefaultDevice(NicProfile requested, boolean[] deviceIds, int deviceId, Network nextNetwork, List nics, NicProfile defaultNic) + throws InsufficientAddressCapacityException, InsufficientVirtualNetworkCapacityException { + Pair rc = new Pair<>(null,null); + Boolean isDefaultNic = false; + if (vm != null && requested != null && requested.isDefaultNic()) { + isDefaultNic = true; + } + + while (deviceIds[deviceId] && deviceId < deviceIds.length) { + deviceId++; + } + + final Pair vmNicPair = allocateNic(requested, nextNetwork, isDefaultNic, deviceId, vm); + NicProfile vmNic = null; + if (vmNicPair != null) { + vmNic = vmNicPair.first(); + if (vmNic == null) { + return rc; + } + deviceId = vmNicPair.second(); + } + + final int devId = vmNic.getDeviceId(); + if (devId >= deviceIds.length) { + throw new IllegalArgumentException("Device id for nic is too large: " + vmNic); + } + if (deviceIds[devId]) { + throw new IllegalArgumentException("Conflicting device id for two different nics: " + vmNic); + } + + deviceIds[devId] = true; + if (vmNic.isDefaultNic()) { + if (defaultNic != null) { + throw new IllegalArgumentException("You cannot specify two nics as default nics: nic 1 = " + defaultNic + "; nic 2 = " + vmNic); + } + defaultNic = vmNic; + } + + nics.add(vmNic); + vm.addNic(vmNic); + saveExtraDhcpOptions(nextNetwork.getUuid(), vmNic.getId(), extraDhcpOptions); + rc.first(defaultNic); + rc.second(deviceId); + return rc; + } + + /** + * private transaction method to get oredered list of Network and NicProfile pair + * @return ordered list of Network and NicProfile pair + * @param networks the map od networks to nic profiles list + */ + private List> getOrderedNetworkNicProfileMapping(final LinkedHashMap> networks) { + List> profilesList = new ArrayList<>(); for (final Map.Entry> network : networks.entrySet()) { - final Network config = network.getKey(); List requestedProfiles = network.getValue(); if (requestedProfiles == null) { requestedProfiles = new ArrayList(); @@ -778,56 +848,71 @@ public void doInTransactionWithoutResult(final TransactionStatus status) throws if (requestedProfiles.isEmpty()) { requestedProfiles.add(null); } - for (final NicProfile requested : requestedProfiles) { - Boolean isDefaultNic = false; - if (vm != null && requested != null && requested.isDefaultNic()) { - isDefaultNic = true; - } - - while (deviceIds[deviceId] && deviceId < deviceIds.length) { - deviceId++; - } - - final Pair vmNicPair = allocateNic(requested, config, isDefaultNic, deviceId, vm); - NicProfile vmNic = null; - if (vmNicPair != null) { - vmNic = vmNicPair.first(); - if (vmNic == null) { - continue; - } - deviceId = vmNicPair.second(); - } - - final int devId = vmNic.getDeviceId(); - if (devId >= deviceIds.length) { - throw new IllegalArgumentException("Device id for nic is too large: " + vmNic); - } - if (deviceIds[devId]) { - throw new IllegalArgumentException("Conflicting device id for two different nics: " + vmNic); + profilesList.add(new Pair(network.getKey(), requested)); + } + } + profilesList.sort(new Comparator>() { + @Override + public int compare(Pair pair1, Pair pair2) { + int profile1Order = Integer.MAX_VALUE; + int profile2Order = Integer.MAX_VALUE; + if (pair1 != null && pair1.second() != null && pair1.second().getOrderIndex() != null) { + profile1Order = pair1.second().getOrderIndex(); } - - deviceIds[devId] = true; - - if (vmNic.isDefaultNic()) { - if (defaultNic != null) { - throw new IllegalArgumentException("You cannot specify two nics as default nics: nic 1 = " + defaultNic + "; nic 2 = " + vmNic); - } - defaultNic = vmNic; + if (pair2 != null && pair2.second() != null && pair2.second().getOrderIndex() != null) { + profile2Order = pair2.second().getOrderIndex(); } + return profile1Order - profile2Order; + } + }); + return profilesList; + } - nics.add(vmNic); - vm.addNic(vmNic); - saveExtraDhcpOptions(config.getUuid(), vmNic.getId(), extraDhcpOptions); + /** + * private transaction method to run over the objects and determine nic requirements + * @return the total numer of nics required + */ + private int determineNumberOfNicsRequired() { + int size = 0; + for (final Network ntwk : networks.keySet()) { + final List profiles = networks.get(ntwk); + if (profiles != null && !profiles.isEmpty()) { + size = size + profiles.size(); + } else { + size = size + 1; } } + + List netprereqs = templateDeployAsIsDetailsDao.listNetworkRequirementsByTemplateId(vm.getTemplate().getId()); + if (size < netprereqs.size()) { + size = netprereqs.size(); + } + return size; + } + + /** + * private transaction method to add nics as required + * @param size the number needed + * @param nics the list of nics present + * @param finalNetwork the network to add the nics to + * @throws InsufficientVirtualNetworkCapacityException great + * @throws InsufficientAddressCapacityException also magnificent, as the name sugests + */ + private void createExtraNics(int size, List nics, Network finalNetwork) throws InsufficientVirtualNetworkCapacityException, InsufficientAddressCapacityException { if (nics.size() != size) { s_logger.warn("Number of nics " + nics.size() + " doesn't match number of requested nics " + size); - throw new CloudRuntimeException("Number of nics " + nics.size() + " doesn't match number of requested networks " + size); - } - - if (nics.size() == 1) { - nics.get(0).setDefaultNic(true); + if (nics.size() > size) { + throw new CloudRuntimeException("Number of nics " + nics.size() + " doesn't match number of requested networks " + size); + } else { + if (finalNetwork == null) { + throw new CloudRuntimeException(String.format("can not assign network to %d remaining required NICs", size - nics.size())); + } + // create extra + for ( int extraNicNum = nics.size() ; extraNicNum < size; extraNicNum ++) { + final Pair vmNicPair = allocateNic(new NicProfile(), finalNetwork, false, extraNicNum, vm); + } + } } } }); @@ -1085,7 +1170,7 @@ boolean isNetworkImplemented(final NetworkVO network) { } Pair implementNetwork(final long networkId, final DeployDestination dest, final ReservationContext context, final boolean isRouter) throws ConcurrentOperationException, - ResourceUnavailableException, InsufficientCapacityException { + ResourceUnavailableException, InsufficientCapacityException { Pair implemented = null; if (!isRouter) { implemented = implementNetwork(networkId, dest, context); @@ -1105,7 +1190,7 @@ Pair implementNetwork(final long networkId, final Deploy @Override @DB public Pair implementNetwork(final long networkId, final DeployDestination dest, final ReservationContext context) throws ConcurrentOperationException, - ResourceUnavailableException, InsufficientCapacityException { + ResourceUnavailableException, InsufficientCapacityException { final Pair implemented = new Pair(null, null); NetworkVO network = _networksDao.findById(networkId); @@ -1396,7 +1481,7 @@ protected boolean reprogramNetworkRules(final long networkId, final Account call } protected boolean prepareElement(final NetworkElement element, final Network network, final NicProfile profile, final VirtualMachineProfile vmProfile, final DeployDestination dest, - final ReservationContext context) throws InsufficientCapacityException, ConcurrentOperationException, ResourceUnavailableException { + final ReservationContext context) throws InsufficientCapacityException, ConcurrentOperationException, ResourceUnavailableException { element.prepare(network, profile, vmProfile, dest, context); if (vmProfile.getType() == Type.User && element.getProvider() != null) { if (_networkModel.areServicesSupportedInNetwork(network.getId(), Service.Dhcp) @@ -1582,11 +1667,11 @@ public int getResourceCount(Network network){ if (element instanceof RedundantResource) { resourceCount= ((RedundantResource) element).getResourceCount(network); break; - } } } - return resourceCount; } + return resourceCount; + } @Override public void configureExtraDhcpOptions(Network network, long nicId, Map extraDhcpOptions) { @@ -1628,12 +1713,12 @@ public void setHypervisorHostname(VirtualMachineProfile vm, DeployDestination de _networkModel.getNetworkTag(vm.getHypervisorType(), network)); for (final NetworkElement element : networkElements) { if (_networkModel.areServicesSupportedInNetwork(network.getId(), Service.UserData) && element instanceof UserDataServiceProvider) { - if (element instanceof ConfigDriveNetworkElement && !migrationSuccessful || element instanceof VirtualRouterElement && migrationSuccessful) { - final UserDataServiceProvider sp = (UserDataServiceProvider) element; - if (!sp.saveHypervisorHostname(profile, network, vm, dest)) { - throw new CloudRuntimeException("Failed to Add hypervisor hostname"); - } + if (element instanceof ConfigDriveNetworkElement && !migrationSuccessful || element instanceof VirtualRouterElement && migrationSuccessful) { + final UserDataServiceProvider sp = (UserDataServiceProvider) element; + if (!sp.saveHypervisorHostname(profile, network, vm, dest)) { + throw new CloudRuntimeException("Failed to Add hypervisor hostname"); } + } } } } @@ -1661,7 +1746,7 @@ public void doInTransactionWithoutResult(final TransactionStatus status) { @Override public void prepare(final VirtualMachineProfile vmProfile, final DeployDestination dest, final ReservationContext context) throws InsufficientCapacityException, ConcurrentOperationException, - ResourceUnavailableException { + ResourceUnavailableException { final List nics = _nicDao.listByVmId(vmProfile.getId()); // we have to implement default nics first - to ensure that default network elements start up first in multiple @@ -2246,9 +2331,9 @@ public Network createGuestNetwork(final long networkOfferingId, final String nam @DB private Network createGuestNetwork(final long networkOfferingId, final String name, final String displayText, final String gateway, final String cidr, String vlanId, - boolean bypassVlanOverlapCheck, String networkDomain, final Account owner, final Long domainId, final PhysicalNetwork pNtwk, - final long zoneId, final ACLType aclType, Boolean subdomainAccess, final Long vpcId, final String ip6Gateway, final String ip6Cidr, - final Boolean isDisplayNetworkEnabled, final String isolatedPvlan, Network.PVlanType isolatedPvlanType, String externalId, final Boolean isPrivateNetwork) throws ConcurrentOperationException, InsufficientCapacityException, ResourceAllocationException { + boolean bypassVlanOverlapCheck, String networkDomain, final Account owner, final Long domainId, final PhysicalNetwork pNtwk, + final long zoneId, final ACLType aclType, Boolean subdomainAccess, final Long vpcId, final String ip6Gateway, final String ip6Cidr, + final Boolean isDisplayNetworkEnabled, final String isolatedPvlan, Network.PVlanType isolatedPvlanType, String externalId, final Boolean isPrivateNetwork) throws ConcurrentOperationException, InsufficientCapacityException, ResourceAllocationException { final NetworkOfferingVO ntwkOff = _networkOfferingDao.findById(networkOfferingId); final DataCenterVO zone = _dcDao.findById(zoneId); @@ -2364,12 +2449,12 @@ private Network createGuestNetwork(final long networkOfferingId, final String na URI secondaryUri = isNotBlank(isolatedPvlan) ? BroadcastDomainType.fromString(isolatedPvlan) : null; //don't allow to specify vlan tag used by physical network for dynamic vlan allocation if (!(bypassVlanOverlapCheck && ntwkOff.getGuestType() == GuestType.Shared) && _dcDao.findVnet(zoneId, pNtwk.getId(), BroadcastDomainType.getValue(uri)).size() > 0) { - throw new InvalidParameterValueException("The VLAN tag " + vlanId + " is already being used for dynamic vlan allocation for the guest network in zone " + throw new InvalidParameterValueException("The VLAN tag to use for new guest network, " + vlanId + " is already being used for dynamic vlan allocation for the guest network in zone " + zone.getName()); } if (secondaryUri != null && !(bypassVlanOverlapCheck && ntwkOff.getGuestType() == GuestType.Shared) && _dcDao.findVnet(zoneId, pNtwk.getId(), BroadcastDomainType.getValue(secondaryUri)).size() > 0) { - throw new InvalidParameterValueException("The VLAN tag " + isolatedPvlan + " is already being used for dynamic vlan allocation for the guest network in zone " + throw new InvalidParameterValueException("The VLAN tag for isolated PVLAN " + isolatedPvlan + " is already being used for dynamic vlan allocation for the guest network in zone " + zone.getName()); } if (! UuidUtils.validateUUID(vlanId)){ @@ -2586,7 +2671,7 @@ private boolean hasGuestBypassVlanOverlapCheck(final boolean bypassVlanOverlapCh return bypassVlanOverlapCheck && (ntwkOff.getGuestType() != GuestType.Isolated || isPrivateNetwork); } - /** + /** * Checks for L2 network offering services. Only 2 cases allowed: * - No services * - User Data service only, provided by ConfigDrive @@ -3014,7 +3099,7 @@ public void reallyRun() { @Override public boolean startNetwork(final long networkId, final DeployDestination dest, final ReservationContext context) throws ConcurrentOperationException, ResourceUnavailableException, - InsufficientCapacityException { + InsufficientCapacityException { // Check if network exists final NetworkVO network = _networksDao.findById(networkId); @@ -3037,7 +3122,7 @@ public boolean startNetwork(final long networkId, final DeployDestination dest, @Override public boolean restartNetwork(final Long networkId, final Account callerAccount, final User callerUser, final boolean cleanup) throws ConcurrentOperationException, ResourceUnavailableException, - InsufficientCapacityException { + InsufficientCapacityException { boolean status = true; boolean restartRequired = false; final NetworkVO network = _networksDao.findById(networkId); @@ -3289,10 +3374,10 @@ protected boolean isSharedNetworkOfferingWithServices(final long networkOffering final NetworkOfferingVO networkOffering = _networkOfferingDao.findById(networkOfferingId); if (networkOffering.getGuestType() == Network.GuestType.Shared && (_networkModel.areServicesSupportedByNetworkOffering(networkOfferingId, Service.SourceNat) - || _networkModel.areServicesSupportedByNetworkOffering(networkOfferingId, Service.StaticNat) - || _networkModel.areServicesSupportedByNetworkOffering(networkOfferingId, Service.Firewall) - || _networkModel.areServicesSupportedByNetworkOffering(networkOfferingId, Service.PortForwarding) || _networkModel.areServicesSupportedByNetworkOffering( - networkOfferingId, Service.Lb))) { + || _networkModel.areServicesSupportedByNetworkOffering(networkOfferingId, Service.StaticNat) + || _networkModel.areServicesSupportedByNetworkOffering(networkOfferingId, Service.Firewall) + || _networkModel.areServicesSupportedByNetworkOffering(networkOfferingId, Service.PortForwarding) || _networkModel.areServicesSupportedByNetworkOffering( + networkOfferingId, Service.Lb))) { return true; } return false; @@ -4164,4 +4249,4 @@ public ConfigKey[] getConfigKeys() { GuestDomainSuffix, NetworkThrottlingRate, MinVRVersion, PromiscuousMode, MacAddressChanges, ForgedTransmits, RollingRestartEnabled}; } -} +} \ No newline at end of file diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/StorageOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/StorageOrchestrator.java new file mode 100644 index 000000000000..85b182a3dc21 --- /dev/null +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/StorageOrchestrator.java @@ -0,0 +1,451 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.engine.orchestration; + +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.Hashtable; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import javax.inject.Inject; +import javax.naming.ConfigurationException; + +import org.apache.cloudstack.api.response.MigrationResponse; +import org.apache.cloudstack.engine.orchestration.service.StorageOrchestrationService; +import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.engine.subsystem.api.storage.SecondaryStorageService; +import org.apache.cloudstack.engine.subsystem.api.storage.SecondaryStorageService.DataObjectResult; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; +import org.apache.cloudstack.framework.async.AsyncCallFuture; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.storage.ImageStoreService.MigrationPolicy; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao; +import org.apache.commons.math3.stat.descriptive.moment.Mean; +import org.apache.commons.math3.stat.descriptive.moment.StandardDeviation; +import org.apache.log4j.Logger; + +import com.cloud.server.StatsCollector; +import com.cloud.storage.DataStoreRole; +import com.cloud.storage.SnapshotVO; +import com.cloud.storage.StorageManager; +import com.cloud.storage.StorageService; +import com.cloud.storage.StorageStats; +import com.cloud.storage.dao.SnapshotDao; +import com.cloud.utils.Pair; +import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.exception.CloudRuntimeException; + +public class StorageOrchestrator extends ManagerBase implements StorageOrchestrationService, Configurable { + + private static final Logger s_logger = Logger.getLogger(StorageOrchestrator.class); + @Inject + SnapshotDataStoreDao snapshotDataStoreDao; + @Inject + SnapshotDao snapshotDao; + @Inject + SnapshotDataFactory snapshotFactory; + @Inject + DataStoreManager dataStoreManager; + @Inject + StatsCollector statsCollector; + @Inject + public StorageService storageService; + @Inject + ConfigurationDao configDao; + @Inject + private SecondaryStorageService secStgSrv; + @Inject + TemplateDataStoreDao templateDataStoreDao; + @Inject + VolumeDataStoreDao volumeDataStoreDao; + @Inject + DataMigrationUtility migrationHelper; + + ConfigKey ImageStoreImbalanceThreshold = new ConfigKey<>("Advanced", Double.class, + "image.store.imbalance.threshold", + "0.3", + "The storage imbalance threshold that is compared with the standard deviation percentage for a storage utilization metric. " + + "The value is a percentage in decimal format.", + true, ConfigKey.Scope.Global); + + Integer numConcurrentCopyTasksPerSSVM = 2; + private double imageStoreCapacityThreshold = 0.90; + + @Override + public String getConfigComponentName() { + return StorageOrchestrationService.class.getName(); + } + + @Override + public ConfigKey[] getConfigKeys() { + return new ConfigKey[]{ImageStoreImbalanceThreshold}; + } + + static class MigrateBlockingQueue extends ArrayBlockingQueue { + + MigrateBlockingQueue(int size) { + super(size); + } + + public boolean offer(T task) { + try { + this.put(task); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + return true; + } + } + + @Override + public boolean configure(String name, Map params) throws ConfigurationException { + numConcurrentCopyTasksPerSSVM = StorageManager.SecStorageMaxMigrateSessions.value(); + return true; + } + + @Override + public MigrationResponse migrateData(Long srcDataStoreId, List destDatastores, MigrationPolicy migrationPolicy) { + List files = new LinkedList<>(); + boolean success = true; + String message = null; + + migrationHelper.checkIfCompleteMigrationPossible(migrationPolicy, srcDataStoreId); + DataStore srcDatastore = dataStoreManager.getDataStore(srcDataStoreId, DataStoreRole.Image); + Map, Long>> snapshotChains = new HashMap<>(); + files = migrationHelper.getSortedValidSourcesList(srcDatastore, snapshotChains); + + if (files.isEmpty()) { + return new MigrationResponse("No files in Image store "+srcDatastore.getId()+ " to migrate", migrationPolicy.toString(), true); + } + Map> storageCapacities = new Hashtable<>(); + for (Long storeId : destDatastores) { + storageCapacities.put(storeId, new Pair<>(null, null)); + } + storageCapacities.put(srcDataStoreId, new Pair<>(null, null)); + if (migrationPolicy == MigrationPolicy.COMPLETE) { + s_logger.debug("Setting source image store "+srcDatastore.getId()+ " to read-only"); + storageService.updateImageStoreStatus(srcDataStoreId, true); + } + + storageCapacities = getStorageCapacities(storageCapacities, srcDataStoreId); + double meanstddev = getStandardDeviation(storageCapacities); + double threshold = ImageStoreImbalanceThreshold.value(); + MigrationResponse response = null; + ThreadPoolExecutor executor = new ThreadPoolExecutor(numConcurrentCopyTasksPerSSVM , numConcurrentCopyTasksPerSSVM, 30, + TimeUnit.MINUTES, new MigrateBlockingQueue<>(numConcurrentCopyTasksPerSSVM)); + Date start = new Date(); + if (meanstddev < threshold && migrationPolicy == MigrationPolicy.BALANCE) { + s_logger.debug("mean std deviation of the image stores is below threshold, no migration required"); + response = new MigrationResponse("Migration not required as system seems balanced", migrationPolicy.toString(), true); + return response; + } + + List>> futures = new ArrayList<>(); + while (true) { + DataObject chosenFileForMigration = null; + if (files.size() > 0) { + chosenFileForMigration = files.remove(0); + } + + storageCapacities = getStorageCapacities(storageCapacities, srcDataStoreId); + List orderedDS = migrationHelper.sortDataStores(storageCapacities); + Long destDatastoreId = orderedDS.get(0); + + if (chosenFileForMigration == null || destDatastoreId == null || (destDatastoreId == srcDatastore.getId() && migrationPolicy == MigrationPolicy.BALANCE) ) { + Pair result = migrateCompleted(destDatastoreId, srcDatastore, files, migrationPolicy); + message = result.first(); + success = result.second(); + break; + } + + if (migrationPolicy == MigrationPolicy.COMPLETE && destDatastoreId == srcDatastore.getId()) { + destDatastoreId = orderedDS.get(1); + } + + if (chosenFileForMigration.getSize() > storageCapacities.get(destDatastoreId).first()) { + s_logger.debug("file: " + chosenFileForMigration.getId() + " too large to be migrated to " + destDatastoreId); + continue; + } + + if (shouldMigrate(chosenFileForMigration, srcDatastore.getId(), destDatastoreId, storageCapacities, snapshotChains, migrationPolicy)) { + storageCapacities = migrateAway(chosenFileForMigration, storageCapacities, snapshotChains, srcDatastore, destDatastoreId, executor, futures); + } else { + if (migrationPolicy == MigrationPolicy.BALANCE) { + continue; + } + message = "Complete migration failed. Please set the source Image store to read-write mode if you want to continue using it"; + success = false; + break; + } + } + Date end = new Date(); + handleSnapshotMigration(srcDataStoreId, start, end, migrationPolicy, futures, storageCapacities, executor); + return handleResponse(futures, migrationPolicy, message, success); + } + + protected Pair migrateCompleted(Long destDatastoreId, DataStore srcDatastore, List files, MigrationPolicy migrationPolicy) { + String message = ""; + boolean success = true; + if (destDatastoreId == srcDatastore.getId() && !files.isEmpty()) { + if (migrationPolicy == MigrationPolicy.BALANCE) { + s_logger.debug("Migration completed : data stores have been balanced "); + if (destDatastoreId == srcDatastore.getId()) { + message = "Seems like source datastore has more free capacity than the destination(s)"; + } + message += "Image stores have been attempted to be balanced"; + success = true; + } else { + message = "Files not completely migrated from "+ srcDatastore.getId() + ". Datastore (source): " + srcDatastore.getId() + "has equal or more free space than destination."+ + " If you want to continue using the Image Store, please change the read-only status using 'update imagestore' command"; + success = false; + } + } else { + message = "Migration completed"; + } + return new Pair(message, success); + } + + protected Map> migrateAway(DataObject chosenFileForMigration, Map> storageCapacities, + Map, Long>> snapshotChains, DataStore srcDatastore, Long destDatastoreId, ThreadPoolExecutor executor, + List>> futures) { + Long fileSize = migrationHelper.getFileSize(chosenFileForMigration, snapshotChains); + storageCapacities = assumeMigrate(storageCapacities, srcDatastore.getId(), destDatastoreId, fileSize); + long activeSsvms = migrationHelper.activeSSVMCount(srcDatastore); + long totalJobs = activeSsvms * numConcurrentCopyTasksPerSSVM; + // Increase thread pool size with increase in number of SSVMs + if ( totalJobs > executor.getCorePoolSize()) { + executor.setMaximumPoolSize((int) (totalJobs)); + executor.setCorePoolSize((int) (totalJobs)); + } + + MigrateDataTask task = new MigrateDataTask(chosenFileForMigration, srcDatastore, dataStoreManager.getDataStore(destDatastoreId, DataStoreRole.Image)); + if (chosenFileForMigration instanceof SnapshotInfo ) { + task.setSnapshotChains(snapshotChains); + } + futures.add((executor.submit(task))); + s_logger.debug("Migration of file " + chosenFileForMigration.getId() + " is initiated"); + return storageCapacities; + } + + + + private MigrationResponse handleResponse(List>> futures, MigrationPolicy migrationPolicy, String message, boolean success) { + int successCount = 0; + for (Future> future : futures) { + try { + AsyncCallFuture res = future.get(); + if (res.get().isSuccess()) { + successCount++; + } + } catch ( InterruptedException | ExecutionException e) { + s_logger.warn("Failed to get result"); + continue; + } + } + message += ". successful migrations: "+successCount; + return new MigrationResponse(message, migrationPolicy.toString(), success); + } + + private void handleSnapshotMigration(Long srcDataStoreId, Date start, Date end, MigrationPolicy policy, + List>> futures, Map> storageCapacities, ThreadPoolExecutor executor) { + DataStore srcDatastore = dataStoreManager.getDataStore(srcDataStoreId, DataStoreRole.Image); + List snaps = snapshotDataStoreDao.findSnapshots(srcDataStoreId, start, end); + if (!snaps.isEmpty()) { + for (SnapshotDataStoreVO snap : snaps) { + SnapshotVO snapshotVO = snapshotDao.findById(snap.getSnapshotId()); + SnapshotInfo snapshotInfo = snapshotFactory.getSnapshot(snapshotVO.getSnapshotId(), DataStoreRole.Image); + SnapshotInfo parentSnapshot = snapshotInfo.getParent(); + + if (parentSnapshot == null && policy == MigrationPolicy.COMPLETE) { + List dstores = migrationHelper.sortDataStores(storageCapacities); + Long storeId = dstores.get(0); + if (storeId.equals(srcDataStoreId)) { + storeId = dstores.get(1); + } + DataStore datastore = dataStoreManager.getDataStore(storeId, DataStoreRole.Image); + futures.add(executor.submit(new MigrateDataTask(snapshotInfo, srcDatastore, datastore))); + } + if (parentSnapshot != null) { + DataStore parentDS = dataStoreManager.getDataStore(parentSnapshot.getDataStore().getId(), DataStoreRole.Image); + if (parentDS.getId() != snapshotInfo.getDataStore().getId()) { + futures.add(executor.submit(new MigrateDataTask(snapshotInfo, srcDatastore, parentDS))); + } + } + } + } + } + + private Map> getStorageCapacities(Map> storageCapacities, Long srcDataStoreId) { + Map> capacities = new Hashtable<>(); + for (Long storeId : storageCapacities.keySet()) { + StorageStats stats = statsCollector.getStorageStats(storeId); + if (stats != null) { + if (storageCapacities.get(storeId) == null || storageCapacities.get(storeId).first() == null || storageCapacities.get(storeId).second() == null) { + capacities.put(storeId, new Pair<>(stats.getCapacityBytes() - stats.getByteUsed(), stats.getCapacityBytes())); + } else { + long totalCapacity = stats.getCapacityBytes(); + Long freeCapacity = totalCapacity - stats.getByteUsed(); + if (storeId.equals(srcDataStoreId) || freeCapacity < storageCapacities.get(storeId).first()) { + capacities.put(storeId, new Pair<>(freeCapacity, totalCapacity)); + } else { + capacities.put(storeId, storageCapacities.get(storeId)); + } + } + } else { + throw new CloudRuntimeException("Stats Collector hasn't yet collected metrics from the Image store, kindly try again later"); + } + } + return capacities; + } + + + /** + * + * @param storageCapacities Map comprising the metrics(free and total capacities) of the images stores considered + * @return mean standard deviation + */ + private double getStandardDeviation(Map> storageCapacities) { + double[] freeCapacities = storageCapacities.values().stream().mapToDouble(x -> ((double) x.first() / x.second())).toArray(); + double mean = calculateStorageMean(freeCapacities); + return (calculateStorageStandardDeviation(freeCapacities, mean) / mean); + } + + /** + * + * @param storageCapacities Map comprising the metrics(free and total capacities) of the images stores considered + * @param srcDsId source image store ID from where data is to be migrated + * @param destDsId destination image store ID to where data is to be migrated + * @param fileSize size of the data object to be migrated so as to recompute the storage metrics + * @return a map - Key: Datastore ID ; Value: Pair + */ + private Map> assumeMigrate(Map> storageCapacities, Long srcDsId, Long destDsId, Long fileSize) { + Map> modifiedCapacities = new Hashtable<>(); + modifiedCapacities.putAll(storageCapacities); + Pair srcDSMetrics = storageCapacities.get(srcDsId); + Pair destDSMetrics = storageCapacities.get(destDsId); + modifiedCapacities.put(srcDsId, new Pair<>(srcDSMetrics.first() + fileSize, srcDSMetrics.second())); + modifiedCapacities.put(destDsId, new Pair<>(destDSMetrics.first() - fileSize, destDSMetrics.second())); + return modifiedCapacities; + } + + /** + * This function determines if migration should in fact take place or not : + * - For Balanced migration - the mean standard deviation is calculated before and after (supposed) migration + * and a decision is made if migration is afterall beneficial + * - For Complete migration - We check if the destination image store has sufficient capacity i.e., below the threshold of (90%) + * and then proceed with the migration + * @param chosenFile file for migration + * @param srcDatastoreId source image store ID from where data is to be migrated + * @param destDatastoreId destination image store ID to where data is to be migrated + * @param storageCapacities Map comprising the metrics(free and total capacities) of the images stores considered + * @param snapshotChains Map containing details of chain of snapshots and their cumulative size + * @param migrationPolicy determines whether a "Balance" or "Complete" migration operation is to be performed + * @return + */ + private boolean shouldMigrate(DataObject chosenFile, Long srcDatastoreId, Long destDatastoreId, Map> storageCapacities, + Map, Long>> snapshotChains, MigrationPolicy migrationPolicy) { + + if (migrationPolicy == MigrationPolicy.BALANCE) { + double meanStdDevCurrent = getStandardDeviation(storageCapacities); + + Long fileSize = migrationHelper.getFileSize(chosenFile, snapshotChains); + Map> proposedCapacities = assumeMigrate(storageCapacities, srcDatastoreId, destDatastoreId, fileSize); + double meanStdDevAfter = getStandardDeviation(proposedCapacities); + + if (meanStdDevAfter > meanStdDevCurrent) { + s_logger.debug("migrating the file doesn't prove to be beneficial, skipping migration"); + return false; + } + + Double threshold = ImageStoreImbalanceThreshold.value(); + if (meanStdDevCurrent > threshold && storageCapacityBelowThreshold(storageCapacities, destDatastoreId)) { + return true; + } + return true; + } else { + if (storageCapacityBelowThreshold(storageCapacities, destDatastoreId)) { + return true; + } + } + return false; + } + + private boolean storageCapacityBelowThreshold(Map> storageCapacities, Long destStoreId) { + Pair imageStoreCapacity = storageCapacities.get(destStoreId); + if (imageStoreCapacity != null && (imageStoreCapacity.first() / (imageStoreCapacity.second() * 1.0)) <= imageStoreCapacityThreshold) { + s_logger.debug("image store: " + destStoreId + " has sufficient capacity to proceed with migration of file"); + return true; + } + s_logger.debug("Image store capacity threshold exceeded, migration not possible"); + return false; + } + + private double calculateStorageMean(double[] storageMetrics) { + return new Mean().evaluate(storageMetrics); + } + + private double calculateStorageStandardDeviation(double[] metricValues, double mean) { + StandardDeviation standardDeviation = new StandardDeviation(false); + return standardDeviation.evaluate(metricValues, mean); + } + + private class MigrateDataTask implements Callable> { + private DataObject file; + private DataStore srcDataStore; + private DataStore destDataStore; + private Map, Long>> snapshotChain; + public MigrateDataTask(DataObject file, DataStore srcDataStore, DataStore destDataStore) { + this.file = file; + this.srcDataStore = srcDataStore; + this.destDataStore = destDataStore; + } + + public void setSnapshotChains(Map, Long>> snapshotChain) { + this.snapshotChain = snapshotChain; + } + + public Map, Long>> getSnapshotChain() { + return snapshotChain; + } + public DataObject getFile() { + return file; + } + + @Override + public AsyncCallFuture call() throws Exception { + return secStgSrv.migrateData(file, srcDataStore, destDataStore, snapshotChain); + } + } +} diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java index 3e68d3a4ab05..a51aae7c8f4c 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java @@ -19,6 +19,7 @@ package org.apache.cloudstack.engine.orchestration; import java.util.ArrayList; +import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -26,10 +27,18 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.agent.api.to.DatadiskTO; +import com.cloud.storage.VolumeDetailVO; +import com.cloud.storage.dao.VMTemplateDetailsDao; +import com.cloud.utils.StringUtils; +import com.cloud.vm.UserVmDetailVO; +import com.cloud.vm.VmDetailConstants; +import com.cloud.vm.dao.UserVmDetailsDao; import org.apache.cloudstack.api.command.admin.vm.MigrateVMCmd; import org.apache.cloudstack.api.command.admin.volume.MigrateVolumeCmdByAdmin; import org.apache.cloudstack.api.command.user.volume.MigrateVolumeCmd; @@ -51,6 +60,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.StorageStrategyFactory; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeService; @@ -68,6 +78,7 @@ import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; +import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; import com.cloud.agent.api.to.DataTO; @@ -141,6 +152,8 @@ import com.cloud.vm.dao.UserVmDao; import static com.cloud.utils.NumbersUtil.toHumanReadableSize; +import static com.cloud.storage.resource.StorageProcessor.REQUEST_TEMPLATE_RELOAD; + public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrationService, Configurable { public enum UserVmCloneType { @@ -197,6 +210,12 @@ public enum UserVmCloneType { protected UserVmCloneSettingDao _vmCloneSettingDao; @Inject StorageStrategyFactory _storageStrategyFactory; + @Inject + VMTemplateDetailsDao templateDetailsDao; + @Inject + TemplateService templateService; + @Inject + UserVmDetailsDao userVmDetailsDao; private final StateMachine2 _volStateMachine; protected List _storagePoolAllocators; @@ -296,6 +315,34 @@ public StoragePool findStoragePool(DiskProfile dskCh, DataCenter dc, Pod pod, Lo return null; } + @Override + public StoragePool findChildDataStoreInDataStoreCluster(DataCenter dc, Pod pod, Long clusterId, Long hostId, VirtualMachine vm, Long datastoreClusterId) { + Long podId = null; + if (pod != null) { + podId = pod.getId(); + } else if (clusterId != null) { + Cluster cluster = _entityMgr.findById(Cluster.class, clusterId); + if (cluster != null) { + podId = cluster.getPodId(); + } + } + List childDatastores = _storagePoolDao.listChildStoragePoolsInDatastoreCluster(datastoreClusterId); + List suitablePools = new ArrayList(); + + for (StoragePoolVO childDatastore: childDatastores) + suitablePools.add((StoragePool)dataStoreMgr.getDataStore(childDatastore.getId(), DataStoreRole.Primary)); + + VirtualMachineProfile profile = new VirtualMachineProfileImpl(vm); + for (StoragePoolAllocator allocator : _storagePoolAllocators) { + DataCenterDeployment plan = new DataCenterDeployment(dc.getId(), podId, clusterId, hostId, null, null); + final List poolList = allocator.reorderPools(suitablePools, profile, plan); + + if (poolList != null && !poolList.isEmpty()) { + return (StoragePool)dataStoreMgr.getDataStore(poolList.get(0).getId(), DataStoreRole.Primary); + } + } + return null; + } public Pair findPod(VirtualMachineTemplate template, ServiceOffering offering, DataCenter dc, long accountId, Set avoids) { for (PodAllocator allocator : _podAllocators) { final Pair pod = allocator.allocateTo(template, offering, dc, accountId, avoids); @@ -484,7 +531,7 @@ protected DiskProfile createDiskCharacteristics(VolumeInfo volume, VirtualMachin @DB public VolumeInfo copyVolumeFromSecToPrimary(VolumeInfo volume, VirtualMachine vm, VirtualMachineTemplate template, DataCenter dc, Pod pod, Long clusterId, ServiceOffering offering, - DiskOffering diskOffering, List avoids, long size, HypervisorType hyperType) throws NoTransitionException { + DiskOffering diskOffering, List avoids, long size, HypervisorType hyperType) throws NoTransitionException { final HashSet avoidPools = new HashSet(avoids); DiskProfile dskCh = createDiskCharacteristics(volume, template, dc, diskOffering); @@ -517,7 +564,7 @@ public VolumeInfo copyVolumeFromSecToPrimary(VolumeInfo volume, VirtualMachine v @DB public VolumeInfo createVolume(VolumeInfo volume, VirtualMachine vm, VirtualMachineTemplate template, DataCenter dc, Pod pod, Long clusterId, ServiceOffering offering, DiskOffering diskOffering, - List avoids, long size, HypervisorType hyperType) { + List avoids, long size, HypervisorType hyperType) { // update the volume's hv_ss_reserve (hypervisor snapshot reserve) from a disk offering (used for managed storage) volume = volService.updateHypervisorSnapshotReserveForVolume(diskOffering, volume.getId(), hyperType); @@ -563,7 +610,7 @@ public VolumeInfo createVolume(VolumeInfo volume, VirtualMachine vm, VirtualMach try { VolumeApiResult result = future.get(); if (result.isFailed()) { - if (result.getResult().contains("request template reload") && (i == 0)) { + if (result.getResult().contains(REQUEST_TEMPLATE_RELOAD) && (i == 0)) { s_logger.debug("Retry template re-deploy for vmware"); continue; } else { @@ -659,7 +706,7 @@ protected DiskProfile toDiskProfile(Volume vol, DiskOffering offering) { @Override public DiskProfile allocateRawVolume(Type type, String name, DiskOffering offering, Long size, Long minIops, Long maxIops, VirtualMachine vm, VirtualMachineTemplate template, Account owner, - Long deviceId) { + Long deviceId) { if (size == null) { size = offering.getDiskSize(); } else { @@ -706,19 +753,23 @@ public DiskProfile allocateRawVolume(Type type, String name, DiskOffering offeri return toDiskProfile(vol, offering); } - @Override - public DiskProfile allocateTemplatedVolume(Type type, String name, DiskOffering offering, Long rootDisksize, Long minIops, Long maxIops, VirtualMachineTemplate template, VirtualMachine vm, - Account owner) { + private DiskProfile allocateTemplatedVolume(Type type, String name, DiskOffering offering, Long rootDisksize, Long minIops, Long maxIops, VirtualMachineTemplate template, VirtualMachine vm, + Account owner, long deviceId, String configurationId) { assert (template.getFormat() != ImageFormat.ISO) : "ISO is not a template really...."; Long size = _tmpltMgr.getTemplateSize(template.getId(), vm.getDataCenterId()); if (rootDisksize != null) { - rootDisksize = rootDisksize * 1024 * 1024 * 1024; - if (rootDisksize > size) { - s_logger.debug("Using root disk size of " + toHumanReadableSize(rootDisksize) + " Bytes for volume " + name); + if (template.isDeployAsIs()) { + // Volume size specified from template deploy-as-is size = rootDisksize; } else { - s_logger.debug("Using root disk size of " + toHumanReadableSize(size) + " Bytes for volume " + name + "since specified root disk size of " + toHumanReadableSize(rootDisksize) + " Bytes is smaller than template"); + rootDisksize = rootDisksize * 1024 * 1024 * 1024; + if (rootDisksize > size) { + s_logger.debug("Using root disk size of " + toHumanReadableSize(rootDisksize) + " Bytes for volume " + name); + size = rootDisksize; + } else { + s_logger.debug("Using root disk size of " + toHumanReadableSize(rootDisksize) + " Bytes for volume " + name + "since specified root disk size of " + rootDisksize + " Bytes is smaller than template"); + } } } @@ -732,13 +783,9 @@ public DiskProfile allocateTemplatedVolume(Type type, String name, DiskOffering } vol.setTemplateId(template.getId()); - if (type.equals(Type.ROOT)) { - vol.setDeviceId(0l); - if (!vm.getType().equals(VirtualMachine.Type.User)) { - vol.setRecreatable(true); - } - } else { - vol.setDeviceId(1l); + vol.setDeviceId(deviceId); + if (type.equals(Type.ROOT) && !vm.getType().equals(VirtualMachine.Type.User)) { + vol.setRecreatable(true); } if (vm.getType() == VirtualMachine.Type.User) { @@ -748,6 +795,11 @@ public DiskProfile allocateTemplatedVolume(Type type, String name, DiskOffering vol = _volsDao.persist(vol); + if (StringUtils.isNotBlank(configurationId)) { + VolumeDetailVO deployConfigurationDetail = new VolumeDetailVO(vol.getId(), VmDetailConstants.DEPLOY_AS_IS_CONFIGURATION, configurationId, false); + _volDetailDao.persist(deployConfigurationDetail); + } + // Create event and update resource count for volumes if vm is a user vm if (vm.getType() == VirtualMachine.Type.User) { @@ -766,6 +818,51 @@ public DiskProfile allocateTemplatedVolume(Type type, String name, DiskOffering return toDiskProfile(vol, offering); } + @Override + public List allocateTemplatedVolumes(Type type, String name, DiskOffering offering, Long rootDisksize, Long minIops, Long maxIops, VirtualMachineTemplate template, VirtualMachine vm, + Account owner) { + int volumesNumber = 1; + List templateAsIsDisks = null; + String configurationId = null; + if (template.isDeployAsIs()) { + UserVmDetailVO configurationDetail = userVmDetailsDao.findDetail(vm.getId(), VmDetailConstants.DEPLOY_AS_IS_CONFIGURATION); + if (configurationDetail != null) { + configurationId = configurationDetail.getValue(); + } + templateAsIsDisks = _tmpltMgr.getTemplateDisksOnImageStore(template.getId(), DataStoreRole.Image, configurationId); + if (CollectionUtils.isNotEmpty(templateAsIsDisks)) { + templateAsIsDisks = templateAsIsDisks.stream() + .filter(x -> !x.isIso()) + .sorted(Comparator.comparing(DatadiskTO::getDiskNumber)) + .collect(Collectors.toList()); + } + volumesNumber = templateAsIsDisks.size(); + } + + if (volumesNumber < 1) { + throw new CloudRuntimeException("Unable to create any volume from template " + template.getName()); + } + + List profiles = new ArrayList<>(); + + for (int number = 0; number < volumesNumber; number++) { + String volumeName = name; + Long volumeSize = rootDisksize; + long deviceId = type.equals(Type.ROOT) ? 0L : 1L; + if (template.isDeployAsIs()) { + int volumeNameSuffix = templateAsIsDisks.get(number).getDiskNumber(); + volumeName = String.format("%s-%d", volumeName, volumeNameSuffix); + volumeSize = templateAsIsDisks.get(number).getVirtualSize(); + deviceId = templateAsIsDisks.get(number).getDiskNumber(); + } + s_logger.info(String.format("adding disk object %s to %s", volumeName, vm.getInstanceName())); + DiskProfile diskProfile = allocateTemplatedVolume(type, volumeName, offering, volumeSize, minIops, maxIops, + template, vm, owner, deviceId, configurationId); + profiles.add(diskProfile); + } + return profiles; + } + private ImageFormat getSupportedImageFormatForCluster(HypervisorType hyperType) { if (hyperType == HypervisorType.XenServer) { return ImageFormat.VHD; @@ -796,7 +893,7 @@ private boolean isSupportedImageFormatForCluster(VolumeInfo volume, HypervisorTy } private VolumeInfo copyVolume(StoragePool rootDiskPool, VolumeInfo volume, VirtualMachine vm, VirtualMachineTemplate rootDiskTmplt, DataCenter dcVO, Pod pod, DiskOffering diskVO, - ServiceOffering svo, HypervisorType rootDiskHyperType) throws NoTransitionException { + ServiceOffering svo, HypervisorType rootDiskHyperType) throws NoTransitionException { if (!isSupportedImageFormatForCluster(volume, rootDiskHyperType)) { throw new InvalidParameterValueException("Failed to attach volume to VM since volumes format " + volume.getFormat().getFileExtension() + " is not compatible with the vm hypervisor type"); @@ -1328,7 +1425,7 @@ private Pair recreateVolume(VolumeVO vol, VirtualMachinePro try { result = future.get(); if (result.isFailed()) { - if (result.getResult().contains("request template reload") && (i == 0)) { + if (result.getResult().contains(REQUEST_TEMPLATE_RELOAD) && (i == 0)) { s_logger.debug("Retry template re-deploy for vmware"); continue; } else { @@ -1704,4 +1801,4 @@ public void doInTransactionWithoutResult(TransactionStatus status) { } }); } -} +} \ No newline at end of file diff --git a/engine/orchestration/src/main/resources/META-INF/cloudstack/core/spring-engine-orchestration-core-context.xml b/engine/orchestration/src/main/resources/META-INF/cloudstack/core/spring-engine-orchestration-core-context.xml index 3ded395bb66f..66335a6b0579 100644 --- a/engine/orchestration/src/main/resources/META-INF/cloudstack/core/spring-engine-orchestration-core-context.xml +++ b/engine/orchestration/src/main/resources/META-INF/cloudstack/core/spring-engine-orchestration-core-context.xml @@ -44,6 +44,11 @@ value="#{storagePoolAllocatorsRegistry.registered}" /> + + + { +public interface VsphereStoragePolicyDao extends GenericDao { + + public VsphereStoragePolicyVO findByPolicyId(Long zoneId, String policyId); + + public List findByZoneId(Long zoneId); - boolean existsOption(long templateId, String key); - TemplateOVFPropertyVO findByTemplateAndKey(long templateId, String key); - void saveOptions(List opts); - List listByTemplateId(long templateId); } diff --git a/engine/schema/src/main/java/com/cloud/dc/dao/VsphereStoragePolicyDaoImpl.java b/engine/schema/src/main/java/com/cloud/dc/dao/VsphereStoragePolicyDaoImpl.java new file mode 100644 index 000000000000..0cdb6ad74220 --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/dc/dao/VsphereStoragePolicyDaoImpl.java @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.dc.dao; + +import com.cloud.dc.VsphereStoragePolicyVO; +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; + +import java.util.List; + +@Component +public class VsphereStoragePolicyDaoImpl extends GenericDaoBase implements VsphereStoragePolicyDao { + + protected static final Logger LOGGER = Logger.getLogger(VsphereStoragePolicyDaoImpl.class); + + private final SearchBuilder zoneSearch; + private final SearchBuilder policySearch; + + public VsphereStoragePolicyDaoImpl() { + super(); + + zoneSearch = createSearchBuilder(); + zoneSearch.and("zoneId", zoneSearch.entity().getZoneId(), SearchCriteria.Op.EQ); + zoneSearch.done(); + + policySearch = createSearchBuilder(); + policySearch.and("zoneId", policySearch.entity().getZoneId(), SearchCriteria.Op.EQ); + policySearch.and("policyId", policySearch.entity().getPolicyId(), SearchCriteria.Op.EQ); + policySearch.done(); + } + + @Override + public VsphereStoragePolicyVO findByPolicyId(Long zoneId, String policyId) { + SearchCriteria sc = policySearch.create(); + sc.setParameters("zoneId", zoneId); + sc.setParameters("policyId", policyId); + return findOneBy(sc); + } + + @Override + public List findByZoneId(Long zoneId) { + SearchCriteria sc = zoneSearch.create(); + sc.setParameters("zoneId", zoneId); + + return listBy(sc); + } +} diff --git a/engine/schema/src/main/java/com/cloud/deployasis/TemplateDeployAsIsDetailVO.java b/engine/schema/src/main/java/com/cloud/deployasis/TemplateDeployAsIsDetailVO.java new file mode 100644 index 000000000000..047d985d0bb6 --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/deployasis/TemplateDeployAsIsDetailVO.java @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.deployasis; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Lob; +import javax.persistence.Table; + +import org.apache.cloudstack.api.ResourceDetail; + +@Entity +@Table(name = "template_deploy_as_is_details") +public class TemplateDeployAsIsDetailVO implements ResourceDetail { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id") + private long id; + + @Column(name = "template_id") + private long resourceId; + + @Column(name = "name") + private String name; + + @Lob + @Column(name = "value", length = 65535) + private String value; + + public TemplateDeployAsIsDetailVO() { + } + + public TemplateDeployAsIsDetailVO(long templateId, String name, String value) { + this.resourceId = templateId; + this.name = name; + this.value = value; + } + + @Override + public long getId() { + return id; + } + + public long getResourceId() { + return resourceId; + } + + public String getName() { + return name; + } + + public String getValue() { + return value; + } + + @Override + public boolean isDisplay() { + return true; + } + + public void setId(long id) { + this.id = id; + } + + public void setResourceId(long resourceId) { + this.resourceId = resourceId; + } + + public void setName(String name) { + this.name = name; + } + + public void setValue(String value) { + this.value = value; + } +} diff --git a/engine/schema/src/main/java/com/cloud/deployasis/UserVmDeployAsIsDetailVO.java b/engine/schema/src/main/java/com/cloud/deployasis/UserVmDeployAsIsDetailVO.java new file mode 100644 index 000000000000..b56b4a494646 --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/deployasis/UserVmDeployAsIsDetailVO.java @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.deployasis; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Lob; +import javax.persistence.Table; + +import org.apache.cloudstack.api.ResourceDetail; + +@Entity +@Table(name = "user_vm_deploy_as_is_details") +public class UserVmDeployAsIsDetailVO implements ResourceDetail { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id") + private long id; + + @Column(name = "vm_id") + private long resourceId; + + @Column(name = "name") + private String name; + + @Lob + @Column(name = "value", length = 65535) + private String value; + + public UserVmDeployAsIsDetailVO() { + } + + public UserVmDeployAsIsDetailVO(long vmId, String name, String value) { + this.resourceId = vmId; + this.name = name; + this.value = value; + } + + @Override + public long getId() { + return id; + } + + public long getResourceId() { + return resourceId; + } + + public String getName() { + return name; + } + + public String getValue() { + return value; + } + + @Override + public boolean isDisplay() { + return true; + } + + public void setId(long id) { + this.id = id; + } + + public void setResourceId(long resourceId) { + this.resourceId = resourceId; + } + + public void setName(String name) { + this.name = name; + } + + public void setValue(String value) { + this.value = value; + } +} diff --git a/engine/schema/src/main/java/com/cloud/deployasis/dao/TemplateDeployAsIsDetailsDao.java b/engine/schema/src/main/java/com/cloud/deployasis/dao/TemplateDeployAsIsDetailsDao.java new file mode 100644 index 000000000000..ebbc1eabeb18 --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/deployasis/dao/TemplateDeployAsIsDetailsDao.java @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.deployasis.dao; + +import com.cloud.agent.api.to.deployasis.OVFPropertyTO; +import com.cloud.deployasis.TemplateDeployAsIsDetailVO; +import com.cloud.utils.db.GenericDao; +import com.cloud.agent.api.to.deployasis.OVFNetworkTO; +import org.apache.cloudstack.resourcedetail.ResourceDetailsDao; + +import java.util.List; + +public interface TemplateDeployAsIsDetailsDao extends GenericDao, ResourceDetailsDao { + + OVFPropertyTO findPropertyByTemplateAndKey(long templateId, String key); + List listDetailsByTemplateIdMatchingPrefix(long templateId, String prefix); + List listNetworkRequirementsByTemplateId(long templateId); +} diff --git a/engine/schema/src/main/java/com/cloud/deployasis/dao/TemplateDeployAsIsDetailsDaoImpl.java b/engine/schema/src/main/java/com/cloud/deployasis/dao/TemplateDeployAsIsDetailsDaoImpl.java new file mode 100644 index 000000000000..198ef6db89bc --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/deployasis/dao/TemplateDeployAsIsDetailsDaoImpl.java @@ -0,0 +1,83 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.deployasis.dao; + +import com.cloud.agent.api.to.deployasis.OVFPropertyTO; +import com.cloud.deployasis.DeployAsIsConstants; +import com.cloud.deployasis.TemplateDeployAsIsDetailVO; +import com.cloud.utils.db.SearchCriteria; +import com.google.gson.Gson; +import com.cloud.agent.api.to.deployasis.OVFNetworkTO; +import org.apache.cloudstack.resourcedetail.ResourceDetailsDaoBase; +import org.springframework.stereotype.Component; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; + +@Component +public class TemplateDeployAsIsDetailsDaoImpl extends ResourceDetailsDaoBase implements TemplateDeployAsIsDetailsDao { + + private Gson gson = new Gson(); + + public TemplateDeployAsIsDetailsDaoImpl() { + } + + @Override + public void addDetail(long resourceId, String key, String value, boolean display) { + super.addDetail(new TemplateDeployAsIsDetailVO(resourceId, key, value)); + } + + @Override + public OVFPropertyTO findPropertyByTemplateAndKey(long templateId, String key) { + SearchCriteria sc = createSearchCriteria(); + sc.addAnd("resourceId", SearchCriteria.Op.EQ, templateId); + sc.addAnd("name", SearchCriteria.Op.EQ, key.startsWith(DeployAsIsConstants.PROPERTY_PREFIX) ? key : DeployAsIsConstants.PROPERTY_PREFIX + key); + OVFPropertyTO property = null; + TemplateDeployAsIsDetailVO detail = findOneBy(sc); + if (detail != null) { + property = gson.fromJson(detail.getValue(), OVFPropertyTO.class); + } + return property; + } + + @Override + public List listDetailsByTemplateIdMatchingPrefix(long templateId, String prefix) { + SearchCriteria ssc = createSearchCriteria(); + ssc.addAnd("resourceId", SearchCriteria.Op.EQ, templateId); + ssc.addAnd("name", SearchCriteria.Op.LIKE, prefix + "%"); + + return search(ssc, null); + } + + @Override + public List listNetworkRequirementsByTemplateId(long templateId) { + List networkDetails = listDetailsByTemplateIdMatchingPrefix(templateId, DeployAsIsConstants.NETWORK_PREFIX); + List networkPrereqs = new ArrayList<>(); + for (TemplateDeployAsIsDetailVO property : networkDetails) { + OVFNetworkTO ovfPropertyTO = gson.fromJson(property.getValue(), OVFNetworkTO.class); + networkPrereqs.add(ovfPropertyTO); + } + networkPrereqs.sort(new Comparator() { + @Override + public int compare(OVFNetworkTO o1, OVFNetworkTO o2) { + return o1.getInstanceID() - o2.getInstanceID(); + } + }); + return networkPrereqs; + } +} diff --git a/engine/schema/src/main/java/com/cloud/deployasis/dao/UserVmDeployAsIsDetailsDao.java b/engine/schema/src/main/java/com/cloud/deployasis/dao/UserVmDeployAsIsDetailsDao.java new file mode 100644 index 000000000000..4366e464c1e6 --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/deployasis/dao/UserVmDeployAsIsDetailsDao.java @@ -0,0 +1,24 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.deployasis.dao; + +import com.cloud.deployasis.UserVmDeployAsIsDetailVO; +import com.cloud.utils.db.GenericDao; +import org.apache.cloudstack.resourcedetail.ResourceDetailsDao; + +public interface UserVmDeployAsIsDetailsDao extends GenericDao, ResourceDetailsDao { +} diff --git a/engine/schema/src/main/java/com/cloud/deployasis/dao/UserVmDeployAsIsDetailsDaoImpl.java b/engine/schema/src/main/java/com/cloud/deployasis/dao/UserVmDeployAsIsDetailsDaoImpl.java new file mode 100644 index 000000000000..8dc5f4a1204e --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/deployasis/dao/UserVmDeployAsIsDetailsDaoImpl.java @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.deployasis.dao; + +import com.cloud.deployasis.UserVmDeployAsIsDetailVO; +import org.apache.cloudstack.resourcedetail.ResourceDetailsDaoBase; +import org.springframework.stereotype.Component; + +@Component +public class UserVmDeployAsIsDetailsDaoImpl extends ResourceDetailsDaoBase implements UserVmDeployAsIsDetailsDao { + + @Override + public void addDetail(long resourceId, String key, String value, boolean display) { + super.addDetail(new UserVmDeployAsIsDetailVO(resourceId, key, value)); + } +} diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java b/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java index 88e8eaca95e9..3d76c8b38c1d 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java @@ -39,6 +39,8 @@ public interface HostDao extends GenericDao, StateDao, StateDao listByHostCapability(Host.Type type, Long clusterId, Long podId, long dcId, String hostCapabilty); List listByClusterAndHypervisorType(long clusterId, HypervisorType hypervisorType); + + HostVO findByName(String name); } diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java b/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java index d28357d39e7c..75304c1b2741 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java @@ -107,6 +107,7 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao protected SearchBuilder UnmanagedApplianceSearch; protected SearchBuilder MaintenanceCountSearch; protected SearchBuilder HostTypeCountSearch; + protected SearchBuilder HostTypeZoneCountSearch; protected SearchBuilder ClusterStatusSearch; protected SearchBuilder TypeNameZoneSearch; protected SearchBuilder AvailHypevisorInZone; @@ -167,6 +168,12 @@ public void init() { HostTypeCountSearch.and("removed", HostTypeCountSearch.entity().getRemoved(), SearchCriteria.Op.NULL); HostTypeCountSearch.done(); + HostTypeZoneCountSearch = createSearchBuilder(); + HostTypeZoneCountSearch.and("type", HostTypeZoneCountSearch.entity().getType(), SearchCriteria.Op.EQ); + HostTypeZoneCountSearch.and("dc", HostTypeZoneCountSearch.entity().getDataCenterId(), SearchCriteria.Op.EQ); + HostTypeZoneCountSearch.and("removed", HostTypeZoneCountSearch.entity().getRemoved(), SearchCriteria.Op.NULL); + HostTypeZoneCountSearch.done(); + TypePodDcStatusSearch = createSearchBuilder(); HostVO entity = TypePodDcStatusSearch.entity(); TypePodDcStatusSearch.and("type", entity.getType(), SearchCriteria.Op.EQ); @@ -447,6 +454,14 @@ public Integer countAllByType(final Host.Type type) { return getCount(sc); } + @Override + public Integer countAllByTypeInZone(long zoneId, Type type) { + SearchCriteria sc = HostTypeCountSearch.create(); + sc.setParameters("type", type); + sc.setParameters("dc", zoneId); + return getCount(sc); + } + @Override public List listByDataCenterId(long id) { SearchCriteria sc = DcSearch.create(); @@ -1261,6 +1276,13 @@ public List listByClusterAndHypervisorType(long clusterId, HypervisorTyp return listBy(sc); } + @Override + public HostVO findByName(String name) { + SearchCriteria sc = NameSearch.create(); + sc.setParameters("name", name); + return findOneBy(sc); + } + private ResultSet executeSqlGetResultsetForMethodFindHostInZoneToExecuteCommand(HypervisorType hypervisorType, long zoneId, TransactionLegacy tx, String sql) throws SQLException { PreparedStatement pstmt = tx.prepareAutoCloseStatement(sql); pstmt.setString(1, Objects.toString(hypervisorType)); diff --git a/engine/schema/src/main/java/com/cloud/secstorage/CommandExecLogDao.java b/engine/schema/src/main/java/com/cloud/secstorage/CommandExecLogDao.java index fb57563131e6..98fc8c8687b8 100644 --- a/engine/schema/src/main/java/com/cloud/secstorage/CommandExecLogDao.java +++ b/engine/schema/src/main/java/com/cloud/secstorage/CommandExecLogDao.java @@ -22,4 +22,5 @@ public interface CommandExecLogDao extends GenericDao { public void expungeExpiredRecords(Date cutTime); + public Integer getCopyCmdCountForSSVM(Long id); } diff --git a/engine/schema/src/main/java/com/cloud/secstorage/CommandExecLogDaoImpl.java b/engine/schema/src/main/java/com/cloud/secstorage/CommandExecLogDaoImpl.java index ac438b0d1173..f89a1bbf4ccb 100644 --- a/engine/schema/src/main/java/com/cloud/secstorage/CommandExecLogDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/secstorage/CommandExecLogDaoImpl.java @@ -17,7 +17,7 @@ package com.cloud.secstorage; import java.util.Date; - +import java.util.List; import org.springframework.stereotype.Component; @@ -30,11 +30,16 @@ public class CommandExecLogDaoImpl extends GenericDaoBase implements CommandExecLogDao { protected final SearchBuilder ExpungeSearch; + protected final SearchBuilder CommandSearch; public CommandExecLogDaoImpl() { ExpungeSearch = createSearchBuilder(); ExpungeSearch.and("created", ExpungeSearch.entity().getCreated(), Op.LT); ExpungeSearch.done(); + + CommandSearch = createSearchBuilder(); + CommandSearch.and("host_id", CommandSearch.entity().getHostId(), Op.EQ); + CommandSearch.and("command_name", CommandSearch.entity().getCommandName(), Op.EQ); } @Override @@ -43,4 +48,13 @@ public void expungeExpiredRecords(Date cutTime) { sc.setParameters("created", cutTime); expunge(sc); } + + @Override + public Integer getCopyCmdCountForSSVM(Long id) { + SearchCriteria sc = CommandSearch.create(); + sc.setParameters("host_id", id); + sc.setParameters("command_name", "CopyCommand"); + List copyCmds = customSearch(sc, null); + return copyCmds.size(); + } } diff --git a/engine/schema/src/main/java/com/cloud/service/dao/ServiceOfferingDetailsDao.java b/engine/schema/src/main/java/com/cloud/service/dao/ServiceOfferingDetailsDao.java index f905ab98dd53..10ba8d574a17 100644 --- a/engine/schema/src/main/java/com/cloud/service/dao/ServiceOfferingDetailsDao.java +++ b/engine/schema/src/main/java/com/cloud/service/dao/ServiceOfferingDetailsDao.java @@ -26,4 +26,5 @@ public interface ServiceOfferingDetailsDao extends GenericDao, ResourceDetailsDao { List findDomainIds(final long resourceId); List findZoneIds(final long resourceId); + String getDetail(Long diskOfferingId, String key); } \ No newline at end of file diff --git a/engine/schema/src/main/java/com/cloud/service/dao/ServiceOfferingDetailsDaoImpl.java b/engine/schema/src/main/java/com/cloud/service/dao/ServiceOfferingDetailsDaoImpl.java index 768402674615..0080174874d9 100644 --- a/engine/schema/src/main/java/com/cloud/service/dao/ServiceOfferingDetailsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/service/dao/ServiceOfferingDetailsDaoImpl.java @@ -58,4 +58,13 @@ public List findZoneIds(long resourceId) { return zoneIds; } + @Override + public String getDetail(Long serviceOfferingId, String key) { + String detailValue = null; + ServiceOfferingDetailsVO serviceOfferingDetail = findDetail(serviceOfferingId, key); + if (serviceOfferingDetail != null) { + detailValue = serviceOfferingDetail.getValue(); + } + return detailValue; + } } diff --git a/engine/schema/src/main/java/com/cloud/storage/TemplateOVFPropertyVO.java b/engine/schema/src/main/java/com/cloud/storage/TemplateOVFPropertyVO.java deleted file mode 100644 index 425b1f22e453..000000000000 --- a/engine/schema/src/main/java/com/cloud/storage/TemplateOVFPropertyVO.java +++ /dev/null @@ -1,167 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloud.storage; - -import com.cloud.agent.api.storage.OVFProperty; - -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.GeneratedValue; -import javax.persistence.GenerationType; -import javax.persistence.Id; -import javax.persistence.Table; - -@Entity -@Table(name = "template_ovf_properties") -public class TemplateOVFPropertyVO implements OVFProperty { - - @Id - @GeneratedValue(strategy = GenerationType.IDENTITY) - @Column(name = "id") - private long id; - - @Column(name = "template_id") - private Long templateId; - - @Column(name = "key") - private String key; - - @Column(name = "type") - private String type; - - @Column(name = "value") - private String value; - - @Column(name = "qualifiers") - private String qualifiers; - - @Column(name = "password") - private Boolean password; - - @Column(name = "user_configurable") - private Boolean userConfigurable; - - @Column(name = "label") - private String label; - - @Column(name = "description") - private String description; - - public TemplateOVFPropertyVO() { - } - - public TemplateOVFPropertyVO(Long templateId, String key, String type, String value, String qualifiers, - Boolean userConfigurable, String label, String description, Boolean password) { - this.templateId = templateId; - this.key = key; - this.type = type; - this.value = value; - this.qualifiers = qualifiers; - this.userConfigurable = userConfigurable; - this.label = label; - this.description = description; - this.password = password; - } - - public long getId() { - return id; - } - - public void setId(long id) { - this.id = id; - } - - public Long getTemplateId() { - return templateId; - } - - public void setTemplateId(Long templateId) { - this.templateId = templateId; - } - - public String getKey() { - return key; - } - - public void setKey(String key) { - this.key = key; - } - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } - - public String getValue() { - return value; - } - - public void setValue(String value) { - this.value = value; - } - - public String getQualifiers() { - return qualifiers; - } - - public void setQualifiers(String qualifiers) { - this.qualifiers = qualifiers; - } - - @Override - public Boolean isUserConfigurable() { - return userConfigurable; - } - - public void setUserConfigurable(Boolean userConfigurable) { - this.userConfigurable = userConfigurable; - } - - public String getLabel() { - return label; - } - - public void setLabel(String label) { - this.label = label; - } - - public String getDescription() { - return description; - } - - public void setDescription(String description) { - this.description = description; - } - - public Boolean isPassword() { - return password; - } - - public void setPassword(Boolean password) { - this.password = password; - } - - @Override - public String toString() { - return String.format("PROP - templateId=%s> key=%s value=%s type=%s qual=%s conf=%s label=%s desc=%s password=%s", - templateId, key, value, type, qualifiers, userConfigurable, label, description, password); - } -} diff --git a/engine/schema/src/main/java/com/cloud/storage/VMTemplateStoragePoolVO.java b/engine/schema/src/main/java/com/cloud/storage/VMTemplateStoragePoolVO.java index 6dfe6eb3e872..69c9c85ab5aa 100644 --- a/engine/schema/src/main/java/com/cloud/storage/VMTemplateStoragePoolVO.java +++ b/engine/schema/src/main/java/com/cloud/storage/VMTemplateStoragePoolVO.java @@ -95,6 +95,9 @@ public class VMTemplateStoragePoolVO implements VMTemplateStorageResourceAssoc, @Enumerated(EnumType.STRING) ObjectInDataStoreStateMachine.State state; + @Column(name = "deployment_option") + private String deploymentOption; + @Override public String getInstallPath() { return installPath; @@ -168,17 +171,18 @@ public Status getDownloadState() { return downloadState; } - public VMTemplateStoragePoolVO(long poolId, long templateId) { + public VMTemplateStoragePoolVO(long poolId, long templateId, String configuration) { super(); this.poolId = poolId; this.templateId = templateId; this.downloadState = Status.NOT_DOWNLOADED; this.state = ObjectInDataStoreStateMachine.State.Allocated; this.markedForGC = false; + this.deploymentOption = configuration; } public VMTemplateStoragePoolVO(long poolId, long templateId, Date lastUpdated, int downloadPercent, Status downloadState, String localDownloadPath, - String errorString, String jobId, String installPath, long templateSize) { + String errorString, String jobId, String installPath, long templateSize, String configuration) { super(); this.poolId = poolId; this.templateId = templateId; @@ -190,6 +194,7 @@ public VMTemplateStoragePoolVO(long poolId, long templateId, Date lastUpdated, i this.jobId = jobId; this.installPath = installPath; this.templateSize = templateSize; + this.deploymentOption = configuration; } protected VMTemplateStoragePoolVO() { @@ -300,4 +305,11 @@ public State getObjectInStoreState() { return this.state; } + public String getDeploymentOption() { + return deploymentOption; + } + + public void setDeploymentOption(String deploymentOption) { + this.deploymentOption = deploymentOption; + } } diff --git a/engine/schema/src/main/java/com/cloud/storage/VMTemplateVO.java b/engine/schema/src/main/java/com/cloud/storage/VMTemplateVO.java index af04099f9a2a..61df40e50d88 100644 --- a/engine/schema/src/main/java/com/cloud/storage/VMTemplateVO.java +++ b/engine/schema/src/main/java/com/cloud/storage/VMTemplateVO.java @@ -152,6 +152,9 @@ public class VMTemplateVO implements VirtualMachineTemplate { @Column(name = "parent_template_id") private Long parentTemplateId; + @Column(name = "deploy_as_is") + private boolean deployAsIs; + @Override public String getUniqueName() { return uniqueName; @@ -192,9 +195,9 @@ private VMTemplateVO(long id, String name, ImageFormat format, boolean isPublic, uuid = UUID.randomUUID().toString(); } - public VMTemplateVO(long id, String name, ImageFormat format, boolean isPublic, boolean featured, boolean isExtractable, TemplateType type, String url, - boolean requiresHvm, int bits, long accountId, String cksum, String displayText, boolean enablePassword, long guestOSId, boolean bootable, - HypervisorType hyperType, String templateTag, Map details, boolean sshKeyEnabled, boolean isDynamicallyScalable, boolean directDownload) { + public VMTemplateVO(long id, String name, ImageFormat format, boolean isPublic, boolean featured, boolean isExtractable, TemplateType type, String url, boolean requiresHvm, int bits, long accountId, String cksum, String displayText, boolean enablePassword, long guestOSId, boolean bootable, + HypervisorType hyperType, String templateTag, Map details, boolean sshKeyEnabled, boolean isDynamicallyScalable, boolean directDownload, + boolean deployAsIs) { this(id, name, format, @@ -219,6 +222,7 @@ public VMTemplateVO(long id, String name, ImageFormat format, boolean isPublic, dynamicallyScalable = isDynamicallyScalable; state = State.Active; this.directDownload = directDownload; + this.deployAsIs = deployAsIs; } public static VMTemplateVO createPreHostIso(Long id, String uniqueName, String name, ImageFormat format, boolean isPublic, boolean featured, TemplateType type, @@ -637,4 +641,11 @@ public void setParentTemplateId(Long parentTemplateId) { this.parentTemplateId = parentTemplateId; } + @Override public boolean isDeployAsIs() { + return deployAsIs; + } + + public void setDeployAsIs(boolean deployAsIs) { + this.deployAsIs = deployAsIs; + } } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolDetailsDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolDetailsDaoImpl.java index 1e74d25632b7..0c39a8c581aa 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolDetailsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolDetailsDaoImpl.java @@ -21,11 +21,19 @@ import org.apache.cloudstack.framework.config.ConfigKey.Scope; import org.apache.cloudstack.framework.config.ScopedConfigStorage; import org.apache.cloudstack.resourcedetail.ResourceDetailsDaoBase; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailVO; import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; + +import javax.inject.Inject; +import java.util.List; public class StoragePoolDetailsDaoImpl extends ResourceDetailsDaoBase implements StoragePoolDetailsDao, ScopedConfigStorage { + @Inject + PrimaryDataStoreDao _storagePoolDao; + public StoragePoolDetailsDaoImpl() { } @@ -42,6 +50,10 @@ public String getConfigValue(long id, ConfigKey key) { @Override public void addDetail(long resourceId, String key, String value, boolean display) { + List ChildPools = _storagePoolDao.listChildStoragePoolsInDatastoreCluster(resourceId); + for(StoragePoolVO childPool : ChildPools) { + super.addDetail(new StoragePoolDetailVO(childPool.getId(), key, value, display)); + } super.addDetail(new StoragePoolDetailVO(resourceId, key, value, display)); } } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/TemplateOVFPropertiesDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/TemplateOVFPropertiesDaoImpl.java deleted file mode 100644 index cf6a280b0348..000000000000 --- a/engine/schema/src/main/java/com/cloud/storage/dao/TemplateOVFPropertiesDaoImpl.java +++ /dev/null @@ -1,78 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloud.storage.dao; - -import com.cloud.storage.TemplateOVFPropertyVO; -import com.cloud.utils.db.GenericDaoBase; -import com.cloud.utils.db.SearchBuilder; -import com.cloud.utils.db.SearchCriteria; -import com.cloud.utils.db.TransactionLegacy; -import org.apache.commons.collections.CollectionUtils; -import org.apache.log4j.Logger; -import org.springframework.stereotype.Component; - -import java.util.List; - -@Component -public class TemplateOVFPropertiesDaoImpl extends GenericDaoBase implements TemplateOVFPropertiesDao { - - private final static Logger s_logger = Logger.getLogger(TemplateOVFPropertiesDaoImpl.class); - - SearchBuilder OptionsSearchBuilder; - - public TemplateOVFPropertiesDaoImpl() { - super(); - OptionsSearchBuilder = createSearchBuilder(); - OptionsSearchBuilder.and("templateid", OptionsSearchBuilder.entity().getTemplateId(), SearchCriteria.Op.EQ); - OptionsSearchBuilder.and("key", OptionsSearchBuilder.entity().getKey(), SearchCriteria.Op.EQ); - OptionsSearchBuilder.done(); - } - - @Override - public boolean existsOption(long templateId, String key) { - return findByTemplateAndKey(templateId, key) != null; - } - - @Override - public TemplateOVFPropertyVO findByTemplateAndKey(long templateId, String key) { - SearchCriteria sc = OptionsSearchBuilder.create(); - sc.setParameters("templateid", templateId); - sc.setParameters("key", key); - return findOneBy(sc); - } - - @Override - public void saveOptions(List opts) { - if (CollectionUtils.isEmpty(opts)) { - return; - } - TransactionLegacy txn = TransactionLegacy.currentTxn(); - txn.start(); - for (TemplateOVFPropertyVO opt : opts) { - persist(opt); - } - txn.commit(); - } - - @Override - public List listByTemplateId(long templateId) { - SearchCriteria sc = OptionsSearchBuilder.create(); - sc.setParameters("templateid", templateId); - return listBy(sc); - } -} diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDetailsDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDetailsDaoImpl.java index 3e7072f6bf01..60e583be3e63 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDetailsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDetailsDaoImpl.java @@ -30,4 +30,4 @@ public class VMTemplateDetailsDaoImpl extends ResourceDetailsDaoBase, StateDao { - public List listByPoolId(long id); + List listByPoolId(long id); - public List listByTemplateId(long templateId); + List listByTemplateId(long templateId); - public VMTemplateStoragePoolVO findByPoolTemplate(long poolId, long templateId); + VMTemplateStoragePoolVO findByPoolTemplate(long poolId, long templateId, String configuration); - public List listByPoolIdAndState(long poolId, ObjectInDataStoreStateMachine.State state); + List listByPoolIdAndState(long poolId, ObjectInDataStoreStateMachine.State state); - public List listByTemplateStatus(long templateId, VMTemplateStoragePoolVO.Status downloadState); + List listByTemplateStatus(long templateId, VMTemplateStoragePoolVO.Status downloadState); - public List listByTemplateStatus(long templateId, VMTemplateStoragePoolVO.Status downloadState, long poolId); + List listByTemplateStatus(long templateId, VMTemplateStoragePoolVO.Status downloadState, long poolId); - public List listByTemplateStatus(long templateId, long datacenterId, VMTemplateStoragePoolVO.Status downloadState); + List listByTemplateStatus(long templateId, long datacenterId, VMTemplateStoragePoolVO.Status downloadState); - public List listByTemplateStatus(long templateId, long datacenterId, long podId, VMTemplateStoragePoolVO.Status downloadState); + List listByTemplateStatus(long templateId, long datacenterId, long podId, VMTemplateStoragePoolVO.Status downloadState); - public List listByTemplateStates(long templateId, VMTemplateStoragePoolVO.Status... states); + List listByTemplateStates(long templateId, VMTemplateStoragePoolVO.Status... states); boolean templateAvailable(long templateId, long poolId); - public VMTemplateStoragePoolVO findByHostTemplate(Long hostId, Long templateId); + VMTemplateStoragePoolVO findByHostTemplate(Long hostId, Long templateId, String configuration); VMTemplateStoragePoolVO findByPoolPath(Long poolId, String path); diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplatePoolDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplatePoolDaoImpl.java index 32874701128d..998df5e7987d 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplatePoolDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplatePoolDaoImpl.java @@ -28,6 +28,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.DataObjectInStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; +import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -73,7 +74,7 @@ public class VMTemplatePoolDaoImpl extends GenericDaoBase listByTemplateId(long templateId) { } @Override - public VMTemplateStoragePoolVO findByPoolTemplate(long poolId, long templateId) { + public VMTemplateStoragePoolVO findByPoolTemplate(long poolId, long templateId, String configuration) { SearchCriteria sc = PoolTemplateSearch.create(); sc.setParameters("pool_id", poolId); sc.setParameters("template_id", templateId); + if (StringUtils.isNotBlank(configuration)) { + sc.setParameters("configuration", configuration); + } return findOneIncludingRemovedBy(sc); } @@ -219,13 +224,17 @@ public List listByTemplateStatus(long templateId, long } - public List listByHostTemplate(long hostId, long templateId) { + public List listByHostTemplate(long hostId, long templateId, String configuration) { TransactionLegacy txn = TransactionLegacy.currentTxn(); List result = new ArrayList(); String sql = HOST_TEMPLATE_SEARCH; + sql += StringUtils.isBlank(configuration) ? "IS NULL" : "= ?"; try(PreparedStatement pstmt = txn.prepareStatement(sql);) { pstmt.setLong(1, hostId); pstmt.setLong(2, templateId); + if (StringUtils.isNotBlank(configuration)) { + pstmt.setString(3, configuration); + } try(ResultSet rs = pstmt.executeQuery();) { while (rs.next()) { // result.add(toEntityBean(rs, false)); TODO: this is buggy in @@ -245,7 +254,7 @@ public List listByHostTemplate(long hostId, long templa @Override public boolean templateAvailable(long templateId, long hostId) { - VMTemplateStorageResourceAssoc tmpltPool = findByPoolTemplate(hostId, templateId); + VMTemplateStorageResourceAssoc tmpltPool = findByPoolTemplate(hostId, templateId, null); if (tmpltPool == null) return false; @@ -262,9 +271,9 @@ public List listByTemplateStates(long templateId, VMTem } @Override - public VMTemplateStoragePoolVO findByHostTemplate(Long hostId, Long templateId) { - List result = listByHostTemplate(hostId, templateId); - return (result.size() == 0) ? null : result.get(1); + public VMTemplateStoragePoolVO findByHostTemplate(Long hostId, Long templateId, String configuration) { + List result = listByHostTemplate(hostId, templateId, configuration); + return (result.size() == 0) ? null : result.get(0); } @Override diff --git a/engine/schema/src/main/java/org/apache/cloudstack/resourcedetail/dao/DiskOfferingDetailsDao.java b/engine/schema/src/main/java/org/apache/cloudstack/resourcedetail/dao/DiskOfferingDetailsDao.java index e201ae27fdc8..815f1693ee54 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/resourcedetail/dao/DiskOfferingDetailsDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/resourcedetail/dao/DiskOfferingDetailsDao.java @@ -26,4 +26,5 @@ public interface DiskOfferingDetailsDao extends GenericDao, ResourceDetailsDao { List findDomainIds(final long resourceId); List findZoneIds(final long resourceId); + String getDetail(Long diskOfferingId, String key); } \ No newline at end of file diff --git a/engine/schema/src/main/java/org/apache/cloudstack/resourcedetail/dao/DiskOfferingDetailsDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/resourcedetail/dao/DiskOfferingDetailsDaoImpl.java index da0ec5bc580d..5408f2d7f036 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/resourcedetail/dao/DiskOfferingDetailsDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/resourcedetail/dao/DiskOfferingDetailsDaoImpl.java @@ -56,4 +56,14 @@ public List findZoneIds(long resourceId) { } return zoneIds; } + + @Override + public String getDetail(Long diskOfferingId, String key) { + String detailValue = null; + DiskOfferingDetailVO diskOfferingDetail = findDetail(diskOfferingId, key); + if (diskOfferingDetail != null) { + detailValue = diskOfferingDetail.getValue(); + } + return detailValue; + } } \ No newline at end of file diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDao.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDao.java index 1861b21a38ad..84cba70e8617 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDao.java @@ -29,7 +29,7 @@ public interface ImageStoreDao extends GenericDao { List findByProvider(String provider); - List findByScope(ZoneScope scope); + List findByZone(ZoneScope scope, Boolean readonly); List findRegionImageStores(); diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDaoImpl.java index 38124ea49e0e..6ecac5ed8094 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDaoImpl.java @@ -77,9 +77,12 @@ public List findByProvider(String provider) { } @Override - public List findByScope(ZoneScope scope) { + public List findByZone(ZoneScope scope, Boolean readonly) { SearchCriteria sc = createSearchCriteria(); sc.addAnd("role", SearchCriteria.Op.EQ, DataStoreRole.Image); + if (readonly != null) { + sc.addAnd("readonly", SearchCriteria.Op.EQ, readonly); + } if (scope.getScopeId() != null) { SearchCriteria scc = createSearchCriteria(); scc.addOr("scope", SearchCriteria.Op.EQ, ScopeType.REGION); diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreVO.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreVO.java index 2c706774a4d8..d24582714868 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreVO.java @@ -74,6 +74,9 @@ public class ImageStoreVO implements ImageStore { @Enumerated(value = EnumType.STRING) private DataStoreRole role; + @Column(name = "readonly") + private boolean readonly = false; + @Column(name = "parent") private String parent; @@ -165,6 +168,14 @@ public Date getCreated() { return created; } + public void setReadonly(boolean readonly) { + this.readonly = readonly; + } + + public boolean isReadonly() { + return readonly; + } + public void setCreated(Date created) { this.created = created; } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java index 2398e91c90c7..5712411a48e6 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java @@ -123,4 +123,8 @@ public interface PrimaryDataStoreDao extends GenericDao { List listLocalStoragePoolByPath(long datacenterId, String path); void deletePoolTags(long poolId); + + List listChildStoragePoolsInDatastoreCluster(long poolId); + + Integer countAll(); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java index fee9dc28bd02..1d1e0a0198db 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java @@ -552,4 +552,19 @@ public List findZoneWideStoragePoolsByHypervisor(long dataCenterI public void deletePoolTags(long poolId) { _tagsDao.deleteTags(poolId); } + + @Override + public List listChildStoragePoolsInDatastoreCluster(long poolId) { + QueryBuilder sc = QueryBuilder.create(StoragePoolVO.class); + sc.and(sc.entity().getParent(), Op.EQ, poolId); + return sc.list(); + } + + @Override + public Integer countAll() { + SearchCriteria sc = createSearchCriteria(); + sc.addAnd("parent", SearchCriteria.Op.EQ, 0); + sc.addAnd("removed", SearchCriteria.Op.NULL); + return getCount(sc); + } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java index 91ea07c5860b..3263cbb5b1a0 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java @@ -16,6 +16,7 @@ // under the License. package org.apache.cloudstack.storage.datastore.db; +import java.util.Date; import java.util.List; import org.apache.cloudstack.engine.subsystem.api.storage.DataObjectInStore; @@ -42,6 +43,8 @@ public interface SnapshotDataStoreDao extends GenericDao listDestroyed(long storeId); List findBySnapshotId(long snapshotId); @@ -72,5 +75,7 @@ public interface SnapshotDataStoreDao extends GenericDao listByState(ObjectInDataStoreStateMachine.State... states); + List findSnapshots(Long storeId, Date start, Date end); + SnapshotDataStoreVO findDestroyedReferenceBySnapshot(long snapshotId, DataStoreRole role); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/StoragePoolVO.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/StoragePoolVO.java index 24fcaa03f56c..c7eb72b469e1 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/StoragePoolVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/StoragePoolVO.java @@ -119,6 +119,9 @@ public class StoragePoolVO implements StoragePool { @Enumerated(value = EnumType.STRING) private HypervisorType hypervisor; + @Column(name = "parent") + private Long parent = 0L; + @Override public long getId() { return id; @@ -373,6 +376,14 @@ public boolean isLocal() { return !isShared(); } + public Long getParent() { + return parent; + } + + public void setParent(Long parent) { + this.parent = parent; + } + @Override public boolean isInMaintenance() { return status == StoragePoolStatus.PrepareForMaintenance || status == StoragePoolStatus.Maintenance || status == StoragePoolStatus.ErrorInMaintenance || diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreDao.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreDao.java index a6e609e7d870..fc695f476779 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreDao.java @@ -66,6 +66,8 @@ public interface TemplateDataStoreDao extends GenericDao listByTemplate(long templateId); + List listByTemplateNotBypassed(long templateId); + TemplateDataStoreVO findByTemplateZoneReady(long templateId, Long zoneId); void duplicateCacheRecordsOnRegionStore(long storeId); diff --git a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml index 9ab5774b301b..67b40101d559 100644 --- a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml +++ b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml @@ -293,6 +293,8 @@ - + + + diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41310to41400.sql b/engine/schema/src/main/resources/META-INF/db/schema-41310to41400.sql index baa7bcf96178..a1a2e742b0c1 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41310to41400.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41310to41400.sql @@ -53,6 +53,33 @@ ALTER TABLE `cloud`.`vm_instance` ADD COLUMN `backup_offering_id` bigint unsigne ALTER TABLE `cloud`.`vm_instance` ADD COLUMN `backup_external_id` varchar(255) DEFAULT NULL COMMENT 'ID of external backup job or container if any'; ALTER TABLE `cloud`.`vm_instance` ADD COLUMN `backup_volumes` text DEFAULT NULL COMMENT 'details of backedup volumes'; +ALTER TABLE `cloud`.`image_store` ADD COLUMN `readonly` boolean DEFAULT false COMMENT 'defines status of image store'; + +ALTER VIEW `cloud`.`image_store_view` AS + select + image_store.id, + image_store.uuid, + image_store.name, + image_store.image_provider_name, + image_store.protocol, + image_store.url, + image_store.scope, + image_store.role, + image_store.readonly, + image_store.removed, + data_center.id data_center_id, + data_center.uuid data_center_uuid, + data_center.name data_center_name, + image_store_details.name detail_name, + image_store_details.value detail_value + from + `cloud`.`image_store` + left join + `cloud`.`data_center` ON image_store.data_center_id = data_center.id + left join + `cloud`.`image_store_details` ON image_store_details.store_id = image_store.id; + + CREATE TABLE IF NOT EXISTS `cloud`.`backups` ( `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT, `uuid` varchar(40) NOT NULL UNIQUE, diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql b/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql index ab715f303f85..cc7b20ba0236 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql @@ -194,3 +194,299 @@ INSERT IGNORE INTO `cloud`.`hypervisor_capabilities`(uuid, hypervisor_type, hype -- Copy XenServer 8.0 hypervisor guest OS mappings to XenServer8.1 INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) SELECT UUID(),'Xenserver', '8.1.0', guest_os_name, guest_os_id, utc_timestamp(), 0 FROM `cloud`.`guest_os_hypervisor` WHERE hypervisor_type='Xenserver' AND hypervisor_version='8.0.0'; + +CREATE TABLE IF NOT EXISTS `cloud`.`vsphere_storage_policy` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT, + `uuid` varchar(255) UNIQUE, + `zone_id` bigint(20) unsigned NOT NULL COMMENT 'id of the zone', + `policy_id` varchar(255) NOT NULL COMMENT 'the identifier of the Storage Policy in vSphere DataCenter', + `name` varchar(255) NOT NULL COMMENT 'name of the storage policy', + `description` text COMMENT 'description of the storage policy', + `update_time` datetime COMMENT 'last updated when policy imported', + `removed` datetime COMMENT 'date removed', + PRIMARY KEY (`id`), + KEY `fk_vsphere_storage_policy__zone_id` (`zone_id`), + UNIQUE KEY (`zone_id`, `policy_id`), + CONSTRAINT `fk_vsphere_storage_policy__zone_id` FOREIGN KEY (`zone_id`) REFERENCES `data_center` (`id`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +ALTER TABLE `cloud`.`storage_pool` ADD COLUMN `parent` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'ID of the Datastore cluster (storage pool) if this is a child in that Datastore cluster'; + +-- Added parent column to support datastore clusters in vmware vsphere +DROP VIEW IF EXISTS `cloud`.`storage_pool_view`; +CREATE VIEW `cloud`.`storage_pool_view` AS + SELECT + `storage_pool`.`id` AS `id`, + `storage_pool`.`uuid` AS `uuid`, + `storage_pool`.`name` AS `name`, + `storage_pool`.`status` AS `status`, + `storage_pool`.`path` AS `path`, + `storage_pool`.`pool_type` AS `pool_type`, + `storage_pool`.`host_address` AS `host_address`, + `storage_pool`.`created` AS `created`, + `storage_pool`.`removed` AS `removed`, + `storage_pool`.`capacity_bytes` AS `capacity_bytes`, + `storage_pool`.`capacity_iops` AS `capacity_iops`, + `storage_pool`.`scope` AS `scope`, + `storage_pool`.`hypervisor` AS `hypervisor`, + `storage_pool`.`storage_provider_name` AS `storage_provider_name`, + `storage_pool`.`parent` AS `parent`, + `cluster`.`id` AS `cluster_id`, + `cluster`.`uuid` AS `cluster_uuid`, + `cluster`.`name` AS `cluster_name`, + `cluster`.`cluster_type` AS `cluster_type`, + `data_center`.`id` AS `data_center_id`, + `data_center`.`uuid` AS `data_center_uuid`, + `data_center`.`name` AS `data_center_name`, + `data_center`.`networktype` AS `data_center_type`, + `host_pod_ref`.`id` AS `pod_id`, + `host_pod_ref`.`uuid` AS `pod_uuid`, + `host_pod_ref`.`name` AS `pod_name`, + `storage_pool_tags`.`tag` AS `tag`, + `op_host_capacity`.`used_capacity` AS `disk_used_capacity`, + `op_host_capacity`.`reserved_capacity` AS `disk_reserved_capacity`, + `async_job`.`id` AS `job_id`, + `async_job`.`uuid` AS `job_uuid`, + `async_job`.`job_status` AS `job_status`, + `async_job`.`account_id` AS `job_account_id` + FROM + ((((((`storage_pool` + LEFT JOIN `cluster` ON ((`storage_pool`.`cluster_id` = `cluster`.`id`))) + LEFT JOIN `data_center` ON ((`storage_pool`.`data_center_id` = `data_center`.`id`))) + LEFT JOIN `host_pod_ref` ON ((`storage_pool`.`pod_id` = `host_pod_ref`.`id`))) + LEFT JOIN `storage_pool_tags` ON (((`storage_pool_tags`.`pool_id` = `storage_pool`.`id`)))) + LEFT JOIN `op_host_capacity` ON (((`storage_pool`.`id` = `op_host_capacity`.`host_id`) + AND (`op_host_capacity`.`capacity_type` IN (3 , 9))))) + LEFT JOIN `async_job` ON (((`async_job`.`instance_id` = `storage_pool`.`id`) + AND (`async_job`.`instance_type` = 'StoragePool') + AND (`async_job`.`job_status` = 0)))); +-- Add passthrough instruction for appliance deployments +ALTER TABLE `cloud`.`vm_template` ADD COLUMN `deploy_as_is` tinyint(1) NOT NULL DEFAULT '0' COMMENT 'True if the template should be deployed with disks and networks as defined by OVF'; + +-- Extend the template details value field +ALTER TABLE `cloud`.`vm_template_details` MODIFY COLUMN `value` TEXT CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL; + +-- Changes to template_view for both deploying multidisk OVA/vApp as is +DROP VIEW IF EXISTS `cloud`.`template_view`; +CREATE VIEW `cloud`.`template_view` AS + SELECT + `vm_template`.`id` AS `id`, + `vm_template`.`uuid` AS `uuid`, + `vm_template`.`unique_name` AS `unique_name`, + `vm_template`.`name` AS `name`, + `vm_template`.`public` AS `public`, + `vm_template`.`featured` AS `featured`, + `vm_template`.`type` AS `type`, + `vm_template`.`hvm` AS `hvm`, + `vm_template`.`bits` AS `bits`, + `vm_template`.`url` AS `url`, + `vm_template`.`format` AS `format`, + `vm_template`.`created` AS `created`, + `vm_template`.`checksum` AS `checksum`, + `vm_template`.`display_text` AS `display_text`, + `vm_template`.`enable_password` AS `enable_password`, + `vm_template`.`dynamically_scalable` AS `dynamically_scalable`, + `vm_template`.`state` AS `template_state`, + `vm_template`.`guest_os_id` AS `guest_os_id`, + `guest_os`.`uuid` AS `guest_os_uuid`, + `guest_os`.`display_name` AS `guest_os_name`, + `vm_template`.`bootable` AS `bootable`, + `vm_template`.`prepopulate` AS `prepopulate`, + `vm_template`.`cross_zones` AS `cross_zones`, + `vm_template`.`hypervisor_type` AS `hypervisor_type`, + `vm_template`.`extractable` AS `extractable`, + `vm_template`.`template_tag` AS `template_tag`, + `vm_template`.`sort_key` AS `sort_key`, + `vm_template`.`removed` AS `removed`, + `vm_template`.`enable_sshkey` AS `enable_sshkey`, + `parent_template`.`id` AS `parent_template_id`, + `parent_template`.`uuid` AS `parent_template_uuid`, + `source_template`.`id` AS `source_template_id`, + `source_template`.`uuid` AS `source_template_uuid`, + `account`.`id` AS `account_id`, + `account`.`uuid` AS `account_uuid`, + `account`.`account_name` AS `account_name`, + `account`.`type` AS `account_type`, + `domain`.`id` AS `domain_id`, + `domain`.`uuid` AS `domain_uuid`, + `domain`.`name` AS `domain_name`, + `domain`.`path` AS `domain_path`, + `projects`.`id` AS `project_id`, + `projects`.`uuid` AS `project_uuid`, + `projects`.`name` AS `project_name`, + `data_center`.`id` AS `data_center_id`, + `data_center`.`uuid` AS `data_center_uuid`, + `data_center`.`name` AS `data_center_name`, + `launch_permission`.`account_id` AS `lp_account_id`, + `template_store_ref`.`store_id` AS `store_id`, + `image_store`.`scope` AS `store_scope`, + `template_store_ref`.`state` AS `state`, + `template_store_ref`.`download_state` AS `download_state`, + `template_store_ref`.`download_pct` AS `download_pct`, + `template_store_ref`.`error_str` AS `error_str`, + `template_store_ref`.`size` AS `size`, + `template_store_ref`.physical_size AS `physical_size`, + `template_store_ref`.`destroyed` AS `destroyed`, + `template_store_ref`.`created` AS `created_on_store`, + `vm_template_details`.`name` AS `detail_name`, + `vm_template_details`.`value` AS `detail_value`, + `resource_tags`.`id` AS `tag_id`, + `resource_tags`.`uuid` AS `tag_uuid`, + `resource_tags`.`key` AS `tag_key`, + `resource_tags`.`value` AS `tag_value`, + `resource_tags`.`domain_id` AS `tag_domain_id`, + `domain`.`uuid` AS `tag_domain_uuid`, + `domain`.`name` AS `tag_domain_name`, + `resource_tags`.`account_id` AS `tag_account_id`, + `account`.`account_name` AS `tag_account_name`, + `resource_tags`.`resource_id` AS `tag_resource_id`, + `resource_tags`.`resource_uuid` AS `tag_resource_uuid`, + `resource_tags`.`resource_type` AS `tag_resource_type`, + `resource_tags`.`customer` AS `tag_customer`, + CONCAT(`vm_template`.`id`, + '_', + IFNULL(`data_center`.`id`, 0)) AS `temp_zone_pair`, + `vm_template`.`direct_download` AS `direct_download`, + `vm_template`.`deploy_as_is` AS `deploy_as_is` + FROM + (((((((((((((`vm_template` + JOIN `guest_os` ON ((`guest_os`.`id` = `vm_template`.`guest_os_id`))) + JOIN `account` ON ((`account`.`id` = `vm_template`.`account_id`))) + JOIN `domain` ON ((`domain`.`id` = `account`.`domain_id`))) + LEFT JOIN `projects` ON ((`projects`.`project_account_id` = `account`.`id`))) + LEFT JOIN `vm_template_details` ON ((`vm_template_details`.`template_id` = `vm_template`.`id`))) + LEFT JOIN `vm_template` `source_template` ON ((`source_template`.`id` = `vm_template`.`source_template_id`))) + LEFT JOIN `template_store_ref` ON (((`template_store_ref`.`template_id` = `vm_template`.`id`) + AND (`template_store_ref`.`store_role` = 'Image') + AND (`template_store_ref`.`destroyed` = 0)))) + LEFT JOIN `vm_template` `parent_template` ON ((`parent_template`.`id` = `vm_template`.`parent_template_id`))) + LEFT JOIN `image_store` ON ((ISNULL(`image_store`.`removed`) + AND (`template_store_ref`.`store_id` IS NOT NULL) + AND (`image_store`.`id` = `template_store_ref`.`store_id`)))) + LEFT JOIN `template_zone_ref` ON (((`template_zone_ref`.`template_id` = `vm_template`.`id`) + AND ISNULL(`template_store_ref`.`store_id`) + AND ISNULL(`template_zone_ref`.`removed`)))) + LEFT JOIN `data_center` ON (((`image_store`.`data_center_id` = `data_center`.`id`) + OR (`template_zone_ref`.`zone_id` = `data_center`.`id`)))) + LEFT JOIN `launch_permission` ON ((`launch_permission`.`template_id` = `vm_template`.`id`))) + LEFT JOIN `resource_tags` ON (((`resource_tags`.`resource_id` = `vm_template`.`id`) + AND ((`resource_tags`.`resource_type` = 'Template') + OR (`resource_tags`.`resource_type` = 'ISO'))))); + +-- Add mincpu, maxcpu, minmemory and maxmemory to the view supporting constrained offerings +DROP VIEW IF EXISTS `cloud`.`service_offering_view`; +CREATE VIEW `cloud`.`service_offering_view` AS + SELECT + `service_offering`.`id` AS `id`, + `disk_offering`.`uuid` AS `uuid`, + `disk_offering`.`name` AS `name`, + `disk_offering`.`display_text` AS `display_text`, + `disk_offering`.`provisioning_type` AS `provisioning_type`, + `disk_offering`.`created` AS `created`, + `disk_offering`.`tags` AS `tags`, + `disk_offering`.`removed` AS `removed`, + `disk_offering`.`use_local_storage` AS `use_local_storage`, + `disk_offering`.`system_use` AS `system_use`, + `disk_offering`.`customized_iops` AS `customized_iops`, + `disk_offering`.`min_iops` AS `min_iops`, + `disk_offering`.`max_iops` AS `max_iops`, + `disk_offering`.`hv_ss_reserve` AS `hv_ss_reserve`, + `disk_offering`.`bytes_read_rate` AS `bytes_read_rate`, + `disk_offering`.`bytes_read_rate_max` AS `bytes_read_rate_max`, + `disk_offering`.`bytes_read_rate_max_length` AS `bytes_read_rate_max_length`, + `disk_offering`.`bytes_write_rate` AS `bytes_write_rate`, + `disk_offering`.`bytes_write_rate_max` AS `bytes_write_rate_max`, + `disk_offering`.`bytes_write_rate_max_length` AS `bytes_write_rate_max_length`, + `disk_offering`.`iops_read_rate` AS `iops_read_rate`, + `disk_offering`.`iops_read_rate_max` AS `iops_read_rate_max`, + `disk_offering`.`iops_read_rate_max_length` AS `iops_read_rate_max_length`, + `disk_offering`.`iops_write_rate` AS `iops_write_rate`, + `disk_offering`.`iops_write_rate_max` AS `iops_write_rate_max`, + `disk_offering`.`iops_write_rate_max_length` AS `iops_write_rate_max_length`, + `disk_offering`.`cache_mode` AS `cache_mode`, + `service_offering`.`cpu` AS `cpu`, + `service_offering`.`speed` AS `speed`, + `service_offering`.`ram_size` AS `ram_size`, + `service_offering`.`nw_rate` AS `nw_rate`, + `service_offering`.`mc_rate` AS `mc_rate`, + `service_offering`.`ha_enabled` AS `ha_enabled`, + `service_offering`.`limit_cpu_use` AS `limit_cpu_use`, + `service_offering`.`host_tag` AS `host_tag`, + `service_offering`.`default_use` AS `default_use`, + `service_offering`.`vm_type` AS `vm_type`, + `service_offering`.`sort_key` AS `sort_key`, + `service_offering`.`is_volatile` AS `is_volatile`, + `service_offering`.`deployment_planner` AS `deployment_planner`, + GROUP_CONCAT(DISTINCT(domain.id)) AS domain_id, + GROUP_CONCAT(DISTINCT(domain.uuid)) AS domain_uuid, + GROUP_CONCAT(DISTINCT(domain.name)) AS domain_name, + GROUP_CONCAT(DISTINCT(domain.path)) AS domain_path, + GROUP_CONCAT(DISTINCT(zone.id)) AS zone_id, + GROUP_CONCAT(DISTINCT(zone.uuid)) AS zone_uuid, + GROUP_CONCAT(DISTINCT(zone.name)) AS zone_name, + IFNULL(`min_compute_details`.`value`, `cpu`) AS min_cpu, + IFNULL(`max_compute_details`.`value`, `cpu`) AS max_cpu, + IFNULL(`min_memory_details`.`value`, `ram_size`) AS min_memory, + IFNULL(`max_memory_details`.`value`, `ram_size`) AS max_memory + FROM + `cloud`.`service_offering` + INNER JOIN + `cloud`.`disk_offering_view` AS `disk_offering` ON service_offering.id = disk_offering.id + LEFT JOIN + `cloud`.`service_offering_details` AS `domain_details` ON `domain_details`.`service_offering_id` = `disk_offering`.`id` AND `domain_details`.`name`='domainid' + LEFT JOIN + `cloud`.`domain` AS `domain` ON FIND_IN_SET(`domain`.`id`, `domain_details`.`value`) + LEFT JOIN + `cloud`.`service_offering_details` AS `zone_details` ON `zone_details`.`service_offering_id` = `disk_offering`.`id` AND `zone_details`.`name`='zoneid' + LEFT JOIN + `cloud`.`data_center` AS `zone` ON FIND_IN_SET(`zone`.`id`, `zone_details`.`value`) + LEFT JOIN + `cloud`.`service_offering_details` AS `min_compute_details` ON `min_compute_details`.`service_offering_id` = `disk_offering`.`id` + AND `min_compute_details`.`name` = 'mincpunumber' + LEFT JOIN + `cloud`.`service_offering_details` AS `max_compute_details` ON `max_compute_details`.`service_offering_id` = `disk_offering`.`id` + AND `max_compute_details`.`name` = 'maxcpunumber' + LEFT JOIN + `cloud`.`service_offering_details` AS `min_memory_details` ON `min_memory_details`.`service_offering_id` = `disk_offering`.`id` + AND `min_memory_details`.`name` = 'minmemory' + LEFT JOIN + `cloud`.`service_offering_details` AS `max_memory_details` ON `max_memory_details`.`service_offering_id` = `disk_offering`.`id` + AND `max_memory_details`.`name` = 'maxmemory' + WHERE + `disk_offering`.`state`='Active' + GROUP BY + `service_offering`.`id`; + +ALTER TABLE `cloud`.`template_spool_ref` +DROP FOREIGN KEY `fk_template_spool_ref__template_id`; + +ALTER TABLE `cloud`.`template_spool_ref` +ADD COLUMN `deployment_option` VARCHAR(255) NULL DEFAULT NULL AFTER `updated`, +ADD INDEX `fk_template_spool_ref__template_id_idx` (`template_id` ASC), +ADD UNIQUE INDEX `index_template_spool_configuration` (`pool_id` ASC, `template_id` ASC, `deployment_option` ASC), +DROP INDEX `i_template_spool_ref__template_id__pool_id` ; + +ALTER TABLE `cloud`.`template_spool_ref` +ADD CONSTRAINT `fk_template_spool_ref__template_id` + FOREIGN KEY (`template_id`) + REFERENCES `cloud`.`vm_template` (`id`) + ON DELETE NO ACTION + ON UPDATE NO ACTION; + +CREATE TABLE `cloud`.`template_deploy_as_is_details` ( + `id` bigint unsigned NOT NULL auto_increment, + `template_id` bigint unsigned NOT NULL COMMENT 'template id', + `name` varchar(255) NOT NULL, + `value` TEXT, + PRIMARY KEY (`id`), + CONSTRAINT `fk_template_deploy_as_is_details__template_id` FOREIGN KEY `fk_template_deploy_as_is_details__template_id`(`template_id`) REFERENCES `vm_template`(`id`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +CREATE TABLE `cloud`.`user_vm_deploy_as_is_details` ( + `id` bigint unsigned NOT NULL auto_increment, + `vm_id` bigint unsigned NOT NULL COMMENT 'virtual machine id', + `name` varchar(255) NOT NULL, + `value` TEXT, + PRIMARY KEY (`id`), + CONSTRAINT `fk_user_vm_deploy_as_is_details__vm_id` FOREIGN KEY `fk_user_vm_deploy_as_is_details__vm_id`(`vm_id`) REFERENCES `vm_instance`(`id`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8; diff --git a/engine/storage/cache/src/main/java/org/apache/cloudstack/storage/cache/manager/StorageCacheManagerImpl.java b/engine/storage/cache/src/main/java/org/apache/cloudstack/storage/cache/manager/StorageCacheManagerImpl.java index 278c80d32475..40eeafef6924 100644 --- a/engine/storage/cache/src/main/java/org/apache/cloudstack/storage/cache/manager/StorageCacheManagerImpl.java +++ b/engine/storage/cache/src/main/java/org/apache/cloudstack/storage/cache/manager/StorageCacheManagerImpl.java @@ -291,7 +291,7 @@ public DataObject createCacheObject(DataObject data, DataStore store) { if (st == ObjectInDataStoreStateMachine.State.Ready) { s_logger.debug("there is already one in the cache store"); - DataObject dataObj = objectInStoreMgr.get(data, store); + DataObject dataObj = objectInStoreMgr.get(data, store, null); dataObj.incRefCount(); existingDataObj = dataObj; } diff --git a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/DataMotionServiceImpl.java b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/DataMotionServiceImpl.java index c2724e648241..ac6c8555da96 100644 --- a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/DataMotionServiceImpl.java +++ b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/DataMotionServiceImpl.java @@ -25,12 +25,6 @@ import javax.inject.Inject; -import com.cloud.storage.Volume; -import com.cloud.storage.VolumeVO; -import com.cloud.storage.dao.VolumeDao; -import org.apache.log4j.Logger; -import org.springframework.stereotype.Component; - import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; import org.apache.cloudstack.engine.subsystem.api.storage.DataMotionService; import org.apache.cloudstack.engine.subsystem.api.storage.DataMotionStrategy; @@ -39,9 +33,14 @@ import org.apache.cloudstack.engine.subsystem.api.storage.StorageStrategyFactory; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; import org.apache.cloudstack.framework.async.AsyncCompletionCallback; +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; import com.cloud.agent.api.to.VirtualMachineTO; import com.cloud.host.Host; +import com.cloud.storage.Volume; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.VolumeDao; import com.cloud.utils.StringUtils; import com.cloud.utils.exception.CloudRuntimeException; diff --git a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageDataMotionStrategy.java b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageDataMotionStrategy.java index e6b5c85b924b..971859685ff1 100644 --- a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageDataMotionStrategy.java +++ b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageDataMotionStrategy.java @@ -195,7 +195,7 @@ protected boolean shouldMigrateVolume(StoragePoolVO sourceStoragePool, Host dest @Override protected void copyTemplateToTargetFilesystemStorageIfNeeded(VolumeInfo srcVolumeInfo, StoragePool srcStoragePool, DataStore destDataStore, StoragePool destStoragePool, Host destHost) { - VMTemplateStoragePoolVO sourceVolumeTemplateStoragePoolVO = vmTemplatePoolDao.findByPoolTemplate(destStoragePool.getId(), srcVolumeInfo.getTemplateId()); + VMTemplateStoragePoolVO sourceVolumeTemplateStoragePoolVO = vmTemplatePoolDao.findByPoolTemplate(destStoragePool.getId(), srcVolumeInfo.getTemplateId(), null); if (sourceVolumeTemplateStoragePoolVO == null && destStoragePool.getPoolType() == StoragePoolType.Filesystem) { DataStore sourceTemplateDataStore = dataStoreManagerImpl.getRandomImageStore(srcVolumeInfo.getDataCenterId()); if (sourceTemplateDataStore != null) { @@ -220,8 +220,8 @@ protected void copyTemplateToTargetFilesystemStorageIfNeeded(VolumeInfo srcVolum * Update the template reference on table "template_spool_ref" (VMTemplateStoragePoolVO). */ protected void updateTemplateReferenceIfSuccessfulCopy(VolumeInfo srcVolumeInfo, StoragePool srcStoragePool, TemplateInfo destTemplateInfo, DataStore destDataStore) { - VMTemplateStoragePoolVO srcVolumeTemplateStoragePoolVO = vmTemplatePoolDao.findByPoolTemplate(srcStoragePool.getId(), srcVolumeInfo.getTemplateId()); - VMTemplateStoragePoolVO destVolumeTemplateStoragePoolVO = new VMTemplateStoragePoolVO(destDataStore.getId(), srcVolumeInfo.getTemplateId()); + VMTemplateStoragePoolVO srcVolumeTemplateStoragePoolVO = vmTemplatePoolDao.findByPoolTemplate(srcStoragePool.getId(), srcVolumeInfo.getTemplateId(), null); + VMTemplateStoragePoolVO destVolumeTemplateStoragePoolVO = new VMTemplateStoragePoolVO(destDataStore.getId(), srcVolumeInfo.getTemplateId(), null); destVolumeTemplateStoragePoolVO.setDownloadPercent(100); destVolumeTemplateStoragePoolVO.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOADED); destVolumeTemplateStoragePoolVO.setState(ObjectInDataStoreStateMachine.State.Ready); diff --git a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java index 4d3ec184ac16..936f0626af32 100644 --- a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java +++ b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java @@ -1710,7 +1710,7 @@ private SnapshotDetailsVO handleSnapshotDetails(long csSnapshotId, String value) * Return expected MigrationOptions for a linked clone volume live storage migration */ protected MigrationOptions createLinkedCloneMigrationOptions(VolumeInfo srcVolumeInfo, VolumeInfo destVolumeInfo, String srcVolumeBackingFile, String srcPoolUuid, Storage.StoragePoolType srcPoolType) { - VMTemplateStoragePoolVO ref = templatePoolDao.findByPoolTemplate(destVolumeInfo.getPoolId(), srcVolumeInfo.getTemplateId()); + VMTemplateStoragePoolVO ref = templatePoolDao.findByPoolTemplate(destVolumeInfo.getPoolId(), srcVolumeInfo.getTemplateId(), null); boolean updateBackingFileReference = ref == null; String backingFile = ref != null ? ref.getInstallPath() : srcVolumeBackingFile; return new MigrationOptions(srcPoolUuid, srcPoolType, backingFile, updateBackingFileReference); @@ -1983,7 +1983,7 @@ private String getVolumeBackingFile(VolumeInfo srcVolumeInfo) { srcVolumeInfo.getTemplateId() != null && srcVolumeInfo.getPoolId() != null) { VMTemplateVO template = _vmTemplateDao.findById(srcVolumeInfo.getTemplateId()); if (template.getFormat() != null && template.getFormat() != Storage.ImageFormat.ISO) { - VMTemplateStoragePoolVO ref = templatePoolDao.findByPoolTemplate(srcVolumeInfo.getPoolId(), srcVolumeInfo.getTemplateId()); + VMTemplateStoragePoolVO ref = templatePoolDao.findByPoolTemplate(srcVolumeInfo.getPoolId(), srcVolumeInfo.getTemplateId(), null); return ref != null ? ref.getInstallPath() : null; } } @@ -2157,8 +2157,8 @@ private List sendModifyTargetsCommand(ModifyTargetsCommand cmd, long hos * Update reference on template_spool_ref table of copied template to destination storage */ protected void updateCopiedTemplateReference(VolumeInfo srcVolumeInfo, VolumeInfo destVolumeInfo) { - VMTemplateStoragePoolVO ref = templatePoolDao.findByPoolTemplate(srcVolumeInfo.getPoolId(), srcVolumeInfo.getTemplateId()); - VMTemplateStoragePoolVO newRef = new VMTemplateStoragePoolVO(destVolumeInfo.getPoolId(), ref.getTemplateId()); + VMTemplateStoragePoolVO ref = templatePoolDao.findByPoolTemplate(srcVolumeInfo.getPoolId(), srcVolumeInfo.getTemplateId(), null); + VMTemplateStoragePoolVO newRef = new VMTemplateStoragePoolVO(destVolumeInfo.getPoolId(), ref.getTemplateId(), null); newRef.setDownloadPercent(100); newRef.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOADED); newRef.setState(ObjectInDataStoreStateMachine.State.Ready); diff --git a/engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageSystemDataMotionTest.java b/engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageSystemDataMotionTest.java index 6971444bcad0..ba7fb74da1d8 100644 --- a/engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageSystemDataMotionTest.java +++ b/engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageSystemDataMotionTest.java @@ -19,6 +19,7 @@ package org.apache.cloudstack.storage.motion; import static org.junit.Assert.assertEquals; +import static org.mockito.ArgumentMatchers.nullable; import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.when; @@ -301,7 +302,7 @@ private void configureAndTestSendCommandTest(Class exc public void copyTemplateToTargetStorageIfNeededTestTemplateAlreadyOnTargetHost() throws AgentUnavailableException, OperationTimedoutException { Answer copyCommandAnswer = Mockito.mock(Answer.class); Mockito.lenient().when(copyCommandAnswer.getResult()).thenReturn(true); - configureAndTestcopyTemplateToTargetStorageIfNeeded(new VMTemplateStoragePoolVO(0l, 0l), StoragePoolType.Filesystem, 0); + configureAndTestcopyTemplateToTargetStorageIfNeeded(new VMTemplateStoragePoolVO(0l, 0l, null), StoragePoolType.Filesystem, 0); } @Test @@ -316,7 +317,7 @@ public void migrateTemplateToTargetStorageIfNeededTestNonDesiredStoragePoolType( if (storagePoolTypeArray[i] == StoragePoolType.Filesystem) { continue; } - configureAndTestcopyTemplateToTargetStorageIfNeeded(new VMTemplateStoragePoolVO(0l, 0l), storagePoolTypeArray[i], 0); + configureAndTestcopyTemplateToTargetStorageIfNeeded(new VMTemplateStoragePoolVO(0l, 0l, null), storagePoolTypeArray[i], 0); } } @@ -353,7 +354,7 @@ private void configureAndTestcopyTemplateToTargetStorageIfNeeded(VMTemplateStora Mockito.when(sourceTemplateInfo.getSize()).thenReturn(0l); Mockito.when(sourceTemplateInfo.getHypervisorType()).thenReturn(HypervisorType.KVM); - Mockito.when(vmTemplatePoolDao.findByPoolTemplate(Mockito.anyLong(), Mockito.anyLong())).thenReturn(vmTemplateStoragePoolVO); + Mockito.when(vmTemplatePoolDao.findByPoolTemplate(Mockito.anyLong(), Mockito.anyLong(), nullable(String.class))).thenReturn(vmTemplateStoragePoolVO); Mockito.when(dataStoreManagerImpl.getRandomImageStore(Mockito.anyLong())).thenReturn(sourceTemplateDataStore); Mockito.when(templateDataFactory.getTemplate(Mockito.anyLong(), Mockito.eq(sourceTemplateDataStore))).thenReturn(sourceTemplateInfo); Mockito.when(templateDataFactory.getTemplate(Mockito.anyLong(), Mockito.eq(destDataStore))).thenReturn(sourceTemplateInfo); @@ -362,7 +363,7 @@ private void configureAndTestcopyTemplateToTargetStorageIfNeeded(VMTemplateStora Mockito.any(TemplateInfo.class), Mockito.any(DataStore.class)); InOrder verifyInOrder = Mockito.inOrder(vmTemplatePoolDao, dataStoreManagerImpl, templateDataFactory, kvmNonManagedStorageDataMotionStrategy); - verifyInOrder.verify(vmTemplatePoolDao, Mockito.times(1)).findByPoolTemplate(Mockito.anyLong(), Mockito.anyLong()); + verifyInOrder.verify(vmTemplatePoolDao, Mockito.times(1)).findByPoolTemplate(Mockito.anyLong(), Mockito.anyLong(), nullable(String.class)); verifyInOrder.verify(dataStoreManagerImpl, Mockito.times(times)).getRandomImageStore(Mockito.anyLong()); verifyInOrder.verify(templateDataFactory, Mockito.times(times)).getTemplate(Mockito.anyLong(), Mockito.eq(sourceTemplateDataStore)); verifyInOrder.verify(templateDataFactory, Mockito.times(times)).getTemplate(Mockito.anyLong(), Mockito.eq(destDataStore)); diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/SecondaryStorageServiceImpl.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/SecondaryStorageServiceImpl.java new file mode 100644 index 000000000000..5a9c4a9f12ea --- /dev/null +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/SecondaryStorageServiceImpl.java @@ -0,0 +1,203 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.storage.image; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; + +import javax.inject.Inject; + +import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; +import org.apache.cloudstack.engine.subsystem.api.storage.DataMotionService; +import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; +import org.apache.cloudstack.engine.subsystem.api.storage.SecondaryStorageService; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; +import org.apache.cloudstack.framework.async.AsyncCallFuture; +import org.apache.cloudstack.framework.async.AsyncCallbackDispatcher; +import org.apache.cloudstack.framework.async.AsyncCompletionCallback; +import org.apache.cloudstack.framework.async.AsyncRpcContext; +import org.apache.cloudstack.storage.command.CopyCmdAnswer; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; + +import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO; +import org.apache.log4j.Logger; + +import com.cloud.secstorage.CommandExecLogDao; +import com.cloud.storage.DataStoreRole; +import com.cloud.utils.Pair; + +public class SecondaryStorageServiceImpl implements SecondaryStorageService { + + private static final Logger s_logger = Logger.getLogger(SecondaryStorageServiceImpl.class); + + @Inject + DataMotionService motionSrv; + @Inject + CommandExecLogDao _cmdExecLogDao; + @Inject + TemplateDataStoreDao templateStoreDao; + @Inject + SnapshotDataStoreDao snapshotStoreDao; + @Inject + VolumeDataStoreDao volumeDataStoreDao; + + private class MigrateDataContext extends AsyncRpcContext { + final DataObject srcData; + final DataObject destData; + final AsyncCallFuture future; + + /** + * @param callback + */ + public MigrateDataContext(AsyncCompletionCallback callback, AsyncCallFuture future, DataObject srcData, DataObject destData, DataStore destStore) { + super(callback); + this.srcData = srcData; + this.destData = destData; + this.future = future; + } + } + + @Override + public AsyncCallFuture migrateData(DataObject srcDataObject, DataStore srcDatastore, DataStore destDatastore, Map, Long>> snapshotChain) { + AsyncCallFuture future = new AsyncCallFuture(); + DataObjectResult res = new DataObjectResult(srcDataObject); + DataObject destDataObject = null; + try { + if (srcDataObject instanceof SnapshotInfo && snapshotChain != null && snapshotChain.containsKey(srcDataObject)) { + for (SnapshotInfo snapshotInfo : snapshotChain.get(srcDataObject).first()) { + destDataObject = destDatastore.create(snapshotInfo); + snapshotInfo.processEvent(ObjectInDataStoreStateMachine.Event.MigrateDataRequested); + destDataObject.processEvent(ObjectInDataStoreStateMachine.Event.MigrateDataRequested); + migrateJob(future, snapshotInfo, destDataObject, destDatastore); + } + } else { + // Check if template in destination store, if yes, do not proceed + if (srcDataObject instanceof TemplateInfo) { + s_logger.debug("Checking if template present at destination"); + TemplateDataStoreVO templateStoreVO = templateStoreDao.findByStoreTemplate(destDatastore.getId(), srcDataObject.getId()); + if (templateStoreVO != null) { + String msg = "Template already exists in destination store"; + s_logger.debug(msg); + res.setResult(msg); + res.setSuccess(true); + future.complete(res); + return future; + } + } + destDataObject = destDatastore.create(srcDataObject); + srcDataObject.processEvent(ObjectInDataStoreStateMachine.Event.MigrateDataRequested); + destDataObject.processEvent(ObjectInDataStoreStateMachine.Event.MigrateDataRequested); + migrateJob(future, srcDataObject, destDataObject, destDatastore); + } + } catch (Exception e) { + s_logger.debug("Failed to copy Data", e); + if (destDataObject != null) { + destDataObject.getDataStore().delete(destDataObject); + } + if (!(srcDataObject instanceof VolumeInfo)) { + srcDataObject.processEvent(ObjectInDataStoreStateMachine.Event.OperationFailed); + } else { + ((VolumeInfo) srcDataObject).processEventOnly(ObjectInDataStoreStateMachine.Event.OperationFailed); + } + res.setResult(e.toString()); + future.complete(res); + } + return future; + } + + protected void migrateJob(AsyncCallFuture future, DataObject srcDataObject, DataObject destDataObject, DataStore destDatastore) throws ExecutionException, InterruptedException { + MigrateDataContext context = new MigrateDataContext(null, future, srcDataObject, destDataObject, destDatastore); + AsyncCallbackDispatcher caller = AsyncCallbackDispatcher.create(this); + caller.setCallback(caller.getTarget().migrateDataCallBack(null, null)).setContext(context); + motionSrv.copyAsync(srcDataObject, destDataObject, caller); + } + + /** + * Callback function to handle state change of source and destination data objects based on the success or failure of the migrate task + */ + protected Void migrateDataCallBack(AsyncCallbackDispatcher callback, MigrateDataContext context) throws ExecutionException, InterruptedException { + DataObject srcData = context.srcData; + DataObject destData = context.destData; + CopyCommandResult result = callback.getResult(); + AsyncCallFuture future = context.future; + DataObjectResult res = new DataObjectResult(srcData); + CopyCmdAnswer answer = (CopyCmdAnswer) result.getAnswer(); + try { + if (!answer.getResult()) { + s_logger.warn("Migration failed for "+srcData.getUuid()); + res.setResult(result.getResult()); + if (!(srcData instanceof VolumeInfo) ) { + srcData.processEvent(ObjectInDataStoreStateMachine.Event.OperationFailed); + destData.processEvent(ObjectInDataStoreStateMachine.Event.MigrationFailed); + destData.processEvent(ObjectInDataStoreStateMachine.Event.DestroyRequested); + } else { + ((VolumeInfo)srcData).processEventOnly(ObjectInDataStoreStateMachine.Event.OperationFailed); + ((VolumeInfo)destData).processEventOnly(ObjectInDataStoreStateMachine.Event.MigrationFailed); + ((VolumeInfo)destData).processEventOnly(ObjectInDataStoreStateMachine.Event.DestroyRequested); + } + + if (destData != null) { + destData.getDataStore().delete(destData); + } + + } else { + if (destData instanceof VolumeInfo) { + ((VolumeInfo) destData).processEventOnly(ObjectInDataStoreStateMachine.Event.OperationSuccessed, answer); + } else { + destData.processEvent(ObjectInDataStoreStateMachine.Event.OperationSuccessed, answer); + } + if (destData instanceof SnapshotInfo) { + SnapshotDataStoreVO snapshotStore = snapshotStoreDao.findBySourceSnapshot(srcData.getId(), DataStoreRole.Image); + SnapshotDataStoreVO destSnapshotStore = snapshotStoreDao.findBySnapshot(srcData.getId(), DataStoreRole.Image); + destSnapshotStore.setPhysicalSize(snapshotStore.getPhysicalSize()); + snapshotStoreDao.update(destSnapshotStore.getId(), destSnapshotStore); + } + + if (destData instanceof VolumeInfo) { + VolumeDataStoreVO srcVolume = volumeDataStoreDao.findByStoreVolume(srcData.getDataStore().getId(), srcData.getId()); + VolumeDataStoreVO destVolume = volumeDataStoreDao.findByStoreVolume(destData.getDataStore().getId(), destData.getId()); + destVolume.setPhysicalSize(srcVolume.getPhysicalSize()); + volumeDataStoreDao.update(destVolume.getId(), destVolume); + } + s_logger.debug("Deleting source data"); + srcData.getDataStore().delete(srcData); + s_logger.debug("Successfully migrated "+srcData.getUuid()); + } + _cmdExecLogDao.expunge(Long.parseLong(answer.getContextParam("cmd"))); + future.complete(res); + } catch (Exception e) { + s_logger.error("Failed to process migrate data callback", e); + res.setResult(e.toString()); + _cmdExecLogDao.expunge(Long.parseLong(answer.getContextParam("cmd"))); + future.complete(res); + } + return null; + } + +} + + diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateDataFactoryImpl.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateDataFactoryImpl.java index 8343a74d60b7..1590fe0bf7de 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateDataFactoryImpl.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateDataFactoryImpl.java @@ -18,21 +18,20 @@ */ package org.apache.cloudstack.storage.image; -import com.cloud.host.HostVO; -import com.cloud.host.dao.HostDao; import java.util.ArrayList; import java.util.List; import javax.inject.Inject; import com.cloud.hypervisor.Hypervisor; +import com.cloud.utils.StringUtils; import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.direct.download.DirectDownloadManager; import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; -import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; @@ -41,6 +40,8 @@ import org.apache.log4j.Logger; import org.springframework.stereotype.Component; +import com.cloud.host.HostVO; +import com.cloud.host.dao.HostDao; import com.cloud.storage.DataStoreRole; import com.cloud.storage.VMTemplateStoragePoolVO; import com.cloud.storage.VMTemplateVO; @@ -65,17 +66,30 @@ public class TemplateDataFactoryImpl implements TemplateDataFactory { @Inject PrimaryDataStoreDao primaryDataStoreDao; + @Override + public TemplateInfo getTemplateOnPrimaryStorage(long templateId, DataStore store, String configuration) { + VMTemplateVO templ = imageDataDao.findById(templateId); + if (store.getRole() == DataStoreRole.Primary) { + VMTemplateStoragePoolVO templatePoolVO = templatePoolDao.findByPoolTemplate(store.getId(), templateId, configuration); + if (templatePoolVO != null) { + String deployAsIsConfiguration = templatePoolVO.getDeploymentOption(); + return TemplateObject.getTemplate(templ, store, deployAsIsConfiguration); + } + } + return null; + } + @Override public TemplateInfo getTemplate(long templateId, DataStore store) { VMTemplateVO templ = imageDataDao.findById(templateId); if (store == null && !templ.isDirectDownload()) { - TemplateObject tmpl = TemplateObject.getTemplate(templ, null); + TemplateObject tmpl = TemplateObject.getTemplate(templ, null, null); return tmpl; } // verify if the given input parameters are consistent with our db data. boolean found = false; if (store.getRole() == DataStoreRole.Primary) { - VMTemplateStoragePoolVO templatePoolVO = templatePoolDao.findByPoolTemplate(store.getId(), templateId); + VMTemplateStoragePoolVO templatePoolVO = templatePoolDao.findByPoolTemplate(store.getId(), templateId, null); if (templatePoolVO != null) { found = true; } @@ -94,7 +108,7 @@ public TemplateInfo getTemplate(long templateId, DataStore store) { } } - TemplateObject tmpl = TemplateObject.getTemplate(templ, store); + TemplateObject tmpl = TemplateObject.getTemplate(templ, store, null); return tmpl; } @@ -130,8 +144,13 @@ public TemplateInfo getReadyTemplateOnImageStore(long templateId, Long zoneId) { } @Override - public TemplateInfo getTemplate(DataObject obj, DataStore store) { - TemplateObject tmpObj = (TemplateObject)this.getTemplate(obj.getId(), store); + public TemplateInfo getTemplate(DataObject obj, DataStore store, String configuration) { + TemplateObject tmpObj; + if (StringUtils.isNotBlank(configuration)) { + tmpObj = (TemplateObject)this.getTemplateOnPrimaryStorage(obj.getId(), store, configuration); + } else { + tmpObj = (TemplateObject)this.getTemplate(obj.getId(), store); + } // carry over url set in passed in data object, for copyTemplate case // where url is generated on demand and not persisted in DB. // need to think of a more generic way to pass these runtime information @@ -217,7 +236,7 @@ public TemplateInfo getReadyBypassedTemplateOnPrimaryStore(long templateId, Long if (pool == null) { throw new CloudRuntimeException("No storage pool found where to download template: " + templateId); } - VMTemplateStoragePoolVO spoolRef = templatePoolDao.findByPoolTemplate(pool, templateId); + VMTemplateStoragePoolVO spoolRef = templatePoolDao.findByPoolTemplate(pool, templateId, null); if (spoolRef == null) { directDownloadManager.downloadTemplate(templateId, pool, hostId); } @@ -230,5 +249,4 @@ public boolean isTemplateMarkedForDirectDownload(long templateId) { VMTemplateVO templateVO = imageDataDao.findById(templateId); return templateVO.isDirectDownload(); } - } diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java index ee056309f793..ed9359d952af 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java @@ -204,7 +204,7 @@ public void createTemplateAsync(TemplateInfo template, DataStore store, AsyncCom // clean up already persisted template_store_ref entry in case of createTemplateCallback is never called TemplateDataStoreVO templateStoreVO = _vmTemplateStoreDao.findByStoreTemplate(store.getId(), template.getId()); if (templateStoreVO != null) { - TemplateInfo tmplObj = _templateFactory.getTemplate(template, store); + TemplateInfo tmplObj = _templateFactory.getTemplate(template, store, null); tmplObj.processEvent(ObjectInDataStoreStateMachine.Event.OperationFailed); } TemplateApiResult result = new TemplateApiResult(template); @@ -253,7 +253,7 @@ public void downloadBootstrapSysTemplate(DataStore store) { @Override public void handleSysTemplateDownload(HypervisorType hostHyper, Long dcId) { Set toBeDownloaded = new HashSet(); - List stores = _storeMgr.getImageStoresByScope(new ZoneScope(dcId)); + List stores = _storeMgr.getImageStoresByScopeExcludingReadOnly(new ZoneScope(dcId)); if (stores == null || stores.isEmpty()) { return; } @@ -412,7 +412,7 @@ public void handleTemplateSync(DataStore store) { VirtualMachineTemplate.Event event = VirtualMachineTemplate.Event.OperationSucceeded; // For multi-disk OVA, check and create data disk templates if (tmplt.getFormat().equals(ImageFormat.OVA)) { - if (!createOvaDataDiskTemplates(_templateFactory.getTemplate(tmlpt.getId(), store))) { + if (!createOvaDataDiskTemplates(_templateFactory.getTemplate(tmlpt.getId(), store), tmplt.isDeployAsIs())) { event = VirtualMachineTemplate.Event.OperationFailed; } } @@ -710,7 +710,7 @@ protected Void createTemplateCallback(AsyncCallbackDispatcher getTemplateDatadisksOnImageStore(TemplateInfo templateInfo, String configurationId) { + ImageStoreEntity tmpltStore = (ImageStoreEntity)templateInfo.getDataStore(); + return tmpltStore.getDataDiskTemplates(templateInfo, configurationId); + } + + @Override + public boolean createOvaDataDiskTemplates(TemplateInfo parentTemplate, boolean deployAsIs) { try { // Get Datadisk template (if any) for OVA List dataDiskTemplates = new ArrayList(); ImageStoreEntity tmpltStore = (ImageStoreEntity)parentTemplate.getDataStore(); - dataDiskTemplates = tmpltStore.getDataDiskTemplates(parentTemplate); + dataDiskTemplates = tmpltStore.getDataDiskTemplates(parentTemplate, null); int diskCount = 0; VMTemplateVO templateVO = _templateDao.findById(parentTemplate.getId()); _templateDao.loadDetails(templateVO); @@ -754,23 +760,27 @@ public boolean createOvaDataDiskTemplates(TemplateInfo parentTemplate) { details = new HashMap<>(); } } + for (DatadiskTO diskTemplate : dataDiskTemplates) { - if (!diskTemplate.isBootable()) { - createChildDataDiskTemplate(diskTemplate, templateVO, parentTemplate, imageStore, diskCount++); - if (!diskTemplate.isIso() && Strings.isNullOrEmpty(details.get(VmDetailConstants.DATA_DISK_CONTROLLER))){ - details.put(VmDetailConstants.DATA_DISK_CONTROLLER, getOvaDiskControllerDetails(diskTemplate, false)); - details.put(VmDetailConstants.DATA_DISK_CONTROLLER + diskTemplate.getDiskId(), getOvaDiskControllerDetails(diskTemplate, false)); - } - } else { - finalizeParentTemplate(diskTemplate, templateVO, parentTemplate, imageStore, diskCount++); - if (Strings.isNullOrEmpty(VmDetailConstants.ROOT_DISK_CONTROLLER)) { - final String rootDiskController = getOvaDiskControllerDetails(diskTemplate, true); - if (!Strings.isNullOrEmpty(rootDiskController)) { - details.put(VmDetailConstants.ROOT_DISK_CONTROLLER, rootDiskController); + if (!deployAsIs) { + if (!diskTemplate.isBootable()) { + createChildDataDiskTemplate(diskTemplate, templateVO, parentTemplate, imageStore, diskCount++); + if (!diskTemplate.isIso() && Strings.isNullOrEmpty(details.get(VmDetailConstants.DATA_DISK_CONTROLLER))){ + details.put(VmDetailConstants.DATA_DISK_CONTROLLER, getOvaDiskControllerDetails(diskTemplate, false)); + details.put(VmDetailConstants.DATA_DISK_CONTROLLER + diskTemplate.getDiskId(), getOvaDiskControllerDetails(diskTemplate, false)); + } + } else { + finalizeParentTemplate(diskTemplate, templateVO, parentTemplate, imageStore, diskCount++); + if (Strings.isNullOrEmpty(VmDetailConstants.ROOT_DISK_CONTROLLER)) { + final String rootDiskController = getOvaDiskControllerDetails(diskTemplate, true); + if (!Strings.isNullOrEmpty(rootDiskController)) { + details.put(VmDetailConstants.ROOT_DISK_CONTROLLER, rootDiskController); + } } } } } + templateVO.setDetails(details); _templateDao.saveDetails(templateVO); return true; @@ -789,7 +799,7 @@ private boolean createChildDataDiskTemplate(DatadiskTO dataDiskTemplate, VMTempl String templateName = dataDiskTemplate.isIso() ? dataDiskTemplate.getPath().substring(dataDiskTemplate.getPath().lastIndexOf(File.separator) + 1) : template.getName() + suffix + diskCount; VMTemplateVO templateVO = new VMTemplateVO(templateId, templateName, format, false, false, false, ttype, template.getUrl(), template.requiresHvm(), template.getBits(), template.getAccountId(), null, templateName, false, guestOsId, false, template.getHypervisorType(), null, - null, false, false, false); + null, false, false, false, false); if (dataDiskTemplate.isIso()){ templateVO.setUniqueName(templateName); } @@ -952,7 +962,7 @@ private AsyncCallFuture syncToRegionStoreAsync(TemplateInfo t AsyncCallFuture future = new AsyncCallFuture(); // no need to create entry on template_store_ref here, since entries are already created when prepareSecondaryStorageForMigration is invoked. // But we need to set default install path so that sync can be done in the right s3 path - TemplateInfo templateOnStore = _templateFactory.getTemplate(template, store); + TemplateInfo templateOnStore = _templateFactory.getTemplate(template, store, null); String installPath = TemplateConstants.DEFAULT_TMPLT_ROOT_DIR + "/" + TemplateConstants.DEFAULT_TMPLT_FIRST_LEVEL_DIR + template.getAccountId() + "/" + template.getId() + "/" + template.getUniqueName(); @@ -1039,7 +1049,7 @@ public AsyncCallFuture copyTemplate(TemplateInfo srcTemplate, throw new CloudRuntimeException("No secondary VM in running state in source template zone "); } - TemplateObject tmplForCopy = (TemplateObject)_templateFactory.getTemplate(srcTemplate, destStore); + TemplateObject tmplForCopy = (TemplateObject)_templateFactory.getTemplate(srcTemplate, destStore, null); if (s_logger.isDebugEnabled()) { s_logger.debug("Setting source template url to " + url); } @@ -1068,7 +1078,7 @@ public AsyncCallFuture copyTemplate(TemplateInfo srcTemplate, // clean up already persisted template_store_ref entry in case of createTemplateCallback is never called TemplateDataStoreVO templateStoreVO = _vmTemplateStoreDao.findByStoreTemplate(destStore.getId(), srcTemplate.getId()); if (templateStoreVO != null) { - TemplateInfo tmplObj = _templateFactory.getTemplate(srcTemplate, destStore); + TemplateInfo tmplObj = _templateFactory.getTemplate(srcTemplate, destStore, null); tmplObj.processEvent(ObjectInDataStoreStateMachine.Event.OperationFailed); } TemplateApiResult res = new TemplateApiResult((TemplateObject)templateOnStore); @@ -1130,7 +1140,7 @@ public AsyncCallFuture prepareTemplateOnPrimary(TemplateInfo @Override public AsyncCallFuture deleteTemplateOnPrimary(TemplateInfo template, StoragePool pool) { - TemplateObject templateObject = (TemplateObject)_templateFactory.getTemplate(template.getId(), (DataStore)pool); + TemplateObject templateObject = (TemplateObject)_templateFactory.getTemplateOnPrimaryStorage(template.getId(), (DataStore)pool, template.getDeployAsIsConfiguration()); templateObject.processEvent(ObjectInDataStoreStateMachine.Event.DestroyRequested); @@ -1241,7 +1251,7 @@ public AsyncCallFuture createDatadiskTemplateAsync(TemplateIn dataDiskTemplateOnStore = (TemplateObject)store.create(dataDiskTemplate); dataDiskTemplateOnStore.processEvent(ObjectInDataStoreStateMachine.Event.CreateOnlyRequested); } else { - dataDiskTemplateOnStore = (TemplateObject) imageFactory.getTemplate(parentTemplate, store); + dataDiskTemplateOnStore = (TemplateObject) imageFactory.getTemplate(parentTemplate, store, null); } try { CreateDataDiskTemplateContext context = new CreateDataDiskTemplateContext(null, dataDiskTemplateOnStore, future); @@ -1261,7 +1271,7 @@ public AsyncCallFuture createDatadiskTemplateAsync(TemplateIn } protected Void createDatadiskTemplateCallback(AsyncCallbackDispatcher callback, - CreateDataDiskTemplateContext context) { + CreateDataDiskTemplateContext context) { DataObject dataDiskTemplate = context.dataDiskTemplate; AsyncCallFuture future = context.getFuture(); CreateCmdResult result = callback.getResult(); @@ -1280,4 +1290,4 @@ protected Void createDatadiskTemplateCallback(AsyncCallbackDispatcher driverMaps; + static final ConfigKey ImageStoreAllocationAlgorithm = new ConfigKey("Advanced", String.class, "image.store.allocation.algorithm", "firstfitleastconsumed", + "firstfitleastconsumed','random' : Order in which hosts within a cluster will be considered for VM/volume allocation", true, ConfigKey.Scope.Global ); + @PostConstruct public void config() { driverMaps = new HashMap(); @@ -110,7 +119,7 @@ public List listImageCacheStores() { @Override public List listImageStoresByScope(ZoneScope scope) { - List stores = dataStoreDao.findByScope(scope); + List stores = dataStoreDao.findByZone(scope, null); List imageStores = new ArrayList(); for (ImageStoreVO store : stores) { imageStores.add(getImageStore(store.getId())); @@ -118,6 +127,24 @@ public List listImageStoresByScope(ZoneScope scope) { return imageStores; } + @Override + public List listImageStoresByScopeExcludingReadOnly(ZoneScope scope) { + String allocationAlgorithm = ImageStoreAllocationAlgorithm.value(); + + List stores = dataStoreDao.findByZone(scope, Boolean.FALSE); + List imageStores = new ArrayList(); + for (ImageStoreVO store : stores) { + imageStores.add(getImageStore(store.getId())); + } + if (allocationAlgorithm.equals("random")) { + Collections.shuffle(imageStores); + return imageStores; + } else if (allocationAlgorithm.equals("firstfitleastconsumed")) { + return orderImageStoresOnFreeCapacity(imageStores); + } + return null; + } + @Override public List listImageStoreByProvider(String provider) { List stores = dataStoreDao.findByProvider(provider); @@ -178,6 +205,31 @@ public int compare(DataStore store1, DataStore store2) { return null; } + @Override + public List orderImageStoresOnFreeCapacity(List imageStores) { + List stores = new ArrayList<>(); + if (imageStores.size() > 1) { + imageStores.sort(new Comparator() { // Sort data stores based on free capacity + @Override + public int compare(DataStore store1, DataStore store2) { + return Long.compare(_statsCollector.imageStoreCurrentFreeCapacity(store1), + _statsCollector.imageStoreCurrentFreeCapacity(store2)); + } + }); + for (DataStore imageStore : imageStores) { + // Return image store if used percentage is less then threshold value i.e. 90%. + if (_statsCollector.imageStoreHasEnoughCapacity(imageStore)) { + stores.add(imageStore); + } + } + } else if (imageStores.size() == 1) { + if (_statsCollector.imageStoreHasEnoughCapacity(imageStores.get(0))) { + stores.add(imageStores.get(0)); + } + } + return stores; + } + @Override public List listImageStoresWithFreeCapacity(List imageStores) { List stores = new ArrayList<>(); @@ -195,4 +247,14 @@ public List listImageStoresWithFreeCapacity(List imageStor } return stores; } + + @Override + public String getConfigComponentName() { + return ImageStoreProviderManager.class.getSimpleName(); + } + + @Override + public ConfigKey[] getConfigKeys() { + return new ConfigKey[] { ImageStoreAllocationAlgorithm }; + } } diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/ImageStoreImpl.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/ImageStoreImpl.java index f54673d2b61e..d4e2c056763e 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/ImageStoreImpl.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/ImageStoreImpl.java @@ -218,8 +218,8 @@ public void deleteExtractUrl(String installPath, String url, Upload.Type entityT } @Override - public List getDataDiskTemplates(DataObject obj) { - return driver.getDataDiskTemplates(obj); + public List getDataDiskTemplates(DataObject obj, String configurationId) { + return driver.getDataDiskTemplates(obj, configurationId); } @Override diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/TemplateObject.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/TemplateObject.java index 25f27a23c1ed..b7a44cd4f08a 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/TemplateObject.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/TemplateObject.java @@ -63,6 +63,7 @@ public class TemplateObject implements TemplateInfo { private DataStore dataStore; private String url; private String installPath; // temporarily set installPath before passing to resource for entries with empty installPath for object store migration case + private String deployAsIsConfiguration; // Temporarily set @Inject VMTemplateDao imageDao; @Inject @@ -80,8 +81,9 @@ protected void configure(VMTemplateVO template, DataStore dataStore) { this.dataStore = dataStore; } - public static TemplateObject getTemplate(VMTemplateVO vo, DataStore store) { + public static TemplateObject getTemplate(VMTemplateVO vo, DataStore store, String configuration) { TemplateObject to = ComponentContext.inject(TemplateObject.class); + to.deployAsIsConfiguration = configuration; to.configure(vo, store); return to; } @@ -190,7 +192,9 @@ public void processEvent(ObjectInDataStoreStateMachine.Event event, Answer answe if (answer instanceof CopyCmdAnswer) { CopyCmdAnswer cpyAnswer = (CopyCmdAnswer)answer; TemplateObjectTO newTemplate = (TemplateObjectTO)cpyAnswer.getNewData(); - VMTemplateStoragePoolVO templatePoolRef = templatePoolDao.findByPoolTemplate(getDataStore().getId(), getId()); + + String deployAsIsConfiguration = newTemplate.getDeployAsIsConfiguration(); + VMTemplateStoragePoolVO templatePoolRef = templatePoolDao.findByPoolTemplate(getDataStore().getId(), getId(), deployAsIsConfiguration); templatePoolRef.setDownloadPercent(100); setTemplateSizeIfNeeded(newTemplate, templatePoolRef); @@ -208,7 +212,9 @@ public void processEvent(ObjectInDataStoreStateMachine.Event event, Answer answe CopyCmdAnswer cpyAnswer = (CopyCmdAnswer)answer; TemplateObjectTO newTemplate = (TemplateObjectTO)cpyAnswer.getNewData(); TemplateDataStoreVO templateStoreRef = templateStoreDao.findByStoreTemplate(getDataStore().getId(), getId()); - templateStoreRef.setInstallPath(newTemplate.getPath()); + if (newTemplate.getPath() != null) { + templateStoreRef.setInstallPath(newTemplate.getPath()); + } templateStoreRef.setDownloadPercent(100); templateStoreRef.setDownloadState(Status.DOWNLOADED); templateStoreRef.setSize(newTemplate.getSize()); @@ -325,6 +331,11 @@ public Long getRefCount() { return null; } + @Override + public String getDeployAsIsConfiguration() { + return deployAsIsConfiguration; + } + @Override public DataTO getTO() { DataTO to = null; @@ -363,6 +374,14 @@ public boolean isDirectDownload() { return this.imageVO.isDirectDownload(); } + @Override + public boolean isDeployAsIs() { + if (this.imageVO == null) { + return false; + } + return this.imageVO.isDeployAsIs(); + } + public void setInstallPath(String installPath) { this.installPath = installPath; } diff --git a/engine/storage/image/src/main/resources/META-INF/cloudstack/core/spring-engine-storage-image-core-context.xml b/engine/storage/image/src/main/resources/META-INF/cloudstack/core/spring-engine-storage-image-core-context.xml index 5c7b05b756a1..805af26324bc 100644 --- a/engine/storage/image/src/main/resources/META-INF/cloudstack/core/spring-engine-storage-image-core-context.xml +++ b/engine/storage/image/src/main/resources/META-INF/cloudstack/core/spring-engine-storage-image-core-context.xml @@ -34,6 +34,10 @@ + + - + diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotObject.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotObject.java index 65d6fa52e667..f107343f0def 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotObject.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotObject.java @@ -18,12 +18,12 @@ */ package org.apache.cloudstack.storage.snapshot; +import java.util.ArrayList; import java.util.Date; +import java.util.List; import javax.inject.Inject; -import org.apache.log4j.Logger; - import org.apache.cloudstack.engine.subsystem.api.storage.DataObjectInStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; @@ -40,6 +40,7 @@ import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; import org.apache.cloudstack.storage.to.SnapshotObjectTO; +import org.apache.log4j.Logger; import com.cloud.agent.api.Answer; import com.cloud.agent.api.to.DataObjectType; @@ -129,6 +130,24 @@ public SnapshotInfo getChild() { return snapshotFactory.getSnapshot(vo.getId(), store); } + @Override + public List getChildren() { + QueryBuilder sc = QueryBuilder.create(SnapshotDataStoreVO.class); + sc.and(sc.entity().getDataStoreId(), Op.EQ, store.getId()); + sc.and(sc.entity().getRole(), Op.EQ, store.getRole()); + sc.and(sc.entity().getState(), Op.NIN, State.Destroying, State.Destroyed, State.Error); + sc.and(sc.entity().getParentSnapshotId(), Op.EQ, getId()); + List vos = sc.list(); + + List children = new ArrayList<>(); + if (vos != null) { + for (SnapshotDataStoreVO vo : vos) { + children.add(snapshotFactory.getSnapshot(vo.getSnapshotId(), DataStoreRole.Image)); + } + } + return children; + } + @Override public boolean isRevertable() { SnapshotStrategy snapshotStrategy = storageStrategyFactory.getSnapshotStrategy(snapshot, SnapshotOperation.REVERT); diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java index 727d10af1300..cfe32c2bd072 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java @@ -26,6 +26,9 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.exception.StorageUnavailableException; +import com.cloud.storage.StoragePoolStatus; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.log4j.Logger; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; @@ -89,7 +92,7 @@ public boolean configure(String name, Map params) throws Configu @Override public List allocateToPool(DiskProfile dskCh, VirtualMachineProfile vmProfile, DeploymentPlan plan, ExcludeList avoid, int returnUpTo) { List pools = select(dskCh, vmProfile, plan, avoid, returnUpTo); - return reOrder(pools, vmProfile, plan); + return reorderPools(pools, vmProfile, plan); } protected List reorderPoolsByCapacity(DeploymentPlan plan, @@ -155,7 +158,8 @@ protected List reorderPoolsByNumberOfVolumes(DeploymentPlan plan, L return reorderedPools; } - protected List reOrder(List pools, VirtualMachineProfile vmProfile, DeploymentPlan plan) { + @Override + public List reorderPools(List pools, VirtualMachineProfile vmProfile, DeploymentPlan plan) { if (pools == null) { return null; } @@ -215,6 +219,29 @@ protected boolean filter(ExcludeList avoid, StoragePool pool, DiskProfile dskCh, Volume volume = volumeDao.findById(dskCh.getVolumeId()); List requestVolumes = new ArrayList<>(); requestVolumes.add(volume); + if (dskCh.getHypervisorType() == HypervisorType.VMware) { + // Skip the parent datastore cluster, consider only child storage pools in it + if (pool.getPoolType() == Storage.StoragePoolType.DatastoreCluster && storageMgr.isStoragePoolDatastoreClusterParent(pool)) { + return false; + } + // Skip the storage pool whose parent datastore cluster is not in UP state. + if (pool.getParent() != 0L) { + StoragePoolVO datastoreCluster = storagePoolDao.findById(pool.getParent()); + if (datastoreCluster == null || (datastoreCluster != null && datastoreCluster.getStatus() != StoragePoolStatus.Up)) { + return false; + } + } + + try { + boolean isStoragePoolStoragepolicyComplaince = storageMgr.isStoragePoolComplaintWithStoragePolicy(requestVolumes, pool); + if (!isStoragePoolStoragepolicyComplaince) { + return false; + } + } catch (StorageUnavailableException e) { + s_logger.warn(String.format("Could not verify storage policy complaince against storage pool %s due to exception %s", pool.getUuid(), e.getMessage())); + return false; + } + } return storageMgr.storagePoolHasEnoughIops(requestVolumes, pool) && storageMgr.storagePoolHasEnoughSpace(requestVolumes, pool, plan.getClusterId()); } diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/DataObjectManagerImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/DataObjectManagerImpl.java index 2caa3ef3f412..777eb376cb61 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/DataObjectManagerImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/DataObjectManagerImpl.java @@ -76,7 +76,7 @@ protected DataObject waitingForCreated(DataObject dataObj, DataStore dataStore) s_logger.debug("waiting too long for template downloading, marked it as failed"); throw new CloudRuntimeException("waiting too long for template downloading, marked it as failed"); } - return objectInDataStoreMgr.get(dataObj, dataStore); + return objectInDataStoreMgr.get(dataObj, dataStore, null); } class CreateContext extends AsyncRpcContext { diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/DataStoreManagerImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/DataStoreManagerImpl.java index 51421e4cd3dd..ff6c4fb5c6a7 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/DataStoreManagerImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/DataStoreManagerImpl.java @@ -72,6 +72,11 @@ public List getImageStoresByScope(ZoneScope scope) { return imageDataStoreMgr.listImageStoresByScope(scope); } + @Override + public List getImageStoresByScopeExcludingReadOnly(ZoneScope scope) { + return imageDataStoreMgr.listImageStoresByScopeExcludingReadOnly(scope); + } + @Override public DataStore getRandomImageStore(long zoneId) { List stores = getImageStoresByScope(new ZoneScope(zoneId)); @@ -81,18 +86,35 @@ public DataStore getRandomImageStore(long zoneId) { return imageDataStoreMgr.getRandomImageStore(stores); } + @Override + public DataStore getRandomUsableImageStore(long zoneId) { + List stores = getImageStoresByScopeExcludingReadOnly(new ZoneScope(zoneId)); + if (stores == null || stores.size() == 0) { + return null; + } + return imageDataStoreMgr.getRandomImageStore(stores); + } + @Override public DataStore getImageStoreWithFreeCapacity(long zoneId) { - List stores = getImageStoresByScope(new ZoneScope(zoneId)); + List stores = getImageStoresByScopeExcludingReadOnly(new ZoneScope(zoneId)); if (stores == null || stores.size() == 0) { return null; } return imageDataStoreMgr.getImageStoreWithFreeCapacity(stores); } + @Override + public DataStore getImageStoreWithFreeCapacity(List imageStores) { + if (imageStores.isEmpty()) { + return null; + } + return imageDataStoreMgr.getImageStoreWithFreeCapacity(imageStores); + } + @Override public List listImageStoresWithFreeCapacity(long zoneId) { - List stores = getImageStoresByScope(new ZoneScope(zoneId)); + List stores = getImageStoresByScopeExcludingReadOnly(new ZoneScope(zoneId)); if (stores == null || stores.size() == 0) { return null; } diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManager.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManager.java index 8f081d3af39d..48acecab6b86 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManager.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManager.java @@ -33,11 +33,11 @@ public interface ObjectInDataStoreManager { boolean deleteIfNotReady(DataObject dataObj); - DataObject get(DataObject dataObj, DataStore store); + DataObject get(DataObject dataObj, DataStore store, String configuration); boolean update(DataObject vo, Event event) throws NoTransitionException, ConcurrentOperationException; - DataObjectInStore findObject(long objId, DataObjectType type, long dataStoreId, DataStoreRole role); + DataObjectInStore findObject(long objId, DataObjectType type, long dataStoreId, DataStoreRole role, String deployAsIsConfiguration); DataObjectInStore findObject(DataObject obj, DataStore store); diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java index 062e89a4247f..da97b22946e1 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java @@ -104,13 +104,21 @@ public ObjectInDataStoreManagerImpl() { // TODO: further investigate why an extra event is sent when it is // alreay Ready for DownloadListener stateMachines.addTransition(State.Ready, Event.OperationSuccessed, State.Ready); + // State transitions for data object migration + stateMachines.addTransition(State.Ready, Event.MigrateDataRequested, State.Migrating); + stateMachines.addTransition(State.Ready, Event.CopyRequested, State.Copying); + stateMachines.addTransition(State.Allocated, Event.MigrateDataRequested, State.Migrating); + stateMachines.addTransition(State.Migrating, Event.MigrationFailed, State.Failed); + stateMachines.addTransition(State.Migrating, Event.MigrationSucceeded, State.Destroyed); + stateMachines.addTransition(State.Migrating, Event.OperationSuccessed, State.Ready); + stateMachines.addTransition(State.Migrating, Event.OperationFailed, State.Ready); } @Override public DataObject create(DataObject obj, DataStore dataStore) { if (dataStore.getRole() == DataStoreRole.Primary) { if (obj.getType() == DataObjectType.TEMPLATE) { - VMTemplateStoragePoolVO vo = new VMTemplateStoragePoolVO(dataStore.getId(), obj.getId()); + VMTemplateStoragePoolVO vo = new VMTemplateStoragePoolVO(dataStore.getId(), obj.getId(), null); vo = templatePoolDao.persist(vo); } else if (obj.getType() == DataObjectType.SNAPSHOT) { SnapshotInfo snapshotInfo = (SnapshotInfo)obj; @@ -189,7 +197,7 @@ public DataObject create(DataObject obj, DataStore dataStore) { } } - return this.get(obj, dataStore); + return this.get(obj, dataStore, null); } @Override @@ -198,7 +206,7 @@ public boolean delete(DataObject dataObj) { DataStore dataStore = dataObj.getDataStore(); if (dataStore.getRole() == DataStoreRole.Primary) { if (dataObj.getType() == DataObjectType.TEMPLATE) { - VMTemplateStoragePoolVO destTmpltPool = templatePoolDao.findByPoolTemplate(dataStore.getId(), objId); + VMTemplateStoragePoolVO destTmpltPool = templatePoolDao.findByPoolTemplate(dataStore.getId(), objId, null); if (destTmpltPool != null) { return templatePoolDao.remove(destTmpltPool.getId()); } else { @@ -246,7 +254,7 @@ public boolean deleteIfNotReady(DataObject dataObj) { DataStore dataStore = dataObj.getDataStore(); if (dataStore.getRole() == DataStoreRole.Primary) { if (dataObj.getType() == DataObjectType.TEMPLATE) { - VMTemplateStoragePoolVO destTmpltPool = templatePoolDao.findByPoolTemplate(dataStore.getId(), objId); + VMTemplateStoragePoolVO destTmpltPool = templatePoolDao.findByPoolTemplate(dataStore.getId(), objId, null); if (destTmpltPool != null && destTmpltPool.getState() != ObjectInDataStoreStateMachine.State.Ready) { return templatePoolDao.remove(destTmpltPool.getId()); } else { @@ -325,9 +333,9 @@ public boolean update(DataObject data, Event event) throws NoTransitionException } @Override - public DataObject get(DataObject dataObj, DataStore store) { + public DataObject get(DataObject dataObj, DataStore store, String configuration) { if (dataObj.getType() == DataObjectType.TEMPLATE) { - return imageFactory.getTemplate(dataObj, store); + return imageFactory.getTemplate(dataObj, store, configuration); } else if (dataObj.getType() == DataObjectType.VOLUME) { return volumeFactory.getVolume(dataObj, store); } else if (dataObj.getType() == DataObjectType.SNAPSHOT) { @@ -339,11 +347,15 @@ public DataObject get(DataObject dataObj, DataStore store) { @Override public DataObjectInStore findObject(DataObject obj, DataStore store) { - return findObject(obj.getId(), obj.getType(), store.getId(), store.getRole()); + String deployAsIsConfiguration = null; + if (obj.getType() == DataObjectType.TEMPLATE) { + deployAsIsConfiguration = ((TemplateInfo) obj).getDeployAsIsConfiguration(); + } + return findObject(obj.getId(), obj.getType(), store.getId(), store.getRole(), deployAsIsConfiguration); } @Override - public DataObjectInStore findObject(long objId, DataObjectType type, long dataStoreId, DataStoreRole role) { + public DataObjectInStore findObject(long objId, DataObjectType type, long dataStoreId, DataStoreRole role, String deployAsIsConfiguration) { DataObjectInStore vo = null; if (role == DataStoreRole.Image || role == DataStoreRole.ImageCache) { switch (type) { @@ -358,7 +370,7 @@ public DataObjectInStore findObject(long objId, DataObjectType type, long dataSt break; } } else if (type == DataObjectType.TEMPLATE && role == DataStoreRole.Primary) { - vo = templatePoolDao.findByPoolTemplate(dataStoreId, objId); + vo = templatePoolDao.findByPoolTemplate(dataStoreId, objId, deployAsIsConfiguration); } else if (type == DataObjectType.SNAPSHOT && role == DataStoreRole.Primary) { vo = snapshotDataStoreDao.findByStoreSnapshot(role, dataStoreId, objId); } else { diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java index 6e8bdaf4b8c1..09b4b1ab3853 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java @@ -264,6 +264,27 @@ protected EndPoint findEndpointForImageStorage(DataStore store) { return RemoteHostEndPoint.getHypervisorHostEndPoint(host); } + @Override + public List findAllEndpointsForScope(DataStore store) { + Long dcId = null; + Scope storeScope = store.getScope(); + if (storeScope.getScopeType() == ScopeType.ZONE) { + dcId = storeScope.getScopeId(); + } + // find ssvm that can be used to download data to store. For zone-wide + // image store, use SSVM for that zone. For region-wide store, + // we can arbitrarily pick one ssvm to do that task + List ssAHosts = listUpAndConnectingSecondaryStorageVmHost(dcId); + if (ssAHosts == null || ssAHosts.isEmpty()) { + return null; + } + List endPoints = new ArrayList(); + for (HostVO host: ssAHosts) { + endPoints.add(RemoteHostEndPoint.getHypervisorHostEndPoint(host)); + } + return endPoints; + } + private List listUpAndConnectingSecondaryStorageVmHost(Long dcId) { QueryBuilder sc = QueryBuilder.create(HostVO.class); if (dcId != null) { @@ -333,7 +354,7 @@ public EndPoint select(DataStore store) { } } - private EndPoint getEndPointFromHostId(Long hostId) { + public EndPoint getEndPointFromHostId(Long hostId) { HostVO host = hostDao.findById(hostId); return RemoteHostEndPoint.getHypervisorHostEndPoint(host); } diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java index dec9b76dbc84..b72dd18e972d 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java @@ -20,19 +20,20 @@ import java.net.URI; import java.net.URISyntaxException; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; import java.util.ArrayList; +import java.util.Collections; import java.util.Date; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import javax.inject.Inject; -import com.cloud.agent.api.storage.OVFPropertyTO; import com.cloud.storage.Upload; -import com.cloud.storage.dao.TemplateOVFPropertiesDao; -import com.cloud.storage.TemplateOVFPropertyVO; -import com.cloud.utils.crypt.DBEncryptionUtil; -import org.apache.commons.collections.CollectionUtils; +import org.apache.cloudstack.storage.image.deployasis.DeployAsIsHelper; import org.apache.log4j.Logger; import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; @@ -47,6 +48,7 @@ import org.apache.cloudstack.framework.async.AsyncRpcContext; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.storage.command.CommandResult; +import org.apache.cloudstack.storage.command.CopyCommand; import org.apache.cloudstack.storage.command.DeleteCommand; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; @@ -54,6 +56,7 @@ import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO; import org.apache.cloudstack.storage.endpoint.DefaultEndPointSelector; +import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; import com.cloud.agent.api.storage.CreateDatadiskTemplateCommand; import com.cloud.agent.api.storage.DownloadAnswer; @@ -61,23 +64,33 @@ import com.cloud.agent.api.storage.GetDatadisksCommand; import com.cloud.agent.api.to.DataObjectType; import com.cloud.agent.api.to.DataTO; +import com.cloud.agent.api.to.DatadiskTO; import com.cloud.alert.AlertManager; +import com.cloud.configuration.Config; +import com.cloud.exception.AgentUnavailableException; +import com.cloud.exception.OperationTimedoutException; +import com.cloud.host.dao.HostDao; +import com.cloud.secstorage.CommandExecLogDao; +import com.cloud.secstorage.CommandExecLogVO; +import com.cloud.storage.StorageManager; import com.cloud.storage.VMTemplateStorageResourceAssoc; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.VMTemplateDao; -import com.cloud.storage.dao.VMTemplateDetailsDao; import com.cloud.storage.dao.VMTemplateZoneDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.storage.download.DownloadMonitor; import com.cloud.user.ResourceLimitService; import com.cloud.user.dao.AccountDao; -import com.cloud.agent.api.to.DatadiskTO; -import com.cloud.utils.net.Proxy; +import com.cloud.utils.NumbersUtil; +import com.cloud.utils.db.TransactionLegacy; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.net.Proxy; +import com.cloud.vm.dao.SecondaryStorageVmDao; public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver { - private static final Logger s_logger = Logger.getLogger(BaseImageStoreDriverImpl.class); + private static final Logger LOGGER = Logger.getLogger(BaseImageStoreDriverImpl.class); + @Inject protected VMTemplateDao _templateDao; @Inject @@ -97,15 +110,23 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver { @Inject AlertManager _alertMgr; @Inject - VMTemplateDetailsDao _templateDetailsDao; - @Inject DefaultEndPointSelector _defaultEpSelector; @Inject AccountDao _accountDao; @Inject ResourceLimitService _resourceLimitMgr; @Inject - TemplateOVFPropertiesDao templateOvfPropertiesDao; + DeployAsIsHelper deployAsIsHelper; + @Inject + HostDao hostDao; + @Inject + CommandExecLogDao _cmdExecLogDao; + @Inject + StorageManager storageMgr; + @Inject + protected SecondaryStorageVmDao _secStorageVmDao; + @Inject + AgentManager agentMgr; protected String _proxy = null; @@ -156,60 +177,37 @@ public void createAsync(DataStore dataStore, DataObject data, AsyncCompletionCal caller.setContext(context); if (data.getType() == DataObjectType.TEMPLATE) { caller.setCallback(caller.getTarget().createTemplateAsyncCallback(null, null)); - if (s_logger.isDebugEnabled()) { - s_logger.debug("Downloading template to data store " + dataStore.getId()); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Downloading template to data store " + dataStore.getId()); } _downloadMonitor.downloadTemplateToStorage(data, caller); } else if (data.getType() == DataObjectType.VOLUME) { caller.setCallback(caller.getTarget().createVolumeAsyncCallback(null, null)); - if (s_logger.isDebugEnabled()) { - s_logger.debug("Downloading volume to data store " + dataStore.getId()); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Downloading volume to data store " + dataStore.getId()); } _downloadMonitor.downloadVolumeToStorage(data, caller); } } - /** - * Persist OVF properties as template details for template with id = templateId - */ - private void persistOVFProperties(List ovfProperties, long templateId) { - List listToPersist = new ArrayList<>(); - for (OVFPropertyTO property : ovfProperties) { - if (!templateOvfPropertiesDao.existsOption(templateId, property.getKey())) { - TemplateOVFPropertyVO option = new TemplateOVFPropertyVO(templateId, property.getKey(), property.getType(), - property.getValue(), property.getQualifiers(), property.isUserConfigurable(), - property.getLabel(), property.getDescription(), property.isPassword()); - if (property.isPassword()) { - String encryptedPassword = DBEncryptionUtil.encrypt(property.getValue()); - option.setValue(encryptedPassword); - } - listToPersist.add(option); - } - } - if (CollectionUtils.isNotEmpty(listToPersist)) { - s_logger.debug("Persisting " + listToPersist.size() + " OVF properties for template " + templateId); - templateOvfPropertiesDao.saveOptions(listToPersist); - } - } - protected Void createTemplateAsyncCallback(AsyncCallbackDispatcher callback, - CreateContext context) { - if (s_logger.isDebugEnabled()) { - s_logger.debug("Performing image store createTemplate async callback"); + CreateContext context) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Performing image store createTemplate async callback"); } DownloadAnswer answer = callback.getResult(); DataObject obj = context.data; DataStore store = obj.getDataStore(); - List ovfProperties = answer.getOvfProperties(); + VMTemplateVO template = _templateDao.findById(obj.getId()); TemplateDataStoreVO tmpltStoreVO = _templateStoreDao.findByStoreTemplate(store.getId(), obj.getId()); if (tmpltStoreVO != null) { if (tmpltStoreVO.getDownloadState() == VMTemplateStorageResourceAssoc.Status.DOWNLOADED) { - if (CollectionUtils.isNotEmpty(ovfProperties)) { - persistOVFProperties(ovfProperties, obj.getId()); + if (template.isDeployAsIs()) { + deployAsIsHelper.persistTemplateDeployAsIsDetails(template.getId(), answer); } - if (s_logger.isDebugEnabled()) { - s_logger.debug("Template is already in DOWNLOADED state, ignore further incoming DownloadAnswer"); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Template is already in DOWNLOADED state, ignore further incoming DownloadAnswer"); } return null; } @@ -233,22 +231,22 @@ protected Void createTemplateAsyncCallback(AsyncCallbackDispatcher caller = context.getParentCallback(); if (answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR || - answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.ABANDONED || answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.UNKNOWN) { + answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.ABANDONED || answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.UNKNOWN) { CreateCmdResult result = new CreateCmdResult(null, null); result.setSuccess(false); result.setResult(answer.getErrorString()); caller.complete(result); String msg = "Failed to register template: " + obj.getUuid() + " with error: " + answer.getErrorString(); _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_UPLOAD_FAILED, _vmTemplateZoneDao.listByTemplateId(obj.getId()).get(0).getZoneId(), null, msg, msg); - s_logger.error(msg); + LOGGER.error(msg); } else if (answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.DOWNLOADED) { if (answer.getCheckSum() != null) { VMTemplateVO templateDaoBuilder = _templateDao.createForUpdate(); templateDaoBuilder.setChecksum(answer.getCheckSum()); _templateDao.update(obj.getId(), templateDaoBuilder); } - if (CollectionUtils.isNotEmpty(ovfProperties)) { - persistOVFProperties(ovfProperties, obj.getId()); + if (template.isDeployAsIs()) { + deployAsIsHelper.persistTemplateDeployAsIsDetails(template.getId(), answer); } CreateCmdResult result = new CreateCmdResult(null, null); @@ -258,7 +256,7 @@ protected Void createTemplateAsyncCallback(AsyncCallbackDispatcher callback, CreateContext context) { + createVolumeAsyncCallback(AsyncCallbackDispatcher callback, CreateContext context) { DownloadAnswer answer = callback.getResult(); DataObject obj = context.data; DataStore store = obj.getDataStore(); @@ -266,8 +264,8 @@ protected Void createTemplateAsyncCallback(AsyncCallbackDispatcher caller = context.getParentCallback(); if (answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR || - answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.ABANDONED || answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.UNKNOWN) { + answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.ABANDONED || answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.UNKNOWN) { CreateCmdResult result = new CreateCmdResult(null, null); result.setSuccess(false); result.setResult(answer.getErrorString()); @@ -299,7 +297,7 @@ protected Void createTemplateAsyncCallback(AsyncCallbackDispatcher callback) { + if (!canCopy(srcdata, destData)) { + return; + } + + if ((srcdata.getType() == DataObjectType.TEMPLATE && destData.getType() == DataObjectType.TEMPLATE) || + (srcdata.getType() == DataObjectType.SNAPSHOT && destData.getType() == DataObjectType.SNAPSHOT) || + (srcdata.getType() == DataObjectType.VOLUME && destData.getType() == DataObjectType.VOLUME)) { + + int nMaxExecutionMinutes = NumbersUtil.parseInt(configDao.getValue(Config.SecStorageCmdExecutionTimeMax.key()), 30); + CopyCommand cmd = new CopyCommand(srcdata.getTO(), destData.getTO(), nMaxExecutionMinutes * 60 * 1000, true); + Answer answer = null; + + // Select host endpoint such that the load is balanced out + List eps = _epSelector.findAllEndpointsForScope(srcdata.getDataStore()); + if (eps == null || eps.isEmpty()) { + String errMsg = "No remote endpoint to send command, check if host or ssvm is down?"; + LOGGER.error(errMsg); + answer = new Answer(cmd, false, errMsg); + } else { + // select endpoint with least number of commands running on them + answer = sendToLeastBusyEndpoint(eps, cmd); + } + CopyCommandResult result = new CopyCommandResult("", answer); + callback.complete(result); + } + } + + private Answer sendToLeastBusyEndpoint(List eps, CopyCommand cmd) { + Answer answer = null; + EndPoint endPoint = null; + Long epId = ssvmWithLeastMigrateJobs(); + if (epId == null) { + Collections.shuffle(eps); + endPoint = eps.get(0); + } else { + List remainingEps = eps.stream().filter(ep -> ep.getId() != epId ).collect(Collectors.toList()); + if (!remainingEps.isEmpty()) { + Collections.shuffle(remainingEps); + endPoint = remainingEps.get(0); + } else { + endPoint = _defaultEpSelector.getEndPointFromHostId(epId); + } + } + CommandExecLogVO execLog = new CommandExecLogVO(endPoint.getId(), _secStorageVmDao.findByInstanceName(hostDao.findById(endPoint.getId()).getName()).getId(), "DataMigrationCommand", 1); + Long cmdExecId = _cmdExecLogDao.persist(execLog).getId(); + String errMsg = null; + try { + answer = agentMgr.send(endPoint.getId(), cmd); + answer.setContextParam("cmd", cmdExecId.toString()); + return answer; + } catch (AgentUnavailableException e) { + errMsg = e.toString(); + LOGGER.debug("Failed to send command, due to Agent:" + endPoint.getId() + ", " + e.toString()); + } catch (OperationTimedoutException e) { + errMsg = e.toString(); + LOGGER.debug("Failed to send command, due to Agent:" + endPoint.getId() + ", " + e.toString()); + } + throw new CloudRuntimeException("Failed to send command, due to Agent:" + endPoint.getId() + ", " + errMsg); } @Override @@ -349,18 +405,18 @@ public void deleteEntityExtractUrl(DataStore store, String installPath, String u } @Override - public List getDataDiskTemplates(DataObject obj) { + public List getDataDiskTemplates(DataObject obj, String configurationId) { List dataDiskDetails = new ArrayList(); - if (s_logger.isDebugEnabled()) { - s_logger.debug("Get the data disks present in the OVA template"); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Get the data disks present in the OVA template"); } DataStore store = obj.getDataStore(); - GetDatadisksCommand cmd = new GetDatadisksCommand(obj.getTO()); + GetDatadisksCommand cmd = new GetDatadisksCommand(obj.getTO(), configurationId); EndPoint ep = _defaultEpSelector.select(store); Answer answer = null; if (ep == null) { String errMsg = "No remote endpoint to send command, check if host or ssvm is down?"; - s_logger.error(errMsg); + LOGGER.error(errMsg); answer = new Answer(cmd, false, errMsg); } else { answer = ep.sendMessage(cmd); @@ -379,14 +435,14 @@ public List getDataDiskTemplates(DataObject obj) { public Void createDataDiskTemplateAsync(TemplateInfo dataDiskTemplate, String path, String diskId, boolean bootable, long fileSize, AsyncCompletionCallback callback) { Answer answer = null; String errMsg = null; - if (s_logger.isDebugEnabled()) { - s_logger.debug("Create Datadisk template: " + dataDiskTemplate.getId()); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Create Datadisk template: " + dataDiskTemplate.getId()); } CreateDatadiskTemplateCommand cmd = new CreateDatadiskTemplateCommand(dataDiskTemplate.getTO(), path, diskId, fileSize, bootable); EndPoint ep = _defaultEpSelector.select(dataDiskTemplate.getDataStore()); if (ep == null) { errMsg = "No remote endpoint to send command, check if host or ssvm is down?"; - s_logger.error(errMsg); + LOGGER.error(errMsg); answer = new Answer(cmd, false, errMsg); } else { answer = ep.sendMessage(cmd); @@ -399,4 +455,28 @@ public Void createDataDiskTemplateAsync(TemplateInfo dataDiskTemplate, String pa callback.complete(result); return null; } + + private Integer getCopyCmdsCountToSpecificSSVM(Long ssvmId) { + return _cmdExecLogDao.getCopyCmdCountForSSVM(ssvmId); + } + + private Long ssvmWithLeastMigrateJobs() { + LOGGER.debug("Picking ssvm from the pool with least commands running on it"); + String query = "select host_id, count(*) from cmd_exec_log group by host_id order by 2 limit 1;"; + TransactionLegacy txn = TransactionLegacy.currentTxn(); + + Long epId = null; + PreparedStatement pstmt = null; + try { + pstmt = txn.prepareAutoCloseStatement(query); + ResultSet rs = pstmt.executeQuery(); + if (rs.getFetchSize() > 0) { + rs.absolute(1); + epId = (long) rs.getInt(1); + } + } catch (SQLException e) { + LOGGER.debug("SQLException caught", e); + } + return epId; + } } diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/ImageStoreDriver.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/ImageStoreDriver.java index 70f40f6f5c06..29071d8e5bb3 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/ImageStoreDriver.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/ImageStoreDriver.java @@ -37,7 +37,7 @@ public interface ImageStoreDriver extends DataStoreDriver { void deleteEntityExtractUrl(DataStore store, String installPath, String url, Upload.Type entityType); - List getDataDiskTemplates(DataObject obj); + List getDataDiskTemplates(DataObject obj, String configurationId); Void createDataDiskTemplateAsync(TemplateInfo dataDiskTemplate, String path, String diskId, boolean bootable, long fileSize, AsyncCompletionCallback callback); } diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/TemplateEntityImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/TemplateEntityImpl.java deleted file mode 100644 index b027c42a86e3..000000000000 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/TemplateEntityImpl.java +++ /dev/null @@ -1,314 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.cloudstack.storage.image; - -import java.lang.reflect.Method; -import java.util.Date; -import java.util.List; -import java.util.Map; - -import org.apache.cloudstack.engine.cloud.entity.api.TemplateEntity; -import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; -import org.apache.cloudstack.storage.image.datastore.ImageStoreInfo; - -import com.cloud.hypervisor.Hypervisor.HypervisorType; -import com.cloud.storage.Storage.ImageFormat; -import com.cloud.storage.Storage.TemplateType; -import com.cloud.template.VirtualMachineTemplate; - -public class TemplateEntityImpl implements TemplateEntity { - protected TemplateInfo templateInfo; - - @Override - public State getState() { - return templateInfo.getState(); - } - - public TemplateEntityImpl(TemplateInfo templateInfo) { - this.templateInfo = templateInfo; - } - - public ImageStoreInfo getImageDataStore() { - return (ImageStoreInfo)templateInfo.getDataStore(); - } - - public long getImageDataStoreId() { - return getImageDataStore().getImageStoreId(); - } - - public TemplateInfo getTemplateInfo() { - return templateInfo; - } - - @Override - public String getUuid() { - return templateInfo.getUuid(); - } - - @Override - public long getId() { - return templateInfo.getId(); - } - - public String getExternalId() { - // TODO Auto-generated method stub - return null; - } - - @Override - public String getCurrentState() { - // TODO Auto-generated method stub - return null; - } - - @Override - public String getDesiredState() { - // TODO Auto-generated method stub - return null; - } - - @Override - public Date getCreatedTime() { - // TODO Auto-generated method stub - return null; - } - - @Override - public Date getLastUpdatedTime() { - // TODO Auto-generated method stub - return null; - } - - @Override - public String getOwner() { - // TODO Auto-generated method stub - return null; - } - - @Override - public Map getDetails() { - // TODO Auto-generated method stub - return null; - } - - @Override - public boolean isDynamicallyScalable() { - return false; - } - - @Override - public void addDetail(String name, String value) { - // TODO Auto-generated method stub - - } - - @Override - public void delDetail(String name, String value) { - // TODO Auto-generated method stub - - } - - @Override - public void updateDetail(String name, String value) { - // TODO Auto-generated method stub - - } - - @Override - public List getApplicableActions() { - // TODO Auto-generated method stub - return null; - } - - @Override - public boolean isFeatured() { - // TODO Auto-generated method stub - return false; - } - - @Override - public boolean isPublicTemplate() { - // TODO Auto-generated method stub - return false; - } - - @Override - public boolean isExtractable() { - // TODO Auto-generated method stub - return false; - } - - @Override - public String getName() { - // TODO Auto-generated method stub - return null; - } - - @Override - public ImageFormat getFormat() { - // TODO Auto-generated method stub - return null; - } - - @Override - public boolean isRequiresHvm() { - // TODO Auto-generated method stub - return false; - } - - @Override - public String getDisplayText() { - // TODO Auto-generated method stub - return null; - } - - @Override - public boolean isEnablePassword() { - // TODO Auto-generated method stub - return false; - } - - @Override - public boolean isEnableSshKey() { - // TODO Auto-generated method stub - return false; - } - - @Override - public boolean isCrossZones() { - // TODO Auto-generated method stub - return false; - } - - @Override - public Date getCreated() { - // TODO Auto-generated method stub - return null; - } - - @Override - public long getGuestOSId() { - // TODO Auto-generated method stub - return 0; - } - - @Override - public boolean isBootable() { - // TODO Auto-generated method stub - return false; - } - - @Override - public TemplateType getTemplateType() { - // TODO Auto-generated method stub - return null; - } - - @Override - public HypervisorType getHypervisorType() { - // TODO Auto-generated method stub - return null; - } - - @Override - public int getBits() { - // TODO Auto-generated method stub - return 0; - } - - @Override - public String getUniqueName() { - // TODO Auto-generated method stub - return null; - } - - @Override - public String getUrl() { - // TODO Auto-generated method stub - return null; - } - - @Override - public String getChecksum() { - // TODO Auto-generated method stub - return null; - } - - @Override - public Long getSourceTemplateId() { - // TODO Auto-generated method stub - return null; - } - - @Override - public String getTemplateTag() { - // TODO Auto-generated method stub - return null; - } - - @Override - public long getAccountId() { - // TODO Auto-generated method stub - return 0; - } - - @Override - public long getDomainId() { - // TODO Auto-generated method stub - return 0; - } - - @Override - public long getPhysicalSize() { - // TODO Auto-generated method stub - return 0; - } - - @Override - public long getVirtualSize() { - // TODO Auto-generated method stub - return 0; - } - - @Override - public Class getEntityType() { - return VirtualMachineTemplate.class; - } - - @Override - public long getUpdatedCount() { - // TODO Auto-generated method stub - return 0; - } - - @Override - public void incrUpdatedCount() { - // TODO Auto-generated method stub - } - - @Override - public Date getUpdated() { - return null; - } - - @Override - public Long getParentTemplateId() { - return null; - } -} diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/datastore/ImageStoreProviderManager.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/datastore/ImageStoreProviderManager.java index 01f2100f77f1..7e2f720042ed 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/datastore/ImageStoreProviderManager.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/datastore/ImageStoreProviderManager.java @@ -36,6 +36,8 @@ public interface ImageStoreProviderManager { List listImageStoresByScope(ZoneScope scope); + List listImageStoresByScopeExcludingReadOnly(ZoneScope scope); + List listImageStoreByProvider(String provider); List listImageCacheStores(Scope scope); @@ -76,4 +78,6 @@ public interface ImageStoreProviderManager { * @return the list of DataStore which have free capacity */ List listImageStoresWithFreeCapacity(List imageStores); + + List orderImageStoresOnFreeCapacity(List imageStores); } diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/db/SnapshotDataStoreDaoImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/db/SnapshotDataStoreDaoImpl.java index 151b9bae8bc3..4e5210d0b3bf 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/db/SnapshotDataStoreDaoImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/db/SnapshotDataStoreDaoImpl.java @@ -27,8 +27,6 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; -import com.cloud.storage.DataStoreRole; -import com.cloud.storage.SnapshotVO; import org.apache.cloudstack.engine.subsystem.api.storage.DataObjectInStore; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.Event; @@ -39,6 +37,8 @@ import org.springframework.stereotype.Component; import com.cloud.hypervisor.Hypervisor; +import com.cloud.storage.DataStoreRole; +import com.cloud.storage.SnapshotVO; import com.cloud.storage.dao.SnapshotDao; import com.cloud.utils.db.DB; import com.cloud.utils.db.Filter; @@ -65,6 +65,7 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase stateSearch; private SearchBuilder parentSnapshotSearch; private SearchBuilder snapshotVOSearch; + private SearchBuilder snapshotCreatedSearch; public static ArrayList hypervisorsSupportingSnapshotsChaining = new ArrayList(); @@ -158,6 +159,11 @@ public boolean configure(String name, Map params) throws Configu snapshotVOSearch.and("volume_id", snapshotVOSearch.entity().getVolumeId(), SearchCriteria.Op.EQ); snapshotVOSearch.done(); + snapshotCreatedSearch = createSearchBuilder(); + snapshotCreatedSearch.and("store_id", snapshotCreatedSearch.entity().getDataStoreId(), Op.EQ); + snapshotCreatedSearch.and("created", snapshotCreatedSearch.entity().getCreated(), Op.BETWEEN); + snapshotCreatedSearch.done(); + return true; } @@ -334,6 +340,15 @@ public SnapshotDataStoreVO findBySnapshot(long snapshotId, DataStoreRole role) { return findOneBy(sc); } + @Override + public SnapshotDataStoreVO findBySourceSnapshot(long snapshotId, DataStoreRole role) { + SearchCriteria sc = snapshotSearch.create(); + sc.setParameters("snapshot_id", snapshotId); + sc.setParameters("store_role", role); + sc.setParameters("state", State.Migrating); + return findOneBy(sc); + } + @Override public List listAllByVolumeAndDataStore(long volumeId, DataStoreRole role) { SearchCriteria sc = volumeSearch.create(); @@ -462,6 +477,15 @@ public List listByState(ObjectInDataStoreStateMachine.State } @Override + public List findSnapshots(Long storeId, Date start, Date end) { + SearchCriteria sc = snapshotCreatedSearch.create(); + sc.setParameters("store_id", storeId); + if (start != null && end != null) { + sc.setParameters("created", start, end); + } + return search(sc, null); + } + public SnapshotDataStoreVO findDestroyedReferenceBySnapshot(long snapshotId, DataStoreRole role) { SearchCriteria sc = snapshotSearch.create(); sc.setParameters("snapshot_id", snapshotId); diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java index 2372e8444cc5..5a0e4eeceede 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java @@ -97,6 +97,7 @@ public boolean configure(String name, Map params) throws Configu templateSearch = createSearchBuilder(); templateSearch.and("template_id", templateSearch.entity().getTemplateId(), SearchCriteria.Op.EQ); + templateSearch.and("download_state", templateSearch.entity().getDownloadState(), SearchCriteria.Op.NEQ); templateSearch.and("destroyed", templateSearch.entity().getDestroyed(), SearchCriteria.Op.EQ); templateSearch.done(); @@ -418,6 +419,15 @@ public List listByTemplate(long templateId) { return search(sc, null); } + @Override + public List listByTemplateNotBypassed(long templateId) { + SearchCriteria sc = templateSearch.create(); + sc.setParameters("template_id", templateId); + sc.setParameters("download_state", Status.BYPASSED); + sc.setParameters("destroyed", false); + return search(sc, null); + } + @Override public TemplateDataStoreVO findByTemplateZone(long templateId, Long zoneId, DataStoreRole role) { // get all elgible image stores diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/deployasis/DeployAsIsHelper.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/deployasis/DeployAsIsHelper.java new file mode 100644 index 000000000000..303161c0c599 --- /dev/null +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/deployasis/DeployAsIsHelper.java @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.image.deployasis; + +import com.cloud.agent.api.storage.DownloadAnswer; +import com.cloud.agent.api.to.NicTO; +import com.cloud.vm.VirtualMachineProfile; + +import java.util.Map; + +public interface DeployAsIsHelper { + + void persistTemplateDeployAsIsDetails(long templateId, DownloadAnswer answer); + Map getVirtualMachineDeployAsIsProperties(VirtualMachineProfile vmId); + + String getAllocatedVirtualMachineTemplatePath(VirtualMachineProfile vm, String configuration, String destStoragePool); + String getAllocatedVirtualMachineDestinationStoragePool(VirtualMachineProfile vm); + + Map getAllocatedVirtualMachineNicsAdapterMapping(VirtualMachineProfile vm, NicTO[] nics); +} diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/deployasis/DeployAsIsHelperImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/deployasis/DeployAsIsHelperImpl.java new file mode 100644 index 000000000000..7e431b1a7296 --- /dev/null +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/deployasis/DeployAsIsHelperImpl.java @@ -0,0 +1,232 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.image.deployasis; + +import com.cloud.agent.api.storage.DownloadAnswer; +import com.cloud.agent.api.to.DataStoreTO; +import com.cloud.agent.api.to.DataTO; +import com.cloud.agent.api.to.DiskTO; +import com.cloud.agent.api.to.NicTO; +import com.cloud.agent.api.to.deployasis.OVFConfigurationTO; +import com.cloud.agent.api.to.deployasis.OVFEulaSectionTO; +import com.cloud.agent.api.to.deployasis.OVFPropertyTO; +import com.cloud.agent.api.to.deployasis.OVFVirtualHardwareItemTO; +import com.cloud.agent.api.to.deployasis.OVFVirtualHardwareSectionTO; +import com.cloud.agent.api.to.deployasis.TemplateDeployAsIsInformationTO; +import com.cloud.deployasis.DeployAsIsConstants; +import com.cloud.deployasis.TemplateDeployAsIsDetailVO; +import com.cloud.deployasis.UserVmDeployAsIsDetailVO; +import com.cloud.deployasis.dao.TemplateDeployAsIsDetailsDao; +import com.cloud.deployasis.dao.UserVmDeployAsIsDetailsDao; +import com.cloud.storage.VMTemplateStoragePoolVO; +import com.cloud.storage.Volume; +import com.cloud.storage.dao.VMTemplatePoolDao; +import com.cloud.utils.compression.CompressionUtil; +import com.cloud.utils.crypt.DBEncryptionUtil; +import com.cloud.vm.VirtualMachineProfile; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.cloud.agent.api.to.deployasis.OVFNetworkTO; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.ArrayUtils; +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; + +import javax.inject.Inject; +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Component +public class DeployAsIsHelperImpl implements DeployAsIsHelper { + + private static final Logger LOGGER = Logger.getLogger(DeployAsIsHelperImpl.class); + private static Gson gson; + + @Inject + private TemplateDeployAsIsDetailsDao templateDeployAsIsDetailsDao; + @Inject + private UserVmDeployAsIsDetailsDao userVmDeployAsIsDetailsDao; + @Inject + private PrimaryDataStoreDao storagePoolDao; + @Inject + private VMTemplatePoolDao templateStoragePoolDao; + + static { + GsonBuilder builder = new GsonBuilder(); + builder.disableHtmlEscaping(); + gson = builder.create(); + } + + public void persistTemplateDeployAsIsDetails(long templateId, DownloadAnswer answer) { + List ovfProperties = answer.getOvfProperties(); + List networkRequirements = answer.getNetworkRequirements(); + OVFVirtualHardwareSectionTO ovfHardwareSection = answer.getOvfHardwareSection(); + List eulaSections = answer.getEulaSections(); + + if (CollectionUtils.isNotEmpty(ovfProperties)) { + persistTemplateDeployAsIsInformationTOList(templateId, ovfProperties); + } + if (CollectionUtils.isNotEmpty(networkRequirements)) { + persistTemplateDeployAsIsInformationTOList(templateId, networkRequirements); + } + if (CollectionUtils.isNotEmpty(eulaSections)) { + persistTemplateDeployAsIsInformationTOList(templateId, eulaSections); + } + if (ovfHardwareSection != null) { + if (CollectionUtils.isNotEmpty(ovfHardwareSection.getConfigurations())) { + persistTemplateDeployAsIsInformationTOList(templateId, ovfHardwareSection.getConfigurations()); + } + if (CollectionUtils.isNotEmpty(ovfHardwareSection.getCommonHardwareItems())) { + persistTemplateDeployAsIsInformationTOList(templateId, ovfHardwareSection.getCommonHardwareItems()); + } + } + } + + @Override + public Map getVirtualMachineDeployAsIsProperties(VirtualMachineProfile vm) { + Map map = new HashMap<>(); + List details = userVmDeployAsIsDetailsDao.listDetails(vm.getId()); + if (CollectionUtils.isNotEmpty(details)) { + for (UserVmDeployAsIsDetailVO detail : details) { + OVFPropertyTO property = templateDeployAsIsDetailsDao.findPropertyByTemplateAndKey(vm.getTemplateId(), detail.getName()); + String value = property.isPassword() ? DBEncryptionUtil.decrypt(detail.getValue()) : detail.getValue(); + map.put(detail.getName(), value); + } + } + return map; + } + + @Override + public String getAllocatedVirtualMachineTemplatePath(VirtualMachineProfile vm, String configuration, String destStoragePool) { + StoragePoolVO storagePoolVO = storagePoolDao.findByUuid(destStoragePool); + VMTemplateStoragePoolVO tmplRef = templateStoragePoolDao.findByPoolTemplate(storagePoolVO.getId(), + vm.getTemplate().getId(), configuration); + if (tmplRef != null) { + return tmplRef.getInstallPath(); + } + return null; + } + + @Override + public String getAllocatedVirtualMachineDestinationStoragePool(VirtualMachineProfile vm) { + if (vm != null) { + if (CollectionUtils.isNotEmpty(vm.getDisks())) { + for (DiskTO disk : vm.getDisks()) { + if (disk.getType() == Volume.Type.ISO) { + continue; + } + DataTO data = disk.getData(); + if (data != null) { + DataStoreTO dataStore = data.getDataStore(); + if (dataStore != null) { + return dataStore.getUuid(); + } + } + } + } + } + return null; + } + + @Override + public Map getAllocatedVirtualMachineNicsAdapterMapping(VirtualMachineProfile vm, NicTO[] nics) { + Map map = new HashMap<>(); + List networks = templateDeployAsIsDetailsDao.listNetworkRequirementsByTemplateId(vm.getTemplateId()); + if (ArrayUtils.isNotEmpty(nics)) { + if (nics.length != networks.size()) { + String msg = "Different number of networks provided vs networks defined in deploy-as-is template"; + LOGGER.error(msg); + return map; + } + for (int i = 0; i < nics.length; i++) { + // The nic Adapter is defined on the resource sub type + map.put(nics[i].getDeviceId(), networks.get(i).getResourceSubType()); + } + } + return map; + } + + private void persistTemplateDeployAsIsInformationTOList(long templateId, + List informationTOList) { + for (TemplateDeployAsIsInformationTO informationTO : informationTOList) { + String propKey = getKeyFromInformationTO(informationTO); + if (LOGGER.isTraceEnabled()) { + LOGGER.trace(String.format("Saving property %s for template %d as detail", propKey, templateId)); + } + String propValue = null; + try { + propValue = getValueFromInformationTO(informationTO); + } catch (RuntimeException re) { + LOGGER.error("gson marshalling of property object fails: " + propKey,re); + } catch (IOException e) { + LOGGER.error("Could not decompress the license for template " + templateId, e); + } + saveTemplateDeployAsIsPropertyAttribute(templateId, propKey, propValue); + } + } + + private String getValueFromInformationTO(TemplateDeployAsIsInformationTO informationTO) throws IOException { + if (informationTO instanceof OVFEulaSectionTO) { + CompressionUtil compressionUtil = new CompressionUtil(); + byte[] compressedLicense = ((OVFEulaSectionTO) informationTO).getCompressedLicense(); + return compressionUtil.decompressByteArary(compressedLicense); + } + return gson.toJson(informationTO); + } + + private String getKeyFromInformationTO(TemplateDeployAsIsInformationTO informationTO) { + if (informationTO instanceof OVFPropertyTO) { + return DeployAsIsConstants.PROPERTY_PREFIX + ((OVFPropertyTO) informationTO).getKey(); + } else if (informationTO instanceof OVFNetworkTO) { + return DeployAsIsConstants.NETWORK_PREFIX + ((OVFNetworkTO) informationTO).getName(); + } else if (informationTO instanceof OVFConfigurationTO) { + return DeployAsIsConstants.CONFIGURATION_PREFIX + + ((OVFConfigurationTO) informationTO).getIndex() + "-" + ((OVFConfigurationTO) informationTO).getId(); + } else if (informationTO instanceof OVFVirtualHardwareItemTO) { + String key = ((OVFVirtualHardwareItemTO) informationTO).getResourceType().getName().trim().replaceAll("\\s","") + + "-" + ((OVFVirtualHardwareItemTO) informationTO).getInstanceId(); + return DeployAsIsConstants.HARDWARE_ITEM_PREFIX + key; + } else if (informationTO instanceof OVFEulaSectionTO) { + return DeployAsIsConstants.EULA_PREFIX + ((OVFEulaSectionTO) informationTO).getIndex() + + "-" + ((OVFEulaSectionTO) informationTO).getInfo(); + } + return null; + } + + private void saveTemplateDeployAsIsPropertyAttribute(long templateId, String key, String value) { + if (LOGGER.isTraceEnabled()) { + LOGGER.trace(String.format("Saving property %s for template %d as detail", key, templateId)); + } + if (templateDeployAsIsDetailsDao.findDetail(templateId,key) != null) { + LOGGER.debug(String.format("Detail '%s' existed for template %d, deleting.", key, templateId)); + templateDeployAsIsDetailsDao.removeDetail(templateId,key); + } + if (LOGGER.isTraceEnabled()) { + LOGGER.trace(String.format("Template detail for template %d to save is '%s': '%s'", templateId, key, value)); + } + TemplateDeployAsIsDetailVO detailVO = new TemplateDeployAsIsDetailVO(templateId, key, value); + LOGGER.debug("Persisting template details " + detailVO.getName() + " from OVF properties for template " + templateId); + templateDeployAsIsDetailsDao.persist(detailVO); + } + +} diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/volume/datastore/PrimaryDataStoreHelper.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/volume/datastore/PrimaryDataStoreHelper.java index 31b57080aec6..ac2c93567c33 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/volume/datastore/PrimaryDataStoreHelper.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/volume/datastore/PrimaryDataStoreHelper.java @@ -161,6 +161,13 @@ public DataStore attachCluster(DataStore store) { pool.setScope(ScopeType.CLUSTER); pool.setStatus(StoragePoolStatus.Up); this.dataStoreDao.update(pool.getId(), pool); + if(pool.getPoolType() == StoragePoolType.DatastoreCluster && pool.getParent() == 0) { + List childDatastores = dataStoreDao.listChildStoragePoolsInDatastoreCluster(pool.getId()); + for (StoragePoolVO child : childDatastores) { + child.setScope(ScopeType.CLUSTER); + this.dataStoreDao.update(child.getId(), child); + } + } return dataStoreMgr.getDataStore(store.getId(), DataStoreRole.Primary); } @@ -178,6 +185,13 @@ public DataStore attachZone(DataStore store, HypervisorType hypervisor) { pool.setHypervisor(hypervisor); pool.setStatus(StoragePoolStatus.Up); this.dataStoreDao.update(pool.getId(), pool); + if(pool.getPoolType() == StoragePoolType.DatastoreCluster && pool.getParent() == 0) { + List childDatastores = dataStoreDao.listChildStoragePoolsInDatastoreCluster(pool.getId()); + for (StoragePoolVO child : childDatastores) { + child.setScope(ScopeType.ZONE); + this.dataStoreDao.update(child.getId(), child); + } + } return dataStoreMgr.getDataStore(store.getId(), DataStoreRole.Primary); } diff --git a/engine/storage/src/main/resources/META-INF/cloudstack/core/spring-engine-storage-core-context.xml b/engine/storage/src/main/resources/META-INF/cloudstack/core/spring-engine-storage-core-context.xml index 33385b5ae001..5cecb224125c 100644 --- a/engine/storage/src/main/resources/META-INF/cloudstack/core/spring-engine-storage-core-context.xml +++ b/engine/storage/src/main/resources/META-INF/cloudstack/core/spring-engine-storage-core-context.xml @@ -72,4 +72,6 @@ + + diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java index 81966784be05..18a7f3c48903 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java @@ -218,12 +218,12 @@ public boolean exists(DataObject data) { } @Override - public TemplateInfo getTemplate(long templateId) { - VMTemplateStoragePoolVO template = templatePoolDao.findByPoolTemplate(getId(), templateId); + public TemplateInfo getTemplate(long templateId, String configuration) { + VMTemplateStoragePoolVO template = templatePoolDao.findByPoolTemplate(getId(), templateId, configuration); if (template == null || template.getState() != ObjectInDataStoreStateMachine.State.Ready) { return null; } - return imageDataFactory.getTemplate(templateId, this); + return imageDataFactory.getTemplateOnPrimaryStorage(templateId, this, configuration); } @Override @@ -241,6 +241,11 @@ public boolean isManaged() { return pdsv.isManaged(); } + @Override + public Long getParent() { + return pdsv.getParent(); + } + private boolean canCloneVolume() { return Boolean.valueOf(getDriver().getCapabilities().get(DataStoreCapabilities.CAN_CREATE_VOLUME_FROM_VOLUME.toString())); } @@ -259,18 +264,26 @@ private boolean canCloneVolume() { */ @Override public DataObject create(DataObject dataObject) { - return create(dataObject, true); + return create(dataObject, true, null); + } + + @Override + public DataObject create(DataObject dataObject, String configuration) { + return create(dataObject, true, configuration); } /** * Please read the comment for the create(DataObject) method if you are planning on passing in "false" for createEntryInTempSpoolRef. + * + * The parameter configuration allows storing multiple configurations of the + * base template appliance in primary storage (VMware supported) - null by default or no configurations */ @Override - public DataObject create(DataObject obj, boolean createEntryInTempSpoolRef) { + public DataObject create(DataObject obj, boolean createEntryInTempSpoolRef, String configuration) { // create template on primary storage if (obj.getType() == DataObjectType.TEMPLATE && (!isManaged() || (createEntryInTempSpoolRef && canCloneVolume()))) { try { - String templateIdPoolIdString = "templateId:" + obj.getId() + "poolId:" + getId(); + String templateIdPoolIdString = "templateId:" + obj.getId() + "poolId:" + getId() + "conf:" + configuration; VMTemplateStoragePoolVO templateStoragePoolRef; GlobalLock lock = GlobalLock.getInternLock(templateIdPoolIdString); if (!lock.lock(5)) { @@ -278,20 +291,20 @@ public DataObject create(DataObject obj, boolean createEntryInTempSpoolRef) { return null; } try { - templateStoragePoolRef = templatePoolDao.findByPoolTemplate(getId(), obj.getId()); + templateStoragePoolRef = templatePoolDao.findByPoolTemplate(getId(), obj.getId(), configuration); if (templateStoragePoolRef == null) { if (s_logger.isDebugEnabled()) { s_logger.debug("Not found (" + templateIdPoolIdString + ") in template_spool_ref, persisting it"); } - templateStoragePoolRef = new VMTemplateStoragePoolVO(getId(), obj.getId()); + templateStoragePoolRef = new VMTemplateStoragePoolVO(getId(), obj.getId(), configuration); templateStoragePoolRef = templatePoolDao.persist(templateStoragePoolRef); } } catch (Throwable t) { if (s_logger.isDebugEnabled()) { s_logger.debug("Failed to insert (" + templateIdPoolIdString + ") to template_spool_ref", t); } - templateStoragePoolRef = templatePoolDao.findByPoolTemplate(getId(), obj.getId()); + templateStoragePoolRef = templatePoolDao.findByPoolTemplate(getId(), obj.getId(), configuration); if (templateStoragePoolRef == null) { throw new CloudRuntimeException("Failed to create template storage pool entry"); } else { @@ -316,7 +329,7 @@ public DataObject create(DataObject obj, boolean createEntryInTempSpoolRef) { } } - return objectInStoreMgr.get(obj, this); + return objectInStoreMgr.get(obj, this, configuration); } @Override diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/provider/DefaultHostListener.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/provider/DefaultHostListener.java index 64533d54d2f8..cb4ff74eed63 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/provider/DefaultHostListener.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/provider/DefaultHostListener.java @@ -18,28 +18,34 @@ */ package org.apache.cloudstack.storage.datastore.provider; -import java.util.List; - -import javax.inject.Inject; - -import org.apache.log4j.Logger; - -import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; -import org.apache.cloudstack.engine.subsystem.api.storage.HypervisorHostListener; -import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; - import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; import com.cloud.agent.api.ModifyStoragePoolAnswer; import com.cloud.agent.api.ModifyStoragePoolCommand; +import com.cloud.agent.api.StoragePoolInfo; import com.cloud.alert.AlertManager; import com.cloud.exception.StorageConflictException; import com.cloud.storage.DataStoreRole; +import com.cloud.storage.Storage; import com.cloud.storage.StoragePool; import com.cloud.storage.StoragePoolHostVO; +import com.cloud.storage.StoragePoolStatus; import com.cloud.storage.dao.StoragePoolHostDao; +import com.cloud.storage.dao.StoragePoolTagsDao; import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.engine.subsystem.api.storage.HypervisorHostListener; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailVO; +import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.commons.lang.StringUtils; +import org.apache.log4j.Logger; + +import javax.inject.Inject; +import java.util.HashMap; +import java.util.List; +import java.util.Map; public class DefaultHostListener implements HypervisorHostListener { private static final Logger s_logger = Logger.getLogger(DefaultHostListener.class); @@ -53,6 +59,10 @@ public class DefaultHostListener implements HypervisorHostListener { StoragePoolHostDao storagePoolHostDao; @Inject PrimaryDataStoreDao primaryStoreDao; + @Inject + StoragePoolDetailsDao storagePoolDetailsDao; + @Inject + StoragePoolTagsDao storagePoolTagsDao; @Override public boolean hostAdded(long hostId) { @@ -90,7 +100,54 @@ public boolean hostConnect(long hostId, long poolId) throws StorageConflictExcep } } } + StoragePoolVO poolVO = this.primaryStoreDao.findById(poolId); + updateStoragePoolHostVOAndDetails(poolVO, hostId, mspAnswer); + + if (pool.getPoolType() == Storage.StoragePoolType.DatastoreCluster) { + for (ModifyStoragePoolAnswer childDataStoreAnswer : ((ModifyStoragePoolAnswer) answer).getDatastoreClusterChildren()) { + StoragePoolInfo childStoragePoolInfo = childDataStoreAnswer.getPoolInfo(); + StoragePoolVO dataStoreVO = primaryStoreDao.findPoolByUUID(childStoragePoolInfo.getUuid()); + if (dataStoreVO != null) { + continue; + } + dataStoreVO = new StoragePoolVO(); + dataStoreVO.setStorageProviderName(poolVO.getStorageProviderName()); + dataStoreVO.setHostAddress(childStoragePoolInfo.getHost()); + dataStoreVO.setPoolType(Storage.StoragePoolType.PreSetup); + dataStoreVO.setPath(childStoragePoolInfo.getHostPath()); + dataStoreVO.setPort(poolVO.getPort()); + dataStoreVO.setName(childStoragePoolInfo.getName()); + dataStoreVO.setUuid(childStoragePoolInfo.getUuid()); + dataStoreVO.setDataCenterId(poolVO.getDataCenterId()); + dataStoreVO.setPodId(poolVO.getPodId()); + dataStoreVO.setClusterId(poolVO.getClusterId()); + dataStoreVO.setStatus(StoragePoolStatus.Up); + dataStoreVO.setUserInfo(poolVO.getUserInfo()); + dataStoreVO.setManaged(poolVO.isManaged()); + dataStoreVO.setCapacityIops(poolVO.getCapacityIops()); + dataStoreVO.setCapacityBytes(childDataStoreAnswer.getPoolInfo().getCapacityBytes()); + dataStoreVO.setUsedBytes(childDataStoreAnswer.getPoolInfo().getCapacityBytes() - childDataStoreAnswer.getPoolInfo().getAvailableBytes()); + dataStoreVO.setHypervisor(poolVO.getHypervisor()); + dataStoreVO.setScope(poolVO.getScope()); + dataStoreVO.setParent(poolVO.getId()); + + Map details = new HashMap<>(); + if(StringUtils.isNotEmpty(childDataStoreAnswer.getPoolType())) { + details.put("pool_type", childDataStoreAnswer.getPoolType()); + } + + List storageTags = storagePoolTagsDao.getStoragePoolTags(poolId); + primaryStoreDao.persist(dataStoreVO, details, storageTags); + + updateStoragePoolHostVOAndDetails(dataStoreVO, hostId, childDataStoreAnswer); + } + } + s_logger.info("Connection established between storage pool " + pool + " and host " + hostId); + return true; + } + + private void updateStoragePoolHostVOAndDetails(StoragePool pool, long hostId, ModifyStoragePoolAnswer mspAnswer) { StoragePoolHostVO poolHost = storagePoolHostDao.findByPoolHost(pool.getId(), hostId); if (poolHost == null) { poolHost = new StoragePoolHostVO(pool.getId(), hostId, mspAnswer.getPoolInfo().getLocalPath().replaceAll("//", "/")); @@ -99,13 +156,17 @@ public boolean hostConnect(long hostId, long poolId) throws StorageConflictExcep poolHost.setLocalPath(mspAnswer.getPoolInfo().getLocalPath().replaceAll("//", "/")); } - StoragePoolVO poolVO = this.primaryStoreDao.findById(poolId); + StoragePoolVO poolVO = this.primaryStoreDao.findById(pool.getId()); poolVO.setUsedBytes(mspAnswer.getPoolInfo().getCapacityBytes() - mspAnswer.getPoolInfo().getAvailableBytes()); poolVO.setCapacityBytes(mspAnswer.getPoolInfo().getCapacityBytes()); + if(StringUtils.isNotEmpty(mspAnswer.getPoolType())) { + StoragePoolDetailVO poolType = storagePoolDetailsDao.findDetail(pool.getId(), "pool_type"); + if (poolType == null) { + StoragePoolDetailVO storagePoolDetailVO = new StoragePoolDetailVO(pool.getId(), "pool_type", mspAnswer.getPoolType(), false); + storagePoolDetailsDao.persist(storagePoolDetailVO); + } + } primaryStoreDao.update(pool.getId(), poolVO); - - s_logger.info("Connection established between storage pool " + pool + " and host " + hostId); - return true; } @Override diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java index 690a1124402d..45509c51c434 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java @@ -21,6 +21,11 @@ import javax.inject.Inject; import com.cloud.storage.MigrationOptions; +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.VolumeDetailVO; +import com.cloud.storage.dao.VMTemplateDao; +import com.cloud.storage.dao.VolumeDetailsDao; +import com.cloud.vm.VmDetailConstants; import org.apache.log4j.Logger; import org.apache.cloudstack.engine.subsystem.api.storage.DataObjectInStore; @@ -72,6 +77,10 @@ public class VolumeObject implements VolumeInfo { VMInstanceDao vmInstanceDao; @Inject DiskOfferingDao diskOfferingDao; + @Inject + VMTemplateDao templateDao; + @Inject + VolumeDetailsDao volumeDetailsDao; private Object payload; private MigrationOptions migrationOptions; private boolean directDownload; @@ -358,7 +367,7 @@ public String getUri() { if (dataStore == null) { throw new CloudRuntimeException("datastore must be set before using this object"); } - DataObjectInStore obj = objectInStoreMgr.findObject(volumeVO.getId(), DataObjectType.VOLUME, dataStore.getId(), dataStore.getRole()); + DataObjectInStore obj = objectInStoreMgr.findObject(volumeVO.getId(), DataObjectType.VOLUME, dataStore.getId(), dataStore.getRole(), null); if (obj.getState() != ObjectInDataStoreStateMachine.State.Ready) { return dataStore.getUri() + "&" + EncodingType.OBJTYPE + "=" + DataObjectType.VOLUME + "&" + EncodingType.SIZE + "=" + volumeVO.getSize() + "&" + EncodingType.NAME + "=" + volumeVO.getName(); @@ -435,6 +444,18 @@ public void processEvent(ObjectInDataStoreStateMachine.Event event) { } } + @Override + public boolean isDeployAsIs() { + VMTemplateVO template = templateDao.findById(getTemplateId()); + return template != null && template.isDeployAsIs(); + } + + @Override + public String getDeployAsIsConfiguration() { + VolumeDetailVO detail = volumeDetailsDao.findDetail(getId(), VmDetailConstants.DEPLOY_AS_IS_CONFIGURATION); + return detail != null ? detail.getValue() : null; + } + @Override public void processEventOnly(ObjectInDataStoreStateMachine.Event event) { try { diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java index 77413ad6c2b6..3ccb7be8f606 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java @@ -121,6 +121,8 @@ import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VirtualMachine; +import static com.cloud.storage.resource.StorageProcessor.REQUEST_TEMPLATE_RELOAD; + @Component public class VolumeServiceImpl implements VolumeService { private static final Logger s_logger = Logger.getLogger(VolumeServiceImpl.class); @@ -458,7 +460,7 @@ private class ManagedCreateBaseImageContext extends AsyncRpcContext { private final AsyncCallFuture _future; public ManagedCreateBaseImageContext(AsyncCompletionCallback callback, VolumeInfo volumeInfo, PrimaryDataStore primaryDatastore, TemplateInfo templateInfo, - AsyncCallFuture future) { + AsyncCallFuture future) { super(callback); _volumeInfo = volumeInfo; @@ -493,7 +495,7 @@ class CreateBaseImageContext extends AsyncRpcContext { long templatePoolId; public CreateBaseImageContext(AsyncCompletionCallback callback, VolumeInfo volume, PrimaryDataStore datastore, TemplateInfo srcTemplate, AsyncCallFuture future, - DataObject destObj, long templatePoolId) { + DataObject destObj, long templatePoolId) { super(callback); this.volume = volume; this.dataStore = datastore; @@ -530,7 +532,7 @@ private TemplateInfo waitForTemplateDownloaded(PrimaryDataStore store, TemplateI int sleepTime = 120; int tries = storagePoolMaxWaitSeconds / sleepTime; while (tries > 0) { - TemplateInfo tmpl = store.getTemplate(template.getId()); + TemplateInfo tmpl = store.getTemplate(template.getId(), null); if (tmpl != null) { return tmpl; } @@ -546,9 +548,10 @@ private TemplateInfo waitForTemplateDownloaded(PrimaryDataStore store, TemplateI @DB protected void createBaseImageAsync(VolumeInfo volume, PrimaryDataStore dataStore, TemplateInfo template, AsyncCallFuture future) { - DataObject templateOnPrimaryStoreObj = dataStore.create(template); + String deployAsIsConfiguration = volume.getDeployAsIsConfiguration(); + DataObject templateOnPrimaryStoreObj = dataStore.create(template, deployAsIsConfiguration); - VMTemplateStoragePoolVO templatePoolRef = _tmpltPoolDao.findByPoolTemplate(dataStore.getId(), template.getId()); + VMTemplateStoragePoolVO templatePoolRef = _tmpltPoolDao.findByPoolTemplate(dataStore.getId(), template.getId(), deployAsIsConfiguration); if (templatePoolRef == null) { throw new CloudRuntimeException("Failed to find template " + template.getUniqueName() + " in storage pool " + dataStore.getId()); } else { @@ -571,8 +574,8 @@ protected void createBaseImageAsync(VolumeInfo volume, PrimaryDataStore dataStor if (s_logger.isDebugEnabled()) { s_logger.info("Unable to acquire lock on VMTemplateStoragePool " + templatePoolRefId); } - templatePoolRef = _tmpltPoolDao.findByPoolTemplate(dataStore.getId(), template.getId()); - if (templatePoolRef != null && templatePoolRef.getState() == ObjectInDataStoreStateMachine.State.Ready) { + templatePoolRef = _tmpltPoolDao.findByPoolTemplate(dataStore.getId(), template.getId(), deployAsIsConfiguration); + if (templatePoolRef != null && templatePoolRef.getState() == ObjectInDataStoreStateMachine.State.Ready && !template.isDeployAsIs()) { s_logger.info( "Unable to acquire lock on VMTemplateStoragePool " + templatePoolRefId + ", But Template " + template.getUniqueName() + " is already copied to primary storage, skip copying"); createVolumeFromBaseImageAsync(volume, templateOnPrimaryStoreObj, dataStore, future); @@ -585,7 +588,7 @@ protected void createBaseImageAsync(VolumeInfo volume, PrimaryDataStore dataStor s_logger.info("lock is acquired for VMTemplateStoragePool " + templatePoolRefId); } try { - if (templatePoolRef.getState() == ObjectInDataStoreStateMachine.State.Ready) { + if (templatePoolRef.getState() == ObjectInDataStoreStateMachine.State.Ready && !template.isDeployAsIs()) { s_logger.info("Template " + template.getUniqueName() + " is already copied to primary storage, skip copying"); createVolumeFromBaseImageAsync(volume, templateOnPrimaryStoreObj, dataStore, future); return; @@ -595,7 +598,7 @@ protected void createBaseImageAsync(VolumeInfo volume, PrimaryDataStore dataStor } catch (Throwable e) { s_logger.debug("failed to create template on storage", e); templateOnPrimaryStoreObj.processEvent(Event.OperationFailed); - dataStore.create(template); // make sure that template_spool_ref entry is still present so that the second thread can acquire the lock + dataStore.create(template, deployAsIsConfiguration); // make sure that template_spool_ref entry is still present so that the second thread can acquire the lock VolumeApiResult result = new VolumeApiResult(volume); result.setResult(e.toString()); future.complete(result); @@ -705,14 +708,16 @@ private class CreateVolumeFromBaseImageContext extends AsyncRpcContext { private final AsyncCallFuture future; private final DataObject templateOnStore; private final SnapshotInfo snapshot; + private final String deployAsIsConfiguration; public CreateVolumeFromBaseImageContext(AsyncCompletionCallback callback, DataObject vo, DataStore primaryStore, DataObject templateOnStore, AsyncCallFuture future, - SnapshotInfo snapshot) { + SnapshotInfo snapshot, String configuration) { super(callback); this.vo = vo; this.future = future; this.templateOnStore = templateOnStore; this.snapshot = snapshot; + this.deployAsIsConfiguration = configuration; } public AsyncCallFuture getFuture() { @@ -722,15 +727,16 @@ public AsyncCallFuture getFuture() { @DB protected void createVolumeFromBaseImageAsync(VolumeInfo volume, DataObject templateOnPrimaryStore, PrimaryDataStore pd, AsyncCallFuture future) { - DataObject volumeOnPrimaryStorage = pd.create(volume); + DataObject volumeOnPrimaryStorage = pd.create(volume, volume.getDeployAsIsConfiguration()); volumeOnPrimaryStorage.processEvent(Event.CreateOnlyRequested); - CreateVolumeFromBaseImageContext context = new CreateVolumeFromBaseImageContext(null, volumeOnPrimaryStorage, pd, templateOnPrimaryStore, future, null); + CreateVolumeFromBaseImageContext context = new CreateVolumeFromBaseImageContext(null, volumeOnPrimaryStorage, pd, templateOnPrimaryStore, future, null, volume.getDeployAsIsConfiguration()); AsyncCallbackDispatcher caller = AsyncCallbackDispatcher.create(this); caller.setCallback(caller.getTarget().createVolumeFromBaseImageCallBack(null, null)); caller.setContext(context); motionSrv.copyAsync(context.templateOnStore, volumeOnPrimaryStorage, caller); + return; } @@ -740,6 +746,7 @@ protected Void createVolumeFromBaseImageCallBack(AsyncCallbackDispatcher createTemplateFuture = new AsyncCallFuture<>(); - TemplateInfo templateOnPrimary = (TemplateInfo)destPrimaryDataStore.create(srcTemplateInfo); + TemplateInfo templateOnPrimary = (TemplateInfo)destPrimaryDataStore.create(srcTemplateInfo, srcTemplateInfo.getDeployAsIsConfiguration()); - VMTemplateStoragePoolVO templatePoolRef = _tmpltPoolDao.findByPoolTemplate(destPrimaryDataStore.getId(), templateOnPrimary.getId()); + VMTemplateStoragePoolVO templatePoolRef = _tmpltPoolDao.findByPoolTemplate(destPrimaryDataStore.getId(), templateOnPrimary.getId(), srcTemplateInfo.getDeployAsIsConfiguration()); if (templatePoolRef == null) { throw new CloudRuntimeException("Failed to find template " + srcTemplateInfo.getUniqueName() + " in storage pool " + destPrimaryDataStore.getId()); @@ -861,7 +868,7 @@ private TemplateInfo createManagedTemplateVolume(TemplateInfo srcTemplateInfo, P * @param destHost The host that we will use for the copy */ private void copyTemplateToManagedTemplateVolume(TemplateInfo srcTemplateInfo, TemplateInfo templateOnPrimary, VMTemplateStoragePoolVO templatePoolRef, PrimaryDataStore destPrimaryDataStore, - Host destHost) { + Host destHost) { AsyncCallFuture copyTemplateFuture = new AsyncCallFuture<>(); int storagePoolMaxWaitSeconds = NumbersUtil.parseInt(configDao.getValue(Config.StoragePoolMaxWaitSeconds.key()), 3600); long templatePoolRefId = templatePoolRef.getId(); @@ -986,7 +993,7 @@ private void sendModifyTargetsCommand(ModifyTargetsCommand cmd, long hostId) { * @param future For async */ private void createManagedVolumeCloneTemplateAsync(VolumeInfo volumeInfo, TemplateInfo templateOnPrimary, PrimaryDataStore destPrimaryDataStore, AsyncCallFuture future) { - VMTemplateStoragePoolVO templatePoolRef = _tmpltPoolDao.findByPoolTemplate(destPrimaryDataStore.getId(), templateOnPrimary.getId()); + VMTemplateStoragePoolVO templatePoolRef = _tmpltPoolDao.findByPoolTemplate(destPrimaryDataStore.getId(), templateOnPrimary.getId(), volumeInfo.getDeployAsIsConfiguration()); if (templatePoolRef == null) { throw new CloudRuntimeException("Failed to find template " + templateOnPrimary.getUniqueName() + " in storage pool " + destPrimaryDataStore.getId()); @@ -1000,7 +1007,7 @@ private void createManagedVolumeCloneTemplateAsync(VolumeInfo volumeInfo, Templa try { volumeInfo.processEvent(Event.CreateOnlyRequested); - CreateVolumeFromBaseImageContext context = new CreateVolumeFromBaseImageContext<>(null, volumeInfo, destPrimaryDataStore, templateOnPrimary, future, null); + CreateVolumeFromBaseImageContext context = new CreateVolumeFromBaseImageContext<>(null, volumeInfo, destPrimaryDataStore, templateOnPrimary, future, null, volumeInfo.getDeployAsIsConfiguration()); AsyncCallbackDispatcher caller = AsyncCallbackDispatcher.create(this); @@ -1021,7 +1028,7 @@ private void createManagedVolumeCopyTemplateAsync(VolumeInfo volumeInfo, Primary try { // Create a volume on managed storage. - TemplateInfo destTemplateInfo = (TemplateInfo)primaryDataStore.create(srcTemplateInfo, false); + TemplateInfo destTemplateInfo = (TemplateInfo)primaryDataStore.create(srcTemplateInfo, false, volumeInfo.getDeployAsIsConfiguration()); AsyncCallFuture createVolumeFuture = createVolumeAsync(volumeInfo, primaryDataStore); VolumeApiResult createVolumeResult = createVolumeFuture.get(); @@ -1109,7 +1116,7 @@ public AsyncCallFuture createManagedStorageVolumeFromTemplateAs if (storageCanCloneVolume && computeSupportsVolumeClone) { s_logger.debug("Storage " + destDataStoreId + " can support cloning using a cached template and compute side is OK with volume cloning."); - TemplateInfo templateOnPrimary = destPrimaryDataStore.getTemplate(srcTemplateInfo.getId()); + TemplateInfo templateOnPrimary = destPrimaryDataStore.getTemplate(srcTemplateInfo.getId(), null); if (templateOnPrimary == null) { templateOnPrimary = createManagedTemplateVolume(srcTemplateInfo, destPrimaryDataStore); @@ -1120,7 +1127,7 @@ public AsyncCallFuture createManagedStorageVolumeFromTemplateAs } // Copy the template to the template volume. - VMTemplateStoragePoolVO templatePoolRef = _tmpltPoolDao.findByPoolTemplate(destPrimaryDataStore.getId(), templateOnPrimary.getId()); + VMTemplateStoragePoolVO templatePoolRef = _tmpltPoolDao.findByPoolTemplate(destPrimaryDataStore.getId(), templateOnPrimary.getId(), null); if (templatePoolRef == null) { throw new CloudRuntimeException("Failed to find template " + srcTemplateInfo.getUniqueName() + " in storage pool " + destPrimaryDataStore.getId()); @@ -1196,8 +1203,8 @@ private HostVO getHost(Long zoneId, HypervisorType hypervisorType, boolean compu @Override public AsyncCallFuture createVolumeFromTemplateAsync(VolumeInfo volume, long dataStoreId, TemplateInfo template) { PrimaryDataStore pd = dataStoreMgr.getPrimaryDataStore(dataStoreId); - TemplateInfo templateOnPrimaryStore = pd.getTemplate(template.getId()); - AsyncCallFuture future = new AsyncCallFuture(); + TemplateInfo templateOnPrimaryStore = pd.getTemplate(template.getId(), volume.getDeployAsIsConfiguration()); + AsyncCallFuture future = new AsyncCallFuture<>(); if (templateOnPrimaryStore == null) { createBaseImageAsync(volume, pd, template, future); @@ -1234,7 +1241,7 @@ public AsyncCallFuture createVolumeFromSnapshot(VolumeInfo volu volumeOnStore.processEvent(Event.CreateOnlyRequested); _volumeDetailsDao.addDetail(volume.getId(), SNAPSHOT_ID, Long.toString(snapshot.getId()), false); - CreateVolumeFromBaseImageContext context = new CreateVolumeFromBaseImageContext(null, volume, store, volumeOnStore, future, snapshot); + CreateVolumeFromBaseImageContext context = new CreateVolumeFromBaseImageContext(null, volume, store, volumeOnStore, future, snapshot, null); AsyncCallbackDispatcher caller = AsyncCallbackDispatcher.create(this); caller.setCallback(caller.getTarget().createVolumeFromSnapshotCallback(null, null)).setContext(context); motionSrv.copyAsync(snapshot, volumeOnStore, caller); @@ -2125,4 +2132,4 @@ public void unmanageVolume(long volumeId) { volDao.remove(vol.getId()); } } -} +} \ No newline at end of file diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java index 6ca698b7589a..7dd73435093d 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java @@ -242,7 +242,9 @@ public long countPendingJobs(String havingInfo, String... cmds) { SearchCriteria sc = asyncJobTypeSearch.create(); sc.setParameters("status", JobInfo.Status.IN_PROGRESS); sc.setParameters("job_cmd", (Object[])cmds); - sc.setParameters("job_info", "%" + havingInfo + "%"); + if (havingInfo != null) { + sc.setParameters("job_info", "%" + havingInfo + "%"); + } List results = customSearch(sc, null); return results.get(0); } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index b792ff22204d..cc47c5532834 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -46,6 +46,7 @@ import org.apache.cloudstack.agent.directdownload.NfsDirectDownloadCommand; import org.apache.cloudstack.storage.command.AttachAnswer; import org.apache.cloudstack.storage.command.AttachCommand; +import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplainceCommand; import org.apache.cloudstack.storage.command.CopyCmdAnswer; import org.apache.cloudstack.storage.command.CopyCommand; import org.apache.cloudstack.storage.command.CreateObjectAnswer; @@ -1833,4 +1834,10 @@ protected boolean isEnoughSpaceForDownloadTemplateOnTemporaryLocation(Long templ } return availableBytes >= templateSize; } + + @Override + public Answer CheckDataStoreStoragePolicyComplaince(CheckDataStoreStoragePolicyComplainceCommand cmd) { + s_logger.info("'CheckDataStoreStoragePolicyComplainceCommand' not currently applicable for KVMStorageProcessor"); + return new Answer(cmd,false,"Not currently applicable for KVMStorageProcessor"); + } } diff --git a/plugins/hypervisors/ovm3/src/main/java/com/cloud/hypervisor/ovm3/resources/Ovm3StorageProcessor.java b/plugins/hypervisors/ovm3/src/main/java/com/cloud/hypervisor/ovm3/resources/Ovm3StorageProcessor.java index 7915586fca3f..dd58bb573d94 100644 --- a/plugins/hypervisors/ovm3/src/main/java/com/cloud/hypervisor/ovm3/resources/Ovm3StorageProcessor.java +++ b/plugins/hypervisors/ovm3/src/main/java/com/cloud/hypervisor/ovm3/resources/Ovm3StorageProcessor.java @@ -24,6 +24,7 @@ import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand; import org.apache.cloudstack.storage.command.AttachAnswer; import org.apache.cloudstack.storage.command.AttachCommand; +import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplainceCommand; import org.apache.cloudstack.storage.command.CopyCmdAnswer; import org.apache.cloudstack.storage.command.CopyCommand; import org.apache.cloudstack.storage.command.CreateObjectAnswer; @@ -826,6 +827,12 @@ public Answer handleDownloadTemplateToPrimaryStorage(DirectDownloadCommand cmd) return null; } + @Override + public Answer CheckDataStoreStoragePolicyComplaince(CheckDataStoreStoragePolicyComplainceCommand cmd) { + LOGGER.info("'CheckDataStoreStoragePolicyComplainceCommand' not applicable used for Ovm3StorageProcessor"); + return new Answer(cmd,false,"Not applicable used for Ovm3StorageProcessor"); + } + @Override public Answer copyVolumeFromPrimaryToPrimary(CopyCommand cmd) { return null; diff --git a/plugins/hypervisors/simulator/src/main/java/com/cloud/resource/SimulatorStorageProcessor.java b/plugins/hypervisors/simulator/src/main/java/com/cloud/resource/SimulatorStorageProcessor.java index e4ef4dfc1f63..16579bc3886e 100644 --- a/plugins/hypervisors/simulator/src/main/java/com/cloud/resource/SimulatorStorageProcessor.java +++ b/plugins/hypervisors/simulator/src/main/java/com/cloud/resource/SimulatorStorageProcessor.java @@ -31,6 +31,7 @@ import org.apache.cloudstack.storage.command.CopyCommand; import org.apache.cloudstack.storage.command.CreateObjectAnswer; import org.apache.cloudstack.storage.command.CreateObjectCommand; +import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplainceCommand; import org.apache.cloudstack.storage.command.DeleteCommand; import org.apache.cloudstack.storage.command.DettachAnswer; import org.apache.cloudstack.storage.command.DettachCommand; @@ -269,4 +270,9 @@ public Answer forgetObject(ForgetObjectCmd cmd) { public Answer copyVolumeFromPrimaryToPrimary(CopyCommand cmd) { return null; } + + @Override + public Answer CheckDataStoreStoragePolicyComplaince(CheckDataStoreStoragePolicyComplainceCommand cmd) { + return new Answer(cmd, true, null); + } } diff --git a/plugins/hypervisors/vmware/pom.xml b/plugins/hypervisors/vmware/pom.xml index f5489488f355..c4dbb1d81b66 100644 --- a/plugins/hypervisors/vmware/pom.xml +++ b/plugins/hypervisors/vmware/pom.xml @@ -72,5 +72,11 @@ wsdl4j wsdl4j + + com.cloud.com.vmware + vmware-pbm + ${cs.vmware.api.version} + compile + diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java index 740b66844b54..d48a5d9b1019 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java @@ -598,7 +598,7 @@ private Long getTemplateSize(VirtualMachineMO template, String vmInternalName, M private VMTemplateVO createVMTemplateRecord(String vmInternalName, long guestOsId, long accountId) { Long nextTemplateId = vmTemplateDao.getNextInSequence(Long.class, "id"); VMTemplateVO templateVO = new VMTemplateVO(nextTemplateId, "Imported-from-" + vmInternalName, Storage.ImageFormat.OVA, false, false, false, Storage.TemplateType.USER, null, - false, 64, accountId, null, "Template imported from VM " + vmInternalName, false, guestOsId, false, HypervisorType.VMware, null, null, false, false, false); + false, 64, accountId, null, "Template imported from VM " + vmInternalName, false, guestOsId, false, HypervisorType.VMware, null, null, false, false, false, false); return vmTemplateDao.persist(templateVO); } @@ -619,7 +619,7 @@ private void updateTemplateRef(long templateId, Long poolId, String templatePath VMTemplateStoragePoolVO templateRef = templateStoragePoolDao.findByPoolPath(poolId, templatePath); if (templateRef == null) { templateRef = new VMTemplateStoragePoolVO(poolId, templateId, null, 100, VMTemplateStorageResourceAssoc.Status.DOWNLOADED, templatePath, null, null, templatePath, - templateSize); + templateSize, null); templateRef.setState(ObjectInDataStoreStateMachine.State.Ready); templateStoragePoolDao.persist(templateRef); } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java index c9f8b0c337a2..5c61887176cf 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java @@ -16,9 +16,7 @@ // under the License. package com.cloud.hypervisor.guru; -import com.cloud.agent.api.storage.OVFPropertyTO; -import com.cloud.agent.api.to.DataStoreTO; -import com.cloud.agent.api.to.DiskTO; +import com.cloud.agent.api.to.DeployAsIsInfoTO; import com.cloud.agent.api.to.NicTO; import com.cloud.agent.api.to.VirtualMachineTO; import com.cloud.exception.InsufficientAddressCapacityException; @@ -35,16 +33,10 @@ import com.cloud.network.dao.NetworkVO; import com.cloud.storage.GuestOSHypervisorVO; import com.cloud.storage.GuestOSVO; -import com.cloud.storage.TemplateOVFPropertyVO; -import com.cloud.storage.VMTemplateStoragePoolVO; -import com.cloud.storage.VMTemplateStorageResourceAssoc; -import com.cloud.storage.Volume; import com.cloud.storage.dao.GuestOSDao; import com.cloud.storage.dao.GuestOSHypervisorDao; -import com.cloud.storage.dao.TemplateOVFPropertiesDao; import com.cloud.storage.dao.VMTemplatePoolDao; import com.cloud.template.VirtualMachineTemplate; -import com.cloud.utils.Pair; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.DomainRouterVO; import com.cloud.vm.NicProfile; @@ -53,10 +45,8 @@ import com.cloud.vm.VmDetailConstants; import com.cloud.vm.dao.DomainRouterDao; import com.cloud.vm.dao.NicDao; -import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; -import org.apache.commons.collections.CollectionUtils; +import org.apache.cloudstack.storage.image.deployasis.DeployAsIsHelper; import org.apache.commons.lang.BooleanUtils; import org.apache.log4j.Logger; @@ -67,10 +57,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; class VmwareVmImplementer { - private static final Logger LOG = Logger.getLogger(VmwareVmImplementer.class); + private static final Logger LOGGER = Logger.getLogger(VmwareVmImplementer.class); @Inject DomainRouterDao domainRouterDao; @@ -89,11 +78,11 @@ class VmwareVmImplementer { @Inject PrimaryDataStoreDao storagePoolDao; @Inject - TemplateOVFPropertiesDao templateOVFPropertiesDao; - @Inject VMTemplatePoolDao templateStoragePoolDao; @Inject VmwareManager vmwareMgr; + @Inject + DeployAsIsHelper deployAsIsHelper; private Boolean globalNestedVirtualisationEnabled; private Boolean globalNestedVPerVMEnabled; @@ -116,10 +105,11 @@ void setGlobalNestedVPerVMEnabled(Boolean globalNestedVPerVMEnabled) { VirtualMachineTO implement(VirtualMachineProfile vm, VirtualMachineTO to, long clusterId) { to.setBootloader(VirtualMachineTemplate.BootloaderType.HVM); - + boolean deployAsIs = vm.getTemplate().isDeployAsIs(); + HostVO host = hostDao.findById(vm.getVirtualMachine().getHostId()); Map details = to.getDetails(); if (details == null) - details = new HashMap(); + details = new HashMap<>(); VirtualMachine.Type vmType = vm.getType(); boolean userVm = !(vmType.equals(VirtualMachine.Type.DomainRouter) || vmType.equals(VirtualMachine.Type.ConsoleProxy) || vmType.equals(VirtualMachine.Type.SecondaryStorageVm)); @@ -133,7 +123,7 @@ VirtualMachineTO implement(VirtualMachineProfile vm, VirtualMachineTO to, long c try { VirtualEthernetCardType.valueOf(nicDeviceType); } catch (Exception e) { - LOG.warn("Invalid NIC device type " + nicDeviceType + " is specified in VM details, switch to default E1000"); + LOGGER.warn("Invalid NIC device type " + nicDeviceType + " is specified in VM details, switch to default E1000"); details.put(VmDetailConstants.NIC_ADAPTER, VirtualEthernetCardType.E1000.toString()); } } @@ -145,7 +135,7 @@ VirtualMachineTO implement(VirtualMachineProfile vm, VirtualMachineTO to, long c try { VirtualEthernetCardType.valueOf(nicDeviceType); } catch (Exception e) { - LOG.warn("Invalid NIC device type " + nicDeviceType + " is specified in VM details, switch to default E1000"); + LOGGER.warn("Invalid NIC device type " + nicDeviceType + " is specified in VM details, switch to default E1000"); details.put(VmDetailConstants.NIC_ADAPTER, VirtualEthernetCardType.E1000.toString()); } } @@ -172,7 +162,7 @@ VirtualMachineTO implement(VirtualMachineProfile vm, VirtualMachineTO to, long c GuestOSVO guestOS = guestOsDao.findByIdIncludingRemoved(vm.getVirtualMachine().getGuestOSId()); to.setOs(guestOS.getDisplayName()); to.setHostName(vm.getHostName()); - HostVO host = hostDao.findById(vm.getVirtualMachine().getHostId()); + GuestOSHypervisorVO guestOsMapping = null; if (host != null) { guestOsMapping = guestOsHypervisorDao.findByOsIdAndHypervisor(guestOS.getId(), Hypervisor.HypervisorType.VMware.toString(), host.getHypervisorVersion()); @@ -183,19 +173,32 @@ VirtualMachineTO implement(VirtualMachineProfile vm, VirtualMachineTO to, long c to.setPlatformEmulator(guestOsMapping.getGuestOsName()); } - List ovfProperties = getOvfPropertyList(vm, details); - - handleOvfProperties(vm, to, details, ovfProperties); + if (deployAsIs) { + setDeployAsIsInfoTO(vm, to, details); + } setDetails(to, details); return to; } + /** + * Set the information relevant for deploy-as-is VMs on the VM TO + */ + private void setDeployAsIsInfoTO(VirtualMachineProfile vm, VirtualMachineTO to, Map details) { + String configuration = details.getOrDefault(VmDetailConstants.DEPLOY_AS_IS_CONFIGURATION, null); + Map properties = deployAsIsHelper.getVirtualMachineDeployAsIsProperties(vm); + String destStoragePool = deployAsIsHelper.getAllocatedVirtualMachineDestinationStoragePool(vm); + String templatePath = deployAsIsHelper.getAllocatedVirtualMachineTemplatePath(vm, configuration, destStoragePool); + Map nicsAdapterMapping = deployAsIsHelper.getAllocatedVirtualMachineNicsAdapterMapping(vm, to.getNics()); + DeployAsIsInfoTO info = new DeployAsIsInfoTO(templatePath, destStoragePool, properties, nicsAdapterMapping); + to.setDeployAsIsInfo(info); + } + private void setDetails(VirtualMachineTO to, Map details) { - if (LOG.isTraceEnabled()) { - for (String key: details.keySet()) { - LOG.trace(String.format("Detail for VM %s: %s => %s",to.getName(), key, details.get(key))); + if (LOGGER.isTraceEnabled()) { + for (String key : details.keySet()) { + LOGGER.trace(String.format("Detail for VM %s: %s => %s", to.getName(), key, details.get(key))); } } to.setDetails(details); @@ -286,56 +289,6 @@ private void configureDomainRouterNicsAndDetails(VirtualMachineProfile vm, Virtu } } - private void handleOvfProperties(VirtualMachineProfile vm, VirtualMachineTO to, Map details, List ovfProperties) { - if (CollectionUtils.isNotEmpty(ovfProperties)) { - removeOvfPropertiesFromDetails(ovfProperties, details); - String templateInstallPath = null; - List rootDiskList = vm.getDisks().stream().filter(x -> x.getType() == Volume.Type.ROOT).collect(Collectors.toList()); - if (rootDiskList.size() != 1) { - throw new CloudRuntimeException("Did not find only one root disk for VM " + vm.getHostName()); - } - - DiskTO rootDiskTO = rootDiskList.get(0); - DataStoreTO dataStore = rootDiskTO.getData().getDataStore(); - StoragePoolVO storagePoolVO = storagePoolDao.findByUuid(dataStore.getUuid()); - long dataCenterId = storagePoolVO.getDataCenterId(); - List pools = storagePoolDao.listByDataCenterId(dataCenterId); - for (StoragePoolVO pool : pools) { - VMTemplateStoragePoolVO ref = templateStoragePoolDao.findByPoolTemplate(pool.getId(), vm.getTemplateId()); - if (ref != null && ref.getDownloadState() == VMTemplateStorageResourceAssoc.Status.DOWNLOADED) { - templateInstallPath = ref.getInstallPath(); - break; - } - } - - if (templateInstallPath == null) { - throw new CloudRuntimeException("Did not find the template install path for template " + vm.getTemplateId() + " on zone " + dataCenterId); - } - - Pair> pair = new Pair>(templateInstallPath, ovfProperties); - to.setOvfProperties(pair); - } - } - - private List getOvfPropertyList(VirtualMachineProfile vm, Map details) { - List ovfProperties = new ArrayList(); - for (String detailKey : details.keySet()) { - if (detailKey.startsWith(ApiConstants.OVF_PROPERTIES)) { - String ovfPropKey = detailKey.replace(ApiConstants.OVF_PROPERTIES + "-", ""); - TemplateOVFPropertyVO templateOVFPropertyVO = templateOVFPropertiesDao.findByTemplateAndKey(vm.getTemplateId(), ovfPropKey); - if (templateOVFPropertyVO == null) { - LOG.warn(String.format("OVF property %s not found on template, discarding", ovfPropKey)); - continue; - } - String ovfValue = details.get(detailKey); - boolean isPassword = templateOVFPropertyVO.isPassword(); - OVFPropertyTO propertyTO = new OVFPropertyTO(ovfPropKey, ovfValue, isPassword); - ovfProperties.add(propertyTO); - } - } - return ovfProperties; - } - private void addReservationDetails(long clusterId, Map details) { details.put(VMwareGuru.VmwareReserveCpu.key(), VMwareGuru.VmwareReserveCpu.valueIn(clusterId).toString()); details.put(VMwareGuru.VmwareReserveMemory.key(), VMwareGuru.VmwareReserveMemory.valueIn(clusterId).toString()); @@ -383,16 +336,6 @@ private void setBootParameters(VirtualMachineProfile vm, VirtualMachineTO to, Ma // details.put(VmDetailConstants.BOOT_TYPE, to.getBootType()); } - /* - Remove OVF properties from details to be sent to hypervisor (avoid duplicate data) - */ - private void removeOvfPropertiesFromDetails(List ovfProperties, Map details) { - for (OVFPropertyTO propertyTO : ovfProperties) { - String key = propertyTO.getKey(); - details.remove(ApiConstants.OVF_PROPERTIES + "-" + key); - } - } - /** * Adds {@code 'nestedVirtualizationFlag'} value to {@code details} due to if it should be enabled or not * @param details vm details should not be null @@ -405,8 +348,8 @@ protected void configureNestedVirtualization(Map details, Virtua Boolean globalNestedVPerVMEnabled = getGlobalNestedVPerVMEnabled(); Boolean shouldEnableNestedVirtualization = shouldEnableNestedVirtualization(globalNestedVirtualisationEnabled, globalNestedVPerVMEnabled, localNestedV); - if(LOG.isDebugEnabled()) { - LOG.debug(String.format( + if(LOGGER.isDebugEnabled()) { + LOGGER.debug(String.format( "Due to '%B'(globalNestedVirtualisationEnabled) and '%B'(globalNestedVPerVMEnabled) I'm adding a flag with value %B to the vm configuration for Nested Virtualisation.", globalNestedVirtualisationEnabled, globalNestedVPerVMEnabled, diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareDatacenterService.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareDatacenterService.java index 53792539ee8e..2e3f98d795a5 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareDatacenterService.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareDatacenterService.java @@ -17,17 +17,19 @@ package com.cloud.hypervisor.vmware; -import java.util.List; - +import com.cloud.dc.VsphereStoragePolicy; +import com.cloud.exception.DiscoveryException; +import com.cloud.exception.ResourceInUseException; +import com.cloud.utils.component.PluggableService; +import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.api.command.admin.zone.AddVmwareDcCmd; +import org.apache.cloudstack.api.command.admin.zone.ImportVsphereStoragePoliciesCmd; import org.apache.cloudstack.api.command.admin.zone.ListVmwareDcsCmd; +import org.apache.cloudstack.api.command.admin.zone.ListVsphereStoragePoliciesCmd; import org.apache.cloudstack.api.command.admin.zone.RemoveVmwareDcCmd; import org.apache.cloudstack.api.command.admin.zone.UpdateVmwareDcCmd; -import com.cloud.exception.DiscoveryException; -import com.cloud.exception.ResourceInUseException; -import com.cloud.utils.component.PluggableService; -import com.cloud.utils.exception.CloudRuntimeException; +import java.util.List; public interface VmwareDatacenterService extends PluggableService { @@ -38,4 +40,9 @@ public interface VmwareDatacenterService extends PluggableService { boolean removeVmwareDatacenter(RemoveVmwareDcCmd cmd) throws IllegalArgumentException, ResourceInUseException; List listVmwareDatacenters(ListVmwareDcsCmd cmd) throws IllegalArgumentException, CloudRuntimeException; + + List importVsphereStoragePolicies(ImportVsphereStoragePoliciesCmd cmd); + + List listVsphereStoragePolicies(ListVsphereStoragePoliciesCmd cmd); + } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareHostService.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareHostService.java index 14630b3b89eb..ea97a6e8d54e 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareHostService.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareHostService.java @@ -17,6 +17,7 @@ package com.cloud.hypervisor.vmware.manager; import com.cloud.agent.api.Command; +import com.cloud.hypervisor.vmware.mo.DatastoreMO; import com.cloud.hypervisor.vmware.mo.VmwareHypervisorHost; import com.cloud.hypervisor.vmware.util.VmwareContext; @@ -27,5 +28,5 @@ public interface VmwareHostService { VmwareHypervisorHost getHyperHost(VmwareContext context, Command cmd); - String getWorkerName(VmwareContext context, Command cmd, int workerSequence); + String getWorkerName(VmwareContext context, Command cmd, int workerSequence, DatastoreMO dsMo) throws Exception; } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java index 9cbaaf7a55c1..a751abf1b8e5 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java @@ -16,43 +16,6 @@ // under the License. package com.cloud.hypervisor.vmware.manager; -import java.io.File; -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; -import java.rmi.RemoteException; -import java.time.Duration; -import java.time.Instant; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.UUID; -import java.util.concurrent.Executors; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - -import javax.inject.Inject; -import javax.naming.ConfigurationException; - -import org.apache.cloudstack.api.command.admin.zone.AddVmwareDcCmd; -import org.apache.cloudstack.api.command.admin.zone.ListVmwareDcsCmd; -import org.apache.cloudstack.api.command.admin.zone.RemoveVmwareDcCmd; -import org.apache.cloudstack.api.command.admin.zone.UpdateVmwareDcCmd; -import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; -import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; -import org.apache.cloudstack.framework.config.ConfigKey; -import org.apache.cloudstack.framework.config.Configurable; -import org.apache.cloudstack.framework.config.dao.ConfigurationDao; -import org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl; -import org.apache.cloudstack.management.ManagementServerHost; -import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; -import org.apache.cloudstack.utils.identity.ManagementServerNode; -import org.apache.log4j.Logger; - import com.amazonaws.util.CollectionUtils; import com.cloud.agent.AgentManager; import com.cloud.agent.Listener; @@ -70,9 +33,12 @@ import com.cloud.dc.ClusterVO; import com.cloud.dc.ClusterVSMMapVO; import com.cloud.dc.DataCenterVO; +import com.cloud.dc.VsphereStoragePolicy; +import com.cloud.dc.VsphereStoragePolicyVO; import com.cloud.dc.dao.ClusterDao; import com.cloud.dc.dao.ClusterVSMMapDao; import com.cloud.dc.dao.DataCenterDao; +import com.cloud.dc.dao.VsphereStoragePolicyDao; import com.cloud.event.ActionEvent; import com.cloud.event.EventTypes; import com.cloud.exception.DiscoveredWithErrorException; @@ -102,6 +68,7 @@ import com.cloud.hypervisor.vmware.mo.HostFirewallSystemMO; import com.cloud.hypervisor.vmware.mo.HostMO; import com.cloud.hypervisor.vmware.mo.HypervisorHostHelper; +import com.cloud.hypervisor.vmware.mo.PbmProfileManagerMO; import com.cloud.hypervisor.vmware.mo.VirtualEthernetCardType; import com.cloud.hypervisor.vmware.mo.VirtualSwitchType; import com.cloud.hypervisor.vmware.mo.VmwareHostType; @@ -142,8 +109,48 @@ import com.cloud.vm.dao.UserVmCloneSettingDao; import com.cloud.vm.dao.VMInstanceDao; import com.google.common.base.Strings; +import com.vmware.pbm.PbmProfile; import com.vmware.vim25.AboutInfo; import com.vmware.vim25.ManagedObjectReference; +import org.apache.cloudstack.api.command.admin.zone.AddVmwareDcCmd; +import org.apache.cloudstack.api.command.admin.zone.ImportVsphereStoragePoliciesCmd; +import org.apache.cloudstack.api.command.admin.zone.ListVmwareDcsCmd; +import org.apache.cloudstack.api.command.admin.zone.ListVsphereStoragePoliciesCmd; +import org.apache.cloudstack.api.command.admin.zone.RemoveVmwareDcCmd; +import org.apache.cloudstack.api.command.admin.zone.UpdateVmwareDcCmd; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl; +import org.apache.cloudstack.management.ManagementServerHost; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.utils.identity.ManagementServerNode; +import org.apache.log4j.Logger; + +import javax.inject.Inject; +import javax.naming.ConfigurationException; +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.rmi.RemoteException; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.UUID; +import java.util.concurrent.Executors; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; public class VmwareManagerImpl extends ManagerBase implements VmwareManager, VmwareStorageMount, Listener, VmwareDatacenterService, Configurable { private static final Logger s_logger = Logger.getLogger(VmwareManagerImpl.class); @@ -208,6 +215,8 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw private UserVmCloneSettingDao cloneSettingDao; @Inject private TemplateManager templateManager; + @Inject + private VsphereStoragePolicyDao vsphereStoragePolicyDao; private String _mountParent; private StorageLayer _storage; @@ -1046,6 +1055,8 @@ public List> getCommands() { cmdList.add(UpdateVmwareDcCmd.class); cmdList.add(RemoveVmwareDcCmd.class); cmdList.add(ListVmwareDcsCmd.class); + cmdList.add(ImportVsphereStoragePoliciesCmd.class); + cmdList.add(ListVsphereStoragePoliciesCmd.class); return cmdList; } @@ -1173,6 +1184,7 @@ public VmwareDatacenterVO addVmwareDatacenter(AddVmwareDcCmd cmd) throws Resourc } context = null; } + importVsphereStoragePoliciesInternal(zoneId, vmwareDc.getId()); return vmwareDc; } @@ -1233,6 +1245,7 @@ public VmwareDatacenter doInTransaction(TransactionStatus status) { hostDetailsDao.persist(host.getId(), hostDetails); } } + importVsphereStoragePoliciesInternal(zoneId, vmwareDc.getId()); return vmwareDc; } return null; @@ -1383,6 +1396,79 @@ private void doesZoneExist(Long zoneId) throws InvalidParameterValueException { } } + @Override + public List importVsphereStoragePolicies(ImportVsphereStoragePoliciesCmd cmd) { + Long zoneId = cmd.getZoneId(); + // Validate Id of zone + doesZoneExist(zoneId); + + final VmwareDatacenterZoneMapVO vmwareDcZoneMap = vmwareDatacenterZoneMapDao.findByZoneId(zoneId); + // Check if zone is associated with VMware DC + if (vmwareDcZoneMap == null) { + throw new CloudRuntimeException("Zone " + zoneId + " is not associated with any VMware datacenter."); + } + + final long vmwareDcId = vmwareDcZoneMap.getVmwareDcId(); + return importVsphereStoragePoliciesInternal(zoneId, vmwareDcId); + } + + public List importVsphereStoragePoliciesInternal(Long zoneId, Long vmwareDcId) { + + // Get DC associated with this zone + VmwareDatacenterVO vmwareDatacenter = vmwareDcDao.findById(vmwareDcId); + String vmwareDcName = vmwareDatacenter.getVmwareDatacenterName(); + String vCenterHost = vmwareDatacenter.getVcenterHost(); + String userName = vmwareDatacenter.getUser(); + String password = vmwareDatacenter.getPassword(); + List storageProfiles = null; + try { + s_logger.debug(String.format("Importing vSphere Storage Policies for the vmware DC %d in zone %d", vmwareDcId, zoneId)); + VmwareContext context = VmwareContextFactory.getContext(vCenterHost, userName, password); + PbmProfileManagerMO profileManagerMO = new PbmProfileManagerMO(context); + storageProfiles = profileManagerMO.getStorageProfiles(); + s_logger.debug(String.format("Import vSphere Storage Policies for the vmware DC %d in zone %d is successful", vmwareDcId, zoneId)); + } catch (Exception e) { + String msg = String.format("Unable to list storage profiles from DC %s due to : %s", vmwareDcName, VmwareHelper.getExceptionMessage(e)); + s_logger.error(msg); + throw new CloudRuntimeException(msg); + } + + for (PbmProfile storageProfile : storageProfiles) { + VsphereStoragePolicyVO storagePolicyVO = vsphereStoragePolicyDao.findByPolicyId(zoneId, storageProfile.getProfileId().getUniqueId()); + if (storagePolicyVO == null) { + storagePolicyVO = new VsphereStoragePolicyVO(zoneId, storageProfile.getProfileId().getUniqueId(), storageProfile.getName(), storageProfile.getDescription()); + vsphereStoragePolicyDao.persist(storagePolicyVO); + } else { + storagePolicyVO.setDescription(storageProfile.getDescription()); + storagePolicyVO.setName(storageProfile.getName()); + vsphereStoragePolicyDao.update(storagePolicyVO.getId(), storagePolicyVO); + } + } + + List allStoragePolicies = vsphereStoragePolicyDao.listAll(); + List finalStorageProfiles = storageProfiles; + List needToMarkRemoved = allStoragePolicies.stream() + .filter(existingPolicy -> !finalStorageProfiles.stream() + .anyMatch(storageProfile -> storageProfile.getProfileId().getUniqueId().equals(existingPolicy.getPolicyId()))) + .collect(Collectors.toList()); + + for (VsphereStoragePolicyVO storagePolicy : needToMarkRemoved) { + vsphereStoragePolicyDao.remove(storagePolicy.getId()); + } + + List storagePolicies = vsphereStoragePolicyDao.listAll(); + return storagePolicies; + } + + @Override + public List listVsphereStoragePolicies(ListVsphereStoragePoliciesCmd cmd) { + List storagePolicies = vsphereStoragePolicyDao.findByZoneId(cmd.getZoneId()); + if (storagePolicies != null) { + return new ArrayList<>(storagePolicies); + } + return Collections.emptyList(); + } + @Override public boolean hasNexusVSM(Long clusterId) { ClusterVSMMapVO vsmMapVo = null; diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareStorageManagerImpl.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareStorageManagerImpl.java index 369e5b6ef6e5..bb83e34f9784 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareStorageManagerImpl.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareStorageManagerImpl.java @@ -343,7 +343,7 @@ public Answer execute(VmwareHostService hostService, BackupSnapshotCommand cmd) if (vmMo == null) { dsMo = new DatastoreMO(hyperHost.getContext(), morDs); - workerVMName = hostService.getWorkerName(context, cmd, 0); + workerVMName = hostService.getWorkerName(context, cmd, 0, dsMo); vmMo = HypervisorHostHelper.createWorkerVM(hyperHost, dsMo, workerVMName); if (vmMo == null) { @@ -362,7 +362,7 @@ public Answer execute(VmwareHostService hostService, BackupSnapshotCommand cmd) } snapshotBackupUuid = backupSnapshotToSecondaryStorage(vmMo, accountId, volumeId, cmd.getVolumePath(), snapshotUuid, secondaryStorageUrl, prevSnapshotUuid, - prevBackupUuid, hostService.getWorkerName(context, cmd, 1), cmd.getNfsVersion()); + prevBackupUuid, hostService.getWorkerName(context, cmd, 1, dsMo), cmd.getNfsVersion()); success = (snapshotBackupUuid != null); if (success) { @@ -428,7 +428,7 @@ public Answer execute(VmwareHostService hostService, CreatePrivateTemplateFromVo } Ternary result = createTemplateFromVolume(vmMo, accountId, templateId, cmd.getUniqueName(), secondaryStoragePoolURL, volumePath, - hostService.getWorkerName(context, cmd, 0), cmd.getNfsVersion()); + hostService.getWorkerName(context, cmd, 0, null), cmd.getNfsVersion()); return new CreatePrivateTemplateAnswer(cmd, true, null, result.first(), result.third(), result.second(), cmd.getUniqueName(), ImageFormat.OVA); @@ -486,13 +486,13 @@ public Answer execute(VmwareHostService hostService, CopyVolumeCommand cmd) { Pair result; if (cmd.toSecondaryStorage()) { result = copyVolumeToSecStorage(hostService, hyperHost, cmd, vmName, volumeId, cmd.getPool().getUuid(), volumePath, secondaryStorageURL, - hostService.getWorkerName(context, cmd, 0), cmd.getNfsVersion()); + hostService.getWorkerName(context, cmd, 0, null), cmd.getNfsVersion()); } else { StorageFilerTO poolTO = cmd.getPool(); ManagedObjectReference morDatastore = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, poolTO.getUuid()); if (morDatastore == null) { - morDatastore = hyperHost.mountDatastore(false, poolTO.getHost(), 0, poolTO.getPath(), poolTO.getUuid().replace("-", "")); + morDatastore = hyperHost.mountDatastore(false, poolTO.getHost(), 0, poolTO.getPath(), poolTO.getUuid().replace("-", ""), true); if (morDatastore == null) { throw new Exception("Unable to mount storage pool on host. storeUrl: " + poolTO.getHost() + ":/" + poolTO.getPath()); @@ -591,7 +591,7 @@ private void copyTemplateFromSecondaryToPrimary(VmwareHypervisorHost hyperHost, } String vmName = templateUuid; - hyperHost.importVmFromOVF(srcFileName, vmName, datastoreMo, "thin"); + hyperHost.importVmFromOVF(srcFileName, vmName, datastoreMo, "thin", null); VirtualMachineMO vmMo = hyperHost.findVmOnHyperHost(vmName); if (vmMo == null) { @@ -912,7 +912,7 @@ private void restoreVolumeFromSecStorage(VmwareHypervisorHost hyperHost, Datasto VirtualMachineMO clonedVm = null; try { - hyperHost.importVmFromOVF(srcOVFFileName, newVolumeName, primaryDsMo, "thin"); + hyperHost.importVmFromOVF(srcOVFFileName, newVolumeName, primaryDsMo, "thin", null); clonedVm = hyperHost.findVmOnHyperHost(newVolumeName); if (clonedVm == null) { throw new Exception("Unable to create container VM for volume creation"); @@ -1025,7 +1025,7 @@ private Pair copyVolumeToSecStorage(VmwareHostService hostServic vmMo.createSnapshot(exportName, "Temporary snapshot for copy-volume command", false, false); } - exportVolumeToSecondaryStorage(vmMo, volumePath, secStorageUrl, "volumes/" + volumeFolder, exportName, hostService.getWorkerName(hyperHost.getContext(), cmd, 1), + exportVolumeToSecondaryStorage(vmMo, volumePath, secStorageUrl, "volumes/" + volumeFolder, exportName, hostService.getWorkerName(hyperHost.getContext(), cmd, 1, null), nfsVersion, clonedWorkerVMNeeded); return new Pair(volumeFolder, exportName); diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index c79c023dac72..b2ac0ccc5da5 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -40,21 +40,22 @@ import java.util.Set; import java.util.TimeZone; import java.util.UUID; +import java.util.stream.Collectors; import javax.naming.ConfigurationException; import javax.xml.datatype.XMLGregorianCalendar; +import com.cloud.agent.api.to.DataTO; +import com.cloud.agent.api.to.DeployAsIsInfoTO; +import com.cloud.agent.api.ValidateVcenterDetailsCommand; import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.storage.command.CopyCommand; -import org.apache.cloudstack.storage.command.StorageSubSystemCommand; import org.apache.cloudstack.storage.configdrive.ConfigDrive; -import org.apache.cloudstack.storage.resource.NfsSecondaryStorageResource; -import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; import org.apache.cloudstack.storage.to.TemplateObjectTO; import org.apache.cloudstack.storage.to.VolumeObjectTO; import org.apache.cloudstack.utils.volume.VirtualMachineDiskInfo; import org.apache.cloudstack.vm.UnmanagedInstanceTO; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.math.NumberUtils; import org.apache.log4j.Logger; @@ -177,7 +178,7 @@ import com.cloud.agent.api.storage.DestroyCommand; import com.cloud.agent.api.storage.MigrateVolumeAnswer; import com.cloud.agent.api.storage.MigrateVolumeCommand; -import com.cloud.agent.api.storage.OVFPropertyTO; +import com.cloud.agent.api.to.deployasis.OVFPropertyTO; import com.cloud.agent.api.storage.PrimaryStorageDownloadAnswer; import com.cloud.agent.api.storage.PrimaryStorageDownloadCommand; import com.cloud.agent.api.storage.ResizeVolumeAnswer; @@ -218,6 +219,7 @@ import com.cloud.hypervisor.vmware.mo.HypervisorHostHelper; import com.cloud.hypervisor.vmware.mo.NetworkDetails; import com.cloud.hypervisor.vmware.mo.TaskMO; +import com.cloud.hypervisor.vmware.mo.StoragepodMO; import com.cloud.hypervisor.vmware.mo.VirtualEthernetCardType; import com.cloud.hypervisor.vmware.mo.VirtualMachineDiskInfoBuilder; import com.cloud.hypervisor.vmware.mo.VirtualMachineMO; @@ -249,7 +251,6 @@ import com.cloud.utils.NumbersUtil; import com.cloud.utils.Pair; import com.cloud.utils.Ternary; -import com.cloud.utils.crypt.DBEncryptionUtil; import com.cloud.utils.db.DB; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.exception.ExceptionUtil; @@ -297,6 +298,7 @@ import com.vmware.vim25.PerfMetricSeries; import com.vmware.vim25.PerfQuerySpec; import com.vmware.vim25.RuntimeFaultFaultMsg; +import com.vmware.vim25.StoragePodSummary; import com.vmware.vim25.ToolsUnavailableFaultMsg; import com.vmware.vim25.VAppOvfSectionInfo; import com.vmware.vim25.VAppOvfSectionSpec; @@ -318,8 +320,8 @@ import com.vmware.vim25.VirtualEthernetCardNetworkBackingInfo; import com.vmware.vim25.VirtualEthernetCardOpaqueNetworkBackingInfo; import com.vmware.vim25.VirtualIDEController; -import com.vmware.vim25.VirtualMachineConfigSpec; import com.vmware.vim25.VirtualMachineBootOptions; +import com.vmware.vim25.VirtualMachineConfigSpec; import com.vmware.vim25.VirtualMachineFileInfo; import com.vmware.vim25.VirtualMachineFileLayoutEx; import com.vmware.vim25.VirtualMachineFileLayoutExFileInfo; @@ -337,9 +339,12 @@ import com.vmware.vim25.VirtualVmxnet3; import com.vmware.vim25.VmConfigInfo; import com.vmware.vim25.VmConfigSpec; -import com.vmware.vim25.VmfsDatastoreInfo; import com.vmware.vim25.VmwareDistributedVirtualSwitchPvlanSpec; import com.vmware.vim25.VmwareDistributedVirtualSwitchVlanIdSpec; +import org.apache.cloudstack.storage.command.CopyCommand; +import org.apache.cloudstack.storage.command.StorageSubSystemCommand; +import org.apache.cloudstack.storage.resource.NfsSecondaryStorageResource; +import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; import static com.cloud.utils.HumanReadableJson.getHumanReadableBytesJson; import static com.cloud.utils.NumbersUtil.toHumanReadableSize; @@ -568,6 +573,8 @@ public Answer executeRequest(Command cmd) { answer = execute((GetUnmanagedInstancesCommand) cmd); } else if (clz == PrepareUnmanageVMInstanceCommand.class) { answer = execute((PrepareUnmanageVMInstanceCommand) cmd); + } else if (clz == ValidateVcenterDetailsCommand.class) { + answer = execute((ValidateVcenterDetailsCommand) cmd); } else { answer = Answer.createUnsupportedCommandAnswer(cmd); } @@ -745,13 +752,30 @@ private Answer execute(ResizeVolumeCommand cmd) { } else if (newSize == oldSize) { return new ResizeVolumeAnswer(cmd, true, "success", newSize * ResourceType.bytesToKiB); } + /* + // FR41 this is yet to fix + ManagedObjectReference morDS1 = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, cmd.getPoolUuid()); + DatastoreMO dsMo1 = new DatastoreMO(hyperHost.getContext(), morDS1); + vmdkDataStorePath = VmwareStorageLayoutHelper.getLegacyDatastorePathFromVmdkFileName(dsMo1, path + VMDK_EXTENSION); + DatastoreFile dsFile1 = new DatastoreFile(vmdkDataStorePath); + + s_logger.debug("vDiskid does not exist for volume " + vmdkDataStorePath + " registering the disk now"); + VirtualStorageObjectManagerMO vStorageObjectManagerMO = new VirtualStorageObjectManagerMO(getServiceContext()); + try { + VStorageObject vStorageObject = vStorageObjectManagerMO.registerVirtualDisk(dsFile1, null, dsMo1.getOwnerDatacenter().second()); + VStorageObjectConfigInfo diskConfigInfo = vStorageObject.getConfig(); + ID vdiskId = diskConfigInfo.getId(); + } catch (Throwable e) { + if (e instanceof AlreadyExistsFaultMsg) { + + } + }*/ if (vmName.equalsIgnoreCase("none")) { // OfflineVmwareMigration: we need to refactor the worker vm creation out for use in migration methods as well as here // OfflineVmwareMigration: this method is 100 lines and needs refactorring anyway // we need to spawn a worker VM to attach the volume to and resize the volume. useWorkerVm = true; - vmName = getWorkerName(getServiceContext(), cmd, 0); String poolId = cmd.getPoolUuid(); @@ -759,6 +783,7 @@ private Answer execute(ResizeVolumeCommand cmd) { // OfflineVmwareMigration: 1. find data(store) ManagedObjectReference morDS = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, poolId); DatastoreMO dsMo = new DatastoreMO(hyperHost.getContext(), morDS); + vmName = getWorkerName(getServiceContext(), cmd, 0, dsMo); s_logger.info("Create worker VM " + vmName); @@ -1724,38 +1749,15 @@ protected StartAnswer execute(StartCommand cmd) { VirtualMachineFileLayoutEx existingVmFileLayout = null; List existingDatastores = new ArrayList(); + DeployAsIsInfoTO deployAsIsInfo = vmSpec.getDeployAsIsInfo(); + boolean deployAsIs = deployAsIsInfo != null; + Pair names = composeVmNames(vmSpec); String vmInternalCSName = names.first(); String vmNameOnVcenter = names.second(); - String dataDiskController = vmSpec.getDetails().get(VmDetailConstants.DATA_DISK_CONTROLLER); - String rootDiskController = vmSpec.getDetails().get(VmDetailConstants.ROOT_DISK_CONTROLLER); DiskTO rootDiskTO = null; - String bootMode = null; - if (vmSpec.getDetails().containsKey(VmDetailConstants.BOOT_MODE)) { - bootMode = vmSpec.getDetails().get(VmDetailConstants.BOOT_MODE); - } - if (null == bootMode) { - bootMode = ApiConstants.BootType.BIOS.toString(); - } - - // If root disk controller is scsi, then data disk controller would also be scsi instead of using 'osdefault' - // This helps avoid mix of different scsi subtype controllers in instance. - if (DiskControllerType.osdefault == DiskControllerType.getType(dataDiskController) && DiskControllerType.lsilogic == DiskControllerType.getType(rootDiskController)) { - dataDiskController = DiskControllerType.scsi.toString(); - } - - // Validate the controller types - dataDiskController = DiskControllerType.getType(dataDiskController).toString(); - rootDiskController = DiskControllerType.getType(rootDiskController).toString(); - - if (DiskControllerType.getType(rootDiskController) == DiskControllerType.none) { - throw new CloudRuntimeException("Invalid root disk controller detected : " + rootDiskController); - } - if (DiskControllerType.getType(dataDiskController) == DiskControllerType.none) { - throw new CloudRuntimeException("Invalid data disk controller detected : " + dataDiskController); - } - - Pair controllerInfo = new Pair(rootDiskController, dataDiskController); + String bootMode = getBootModeFromVmSpec(vmSpec, deployAsIs); + Pair controllerInfo = getControllerInfoFromVmSpec(vmSpec, deployAsIs); Boolean systemVm = vmSpec.getType().isUsedBySystem(); // Thus, vmInternalCSName always holds i-x-y, the cloudstack generated internal VM name. @@ -1775,7 +1777,9 @@ protected StartAnswer execute(StartCommand cmd) { s_logger.error(msg); throw new Exception(msg); } - String guestOsId = translateGuestOsIdentifier(vmSpec.getArch(), vmSpec.getOs(), vmSpec.getPlatformEmulator()).value(); + + DiskTO[] specDisks = vmSpec.getDisks(); + String guestOsId = getGuestOsIdFromVmSpec(vmSpec, deployAsIs); DiskTO[] disks = validateDisks(vmSpec.getDisks()); assert (disks.length > 0); NicTO[] nics = vmSpec.getNics(); @@ -1795,11 +1799,13 @@ protected StartAnswer execute(StartCommand cmd) { } VirtualMachineDiskInfoBuilder diskInfoBuilder = null; + VirtualDevice[] nicDevices = null; VirtualMachineMO vmMo = hyperHost.findVmOnHyperHost(vmInternalCSName); DiskControllerType systemVmScsiControllerType = DiskControllerType.lsilogic; int firstScsiControllerBusNum = 0; int numScsiControllerForSystemVm = 1; boolean hasSnapshot = false; + if (vmMo != null) { s_logger.info("VM " + vmInternalCSName + " already exists, tear down devices for reconfiguration"); if (getVmPowerState(vmMo) != PowerState.PowerOff) @@ -1808,15 +1814,11 @@ protected StartAnswer execute(StartCommand cmd) { // retrieve disk information before we tear down diskInfoBuilder = vmMo.getDiskInfoBuilder(); hasSnapshot = vmMo.hasSnapshot(); - if (!hasSnapshot) - vmMo.tearDownDevices(new Class[]{VirtualDisk.class, VirtualEthernetCard.class}); - else - vmMo.tearDownDevices(new Class[]{VirtualEthernetCard.class}); - if (systemVm) { - ensureScsiDiskControllers(vmMo, systemVmScsiControllerType.toString(), numScsiControllerForSystemVm, firstScsiControllerBusNum); - } else { - ensureDiskControllers(vmMo, controllerInfo); - } + nicDevices = vmMo.getNicDevices(); + + tearDownVmDevices(vmMo, hasSnapshot, deployAsIs); + ensureDiskControllersInternal(vmMo, systemVm, controllerInfo, systemVmScsiControllerType, + numScsiControllerForSystemVm, firstScsiControllerBusNum, deployAsIs); } else { ManagedObjectReference morDc = hyperHost.getHyperHostDatacenter(); assert (morDc != null); @@ -1834,17 +1836,10 @@ protected StartAnswer execute(StartCommand cmd) { diskInfoBuilder = vmMo.getDiskInfoBuilder(); hasSnapshot = vmMo.hasSnapshot(); - if (!hasSnapshot) - vmMo.tearDownDevices(new Class[]{VirtualDisk.class, VirtualEthernetCard.class}); - else - vmMo.tearDownDevices(new Class[]{VirtualEthernetCard.class}); - - if (systemVm) { - // System volumes doesn't require more than 1 SCSI controller as there is no requirement for data volumes. - ensureScsiDiskControllers(vmMo, systemVmScsiControllerType.toString(), numScsiControllerForSystemVm, firstScsiControllerBusNum); - } else { - ensureDiskControllers(vmMo, controllerInfo); - } + + tearDownVmDevices(vmMo, hasSnapshot, deployAsIs); + ensureDiskControllersInternal(vmMo, systemVm, controllerInfo, systemVmScsiControllerType, + numScsiControllerForSystemVm, firstScsiControllerBusNum, deployAsIs); } else { // If a VM with the same name is found in a different cluster in the DC, unregister the old VM and configure a new VM (cold-migration). VirtualMachineMO existingVmInDc = dcMo.findVm(vmInternalCSName); @@ -1856,45 +1851,56 @@ protected StartAnswer execute(StartCommand cmd) { existingDatastores = existingVmInDc.getAllDatastores(); existingVmInDc.unregisterVm(); } - Pair rootDiskDataStoreDetails = null; - for (DiskTO vol : disks) { - if (vol.getType() == Volume.Type.ROOT) { - Map details = vol.getDetails(); - boolean managed = false; - - if (details != null) { - managed = Boolean.parseBoolean(details.get(DiskTO.MANAGED)); - } - if (managed) { - String datastoreName = VmwareResource.getDatastoreName(details.get(DiskTO.IQN)); + if (deployAsIs) { + if (s_logger.isTraceEnabled()) { + s_logger.trace("Deploying OVA from template as-is"); + } + String deployAsIsTemplate = deployAsIsInfo.getTemplatePath(); + String destDatastore = deployAsIsInfo.getDestStoragePool(); + vmMo = _storageProcessor.cloneVMFromTemplate(deployAsIsTemplate, vmInternalCSName, destDatastore); + mapSpecDisksToClonedDisks(vmMo, vmInternalCSName, specDisks); + } else { + Pair rootDiskDataStoreDetails = null; + for (DiskTO vol : disks) { + if (vol.getType() == Volume.Type.ROOT) { + Map details = vol.getDetails(); + boolean managed = false; + + if (details != null) { + managed = Boolean.parseBoolean(details.get(DiskTO.MANAGED)); + } - rootDiskDataStoreDetails = dataStoresDetails.get(datastoreName); - } else { - DataStoreTO primaryStore = vol.getData().getDataStore(); + if (managed) { + String datastoreName = VmwareResource.getDatastoreName(details.get(DiskTO.IQN)); + + rootDiskDataStoreDetails = dataStoresDetails.get(datastoreName); + } else { + DataStoreTO primaryStore = vol.getData().getDataStore(); - rootDiskDataStoreDetails = dataStoresDetails.get(primaryStore.getUuid()); + rootDiskDataStoreDetails = dataStoresDetails.get(primaryStore.getUuid()); + } } } - } - assert (vmSpec.getMinSpeed() != null) && (rootDiskDataStoreDetails != null); + assert (vmSpec.getMinSpeed() != null) && (rootDiskDataStoreDetails != null); - boolean vmFolderExists = rootDiskDataStoreDetails.second().folderExists(String.format("[%s]", rootDiskDataStoreDetails.second().getName()), vmNameOnVcenter); - String vmxFileFullPath = dsRootVolumeIsOn.searchFileInSubFolders(vmNameOnVcenter + ".vmx", false, VmwareManager.s_vmwareSearchExcludeFolder.value()); - if (vmFolderExists && vmxFileFullPath != null) { // VM can be registered only if .vmx is present. - registerVm(vmNameOnVcenter, dsRootVolumeIsOn); - vmMo = hyperHost.findVmOnHyperHost(vmInternalCSName); - if (vmMo != null) { - if (s_logger.isDebugEnabled()) { - s_logger.debug("Found registered vm " + vmInternalCSName + " at host " + hyperHost.getHyperHostName()); + boolean vmFolderExists = rootDiskDataStoreDetails.second().folderExists(String.format("[%s]", rootDiskDataStoreDetails.second().getName()), vmNameOnVcenter); + String vmxFileFullPath = dsRootVolumeIsOn.searchFileInSubFolders(vmNameOnVcenter + ".vmx", false, VmwareManager.s_vmwareSearchExcludeFolder.value()); + if (vmFolderExists && vmxFileFullPath != null) { // VM can be registered only if .vmx is present. + registerVm(vmNameOnVcenter, dsRootVolumeIsOn); + vmMo = hyperHost.findVmOnHyperHost(vmInternalCSName); + if (vmMo != null) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("Found registered vm " + vmInternalCSName + " at host " + hyperHost.getHyperHostName()); + } } + tearDownVm(vmMo); + } else if (!hyperHost.createBlankVm(vmNameOnVcenter, vmInternalCSName, vmSpec.getCpus(), vmSpec.getMaxSpeed().intValue(), getReservedCpuMHZ(vmSpec), + vmSpec.getLimitCpuUse(), (int) (vmSpec.getMaxRam() / ResourceType.bytesToMiB), getReservedMemoryMb(vmSpec), guestOsId, rootDiskDataStoreDetails.first(), false, + controllerInfo, systemVm)) { + throw new Exception("Failed to create VM. vmName: " + vmInternalCSName); } - tearDownVm(vmMo); - } else if (!hyperHost.createBlankVm(vmNameOnVcenter, vmInternalCSName, vmSpec.getCpus(), vmSpec.getMaxSpeed().intValue(), getReservedCpuMHZ(vmSpec), - vmSpec.getLimitCpuUse(), (int) (vmSpec.getMaxRam() / ResourceType.bytesToMiB), getReservedMemoryMb(vmSpec), guestOsId, rootDiskDataStoreDetails.first(), false, - controllerInfo, systemVm)) { - throw new Exception("Failed to create VM. vmName: " + vmInternalCSName); } } @@ -1904,13 +1910,26 @@ protected StartAnswer execute(StartCommand cmd) { } } - int totalChangeDevices = disks.length + nics.length; + // The number of disks changed must be 0 for install as is, as the VM is a clone from the template as-is + int disksChanges = !deployAsIs ? disks.length : 0; + int totalChangeDevices = disksChanges + nics.length; + int hackDeviceCount = 0; + if (diskInfoBuilder != null) { + hackDeviceCount += diskInfoBuilder.getDiskCount(); + } + hackDeviceCount += nicDevices == null ? 0 : nicDevices.length; + if (s_logger.isTraceEnabled()) { + s_logger.trace(String.format("current count(s) desired: %d/ found:%d. now adding device to device count for vApp config ISO", totalChangeDevices, hackDeviceCount)); + } + if (deployAsIsInfo != null && deployAsIsInfo.getProperties() != null) { + totalChangeDevices++; + } DiskTO volIso = null; if (vmSpec.getType() != VirtualMachine.Type.User) { // system VM needs a patch ISO totalChangeDevices++; - } else { + } else if (!deployAsIs) { volIso = getIsoDiskTO(disks); if (volIso == null) totalChangeDevices++; @@ -1918,246 +1937,249 @@ protected StartAnswer execute(StartCommand cmd) { VirtualMachineConfigSpec vmConfigSpec = new VirtualMachineConfigSpec(); - VmwareHelper.setBasicVmConfig(vmConfigSpec, vmSpec.getCpus(), vmSpec.getMaxSpeed(), getReservedCpuMHZ(vmSpec), (int) (vmSpec.getMaxRam() / (1024 * 1024)), - getReservedMemoryMb(vmSpec), guestOsId, vmSpec.getLimitCpuUse()); - - // Check for multi-cores per socket settings - int numCoresPerSocket = 1; - String coresPerSocket = vmSpec.getDetails().get(VmDetailConstants.CPU_CORE_PER_SOCKET); - if (coresPerSocket != null) { - String apiVersion = HypervisorHostHelper.getVcenterApiVersion(vmMo.getContext()); - // Property 'numCoresPerSocket' is supported since vSphere API 5.0 - if (apiVersion.compareTo("5.0") >= 0) { - numCoresPerSocket = NumbersUtil.parseInt(coresPerSocket, 1); - vmConfigSpec.setNumCoresPerSocket(numCoresPerSocket); - } - } - - // Check for hotadd settings - vmConfigSpec.setMemoryHotAddEnabled(vmMo.isMemoryHotAddSupported(guestOsId)); - - String hostApiVersion = ((HostMO) hyperHost).getHostAboutInfo().getApiVersion(); - if (numCoresPerSocket > 1 && hostApiVersion.compareTo("5.0") < 0) { - s_logger.warn("Dynamic scaling of CPU is not supported for Virtual Machines with multi-core vCPUs in case of ESXi hosts 4.1 and prior. Hence CpuHotAdd will not be" - + " enabled for Virtual Machine: " + vmInternalCSName); - vmConfigSpec.setCpuHotAddEnabled(false); - } else { - vmConfigSpec.setCpuHotAddEnabled(vmMo.isCpuHotAddSupported(guestOsId)); - } - - configNestedHVSupport(vmMo, vmSpec, vmConfigSpec); - - VirtualDeviceConfigSpec[] deviceConfigSpecArray = new VirtualDeviceConfigSpec[totalChangeDevices]; int i = 0; int ideUnitNumber = 0; int scsiUnitNumber = 0; int ideControllerKey = vmMo.getIDEDeviceControllerKey(); int scsiControllerKey = vmMo.getScsiDeviceControllerKeyNoException(); - int controllerKey; - - // - // Setup ISO device - // + VirtualDeviceConfigSpec[] deviceConfigSpecArray = new VirtualDeviceConfigSpec[totalChangeDevices]; + DiskTO[] sortedDisks = sortVolumesByDeviceId(disks); - // prepare systemvm patch ISO - if (vmSpec.getType() != VirtualMachine.Type.User) { - // attach ISO (for patching of system VM) - Pair secStoreUrlAndId = mgr.getSecondaryStorageStoreUrlAndId(Long.parseLong(_dcId)); - String secStoreUrl = secStoreUrlAndId.first(); - Long secStoreId = secStoreUrlAndId.second(); - if (secStoreUrl == null) { - String msg = "secondary storage for dc " + _dcId + " is not ready yet?"; - throw new Exception(msg); + VmwareHelper.setBasicVmConfig(vmConfigSpec, vmSpec.getCpus(), vmSpec.getMaxSpeed(), getReservedCpuMHZ(vmSpec), (int) (vmSpec.getMaxRam() / (1024 * 1024)), + getReservedMemoryMb(vmSpec), guestOsId, vmSpec.getLimitCpuUse(), deployAsIs); + + if (!deployAsIs) { + // Check for multi-cores per socket settings + int numCoresPerSocket = 1; + String coresPerSocket = vmSpec.getDetails().get(VmDetailConstants.CPU_CORE_PER_SOCKET); + if (coresPerSocket != null) { + String apiVersion = HypervisorHostHelper.getVcenterApiVersion(vmMo.getContext()); + // Property 'numCoresPerSocket' is supported since vSphere API 5.0 + if (apiVersion.compareTo("5.0") >= 0) { + numCoresPerSocket = NumbersUtil.parseInt(coresPerSocket, 1); + vmConfigSpec.setNumCoresPerSocket(numCoresPerSocket); + } } - mgr.prepareSecondaryStorageStore(secStoreUrl, secStoreId); - ManagedObjectReference morSecDs = prepareSecondaryDatastoreOnHost(secStoreUrl); - if (morSecDs == null) { - String msg = "Failed to prepare secondary storage on host, secondary store url: " + secStoreUrl; - throw new Exception(msg); - } - DatastoreMO secDsMo = new DatastoreMO(hyperHost.getContext(), morSecDs); + // Check for hotadd settings + vmConfigSpec.setMemoryHotAddEnabled(vmMo.isMemoryHotAddSupported(guestOsId)); - deviceConfigSpecArray[i] = new VirtualDeviceConfigSpec(); - Pair isoInfo = VmwareHelper.prepareIsoDevice(vmMo, - String.format("[%s] systemvm/%s", secDsMo.getName(), mgr.getSystemVMIsoFileNameOnDatastore()), secDsMo.getMor(), true, true, ideUnitNumber++, i + 1); - deviceConfigSpecArray[i].setDevice(isoInfo.first()); - if (isoInfo.second()) { - if (s_logger.isDebugEnabled()) - s_logger.debug("Prepare ISO volume at new device " + _gson.toJson(isoInfo.first())); - deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.ADD); + String hostApiVersion = ((HostMO) hyperHost).getHostAboutInfo().getApiVersion(); + if (numCoresPerSocket > 1 && hostApiVersion.compareTo("5.0") < 0) { + s_logger.warn("Dynamic scaling of CPU is not supported for Virtual Machines with multi-core vCPUs in case of ESXi hosts 4.1 and prior. Hence CpuHotAdd will not be" + + " enabled for Virtual Machine: " + vmInternalCSName); + vmConfigSpec.setCpuHotAddEnabled(false); } else { - if (s_logger.isDebugEnabled()) - s_logger.debug("Prepare ISO volume at existing device " + _gson.toJson(isoInfo.first())); - deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.EDIT); + vmConfigSpec.setCpuHotAddEnabled(vmMo.isCpuHotAddSupported(guestOsId)); } - i++; - } else { - // Note: we will always plug a CDROM device - if (volIso != null) { - for (DiskTO vol : disks) { - if (vol.getType() == Volume.Type.ISO) { - - TemplateObjectTO iso = (TemplateObjectTO) vol.getData(); - - if (iso.getPath() != null && !iso.getPath().isEmpty()) { - DataStoreTO imageStore = iso.getDataStore(); - if (!(imageStore instanceof NfsTO)) { - s_logger.debug("unsupported protocol"); - throw new Exception("unsupported protocol"); - } - NfsTO nfsImageStore = (NfsTO) imageStore; - String isoPath = nfsImageStore.getUrl() + File.separator + iso.getPath(); - Pair isoDatastoreInfo = getIsoDatastoreInfo(hyperHost, isoPath); - assert (isoDatastoreInfo != null); - assert (isoDatastoreInfo.second() != null); - - deviceConfigSpecArray[i] = new VirtualDeviceConfigSpec(); - Pair isoInfo = - VmwareHelper.prepareIsoDevice(vmMo, isoDatastoreInfo.first(), isoDatastoreInfo.second(), true, true, ideUnitNumber++, i + 1); - deviceConfigSpecArray[i].setDevice(isoInfo.first()); - if (isoInfo.second()) { - if (s_logger.isDebugEnabled()) - s_logger.debug("Prepare ISO volume at new device " + _gson.toJson(isoInfo.first())); - deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.ADD); - } else { - if (s_logger.isDebugEnabled()) - s_logger.debug("Prepare ISO volume at existing device " + _gson.toJson(isoInfo.first())); - deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.EDIT); - } - } - i++; - } + + configNestedHVSupport(vmMo, vmSpec, vmConfigSpec); + + int controllerKey; + + // + // Setup ISO device + // + + // prepare systemvm patch ISO + if (vmSpec.getType() != VirtualMachine.Type.User) { + // attach ISO (for patching of system VM) + Pair secStoreUrlAndId = mgr.getSecondaryStorageStoreUrlAndId(Long.parseLong(_dcId)); + String secStoreUrl = secStoreUrlAndId.first(); + Long secStoreId = secStoreUrlAndId.second(); + if (secStoreUrl == null) { + String msg = "secondary storage for dc " + _dcId + " is not ready yet?"; + throw new Exception(msg); } - } else { + mgr.prepareSecondaryStorageStore(secStoreUrl, secStoreId); + + ManagedObjectReference morSecDs = prepareSecondaryDatastoreOnHost(secStoreUrl); + if (morSecDs == null) { + String msg = "Failed to prepare secondary storage on host, secondary store url: " + secStoreUrl; + throw new Exception(msg); + } + DatastoreMO secDsMo = new DatastoreMO(hyperHost.getContext(), morSecDs); + deviceConfigSpecArray[i] = new VirtualDeviceConfigSpec(); - Pair isoInfo = VmwareHelper.prepareIsoDevice(vmMo, null, null, true, true, ideUnitNumber++, i + 1); + Pair isoInfo = VmwareHelper.prepareIsoDevice(vmMo, + String.format("[%s] systemvm/%s", secDsMo.getName(), mgr.getSystemVMIsoFileNameOnDatastore()), secDsMo.getMor(), true, true, ideUnitNumber++, i + 1); deviceConfigSpecArray[i].setDevice(isoInfo.first()); if (isoInfo.second()) { if (s_logger.isDebugEnabled()) - s_logger.debug("Prepare ISO volume at existing device " + _gson.toJson(isoInfo.first())); - + s_logger.debug("Prepare ISO volume at new device " + _gson.toJson(isoInfo.first())); deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.ADD); } else { if (s_logger.isDebugEnabled()) s_logger.debug("Prepare ISO volume at existing device " + _gson.toJson(isoInfo.first())); - deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.EDIT); } i++; + } else { + // Note: we will always plug a CDROM device + if (volIso != null) { + for (DiskTO vol : disks) { + if (vol.getType() == Volume.Type.ISO) { + + TemplateObjectTO iso = (TemplateObjectTO) vol.getData(); + + if (iso.getPath() != null && !iso.getPath().isEmpty()) { + DataStoreTO imageStore = iso.getDataStore(); + if (!(imageStore instanceof NfsTO)) { + s_logger.debug("unsupported protocol"); + throw new Exception("unsupported protocol"); + } + NfsTO nfsImageStore = (NfsTO) imageStore; + String isoPath = nfsImageStore.getUrl() + File.separator + iso.getPath(); + Pair isoDatastoreInfo = getIsoDatastoreInfo(hyperHost, isoPath); + assert (isoDatastoreInfo != null); + assert (isoDatastoreInfo.second() != null); + + deviceConfigSpecArray[i] = new VirtualDeviceConfigSpec(); + Pair isoInfo = + VmwareHelper.prepareIsoDevice(vmMo, isoDatastoreInfo.first(), isoDatastoreInfo.second(), true, true, ideUnitNumber++, i + 1); + deviceConfigSpecArray[i].setDevice(isoInfo.first()); + if (isoInfo.second()) { + if (s_logger.isDebugEnabled()) + s_logger.debug("Prepare ISO volume at new device " + _gson.toJson(isoInfo.first())); + deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.ADD); + } else { + if (s_logger.isDebugEnabled()) + s_logger.debug("Prepare ISO volume at existing device " + _gson.toJson(isoInfo.first())); + deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.EDIT); + } + } + i++; + } + } + } else { + deviceConfigSpecArray[i] = new VirtualDeviceConfigSpec(); + Pair isoInfo = VmwareHelper.prepareIsoDevice(vmMo, null, null, true, true, ideUnitNumber++, i + 1); + deviceConfigSpecArray[i].setDevice(isoInfo.first()); + if (isoInfo.second()) { + if (s_logger.isDebugEnabled()) + s_logger.debug("Prepare ISO volume at existing device " + _gson.toJson(isoInfo.first())); + + deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.ADD); + } else { + if (s_logger.isDebugEnabled()) + s_logger.debug("Prepare ISO volume at existing device " + _gson.toJson(isoInfo.first())); + + deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.EDIT); + } + i++; + } } - } - // - // Setup ROOT/DATA disk devices - // - DiskTO[] sortedDisks = sortVolumesByDeviceId(disks); - for (DiskTO vol : sortedDisks) { - if (vol.getType() == Volume.Type.ISO) - continue; + // + // Setup ROOT/DATA disk devices + // + for (DiskTO vol : sortedDisks) { + if (vol.getType() == Volume.Type.ISO) + continue; - VirtualMachineDiskInfo matchingExistingDisk = getMatchingExistingDisk(diskInfoBuilder, vol, hyperHost, context); - controllerKey = getDiskController(matchingExistingDisk, vol, vmSpec, ideControllerKey, scsiControllerKey); - String diskController = getDiskController(vmMo, matchingExistingDisk, vol, new Pair(rootDiskController, dataDiskController)); - - if (DiskControllerType.getType(diskController) == DiskControllerType.osdefault) { - diskController = vmMo.getRecommendedDiskController(null); - } - if (DiskControllerType.getType(diskController) == DiskControllerType.ide) { - controllerKey = vmMo.getIDEControllerKey(ideUnitNumber); - if (vol.getType() == Volume.Type.DATADISK) { - // Could be result of flip due to user configured setting or "osdefault" for data disks - // Ensure maximum of 2 data volumes over IDE controller, 3 includeing root volume - if (vmMo.getNumberOfVirtualDisks() > 3) { - throw new CloudRuntimeException("Found more than 3 virtual disks attached to this VM [" + vmMo.getVmName() + "]. Unable to implement the disks over " - + diskController + " controller, as maximum number of devices supported over IDE controller is 4 includeing CDROM device."); - } - } - } else { - if (VmwareHelper.isReservedScsiDeviceNumber(scsiUnitNumber)) { - scsiUnitNumber++; - } + VirtualMachineDiskInfo matchingExistingDisk = getMatchingExistingDisk(diskInfoBuilder, vol, hyperHost, context); + controllerKey = getDiskController(matchingExistingDisk, vol, vmSpec, ideControllerKey, scsiControllerKey); + String diskController = getDiskController(vmMo, matchingExistingDisk, vol, controllerInfo); - controllerKey = vmMo.getScsiDiskControllerKeyNoException(diskController, scsiUnitNumber); - if (controllerKey == -1) { - // This may happen for ROOT legacy VMs which doesn't have recommended disk controller when global configuration parameter 'vmware.root.disk.controller' is set to "osdefault" - // Retrieve existing controller and use. - Ternary vmScsiControllerInfo = vmMo.getScsiControllerInfo(); - DiskControllerType existingControllerType = vmScsiControllerInfo.third(); - controllerKey = vmMo.getScsiDiskControllerKeyNoException(existingControllerType.toString(), scsiUnitNumber); + if (DiskControllerType.getType(diskController) == DiskControllerType.osdefault) { + diskController = vmMo.getRecommendedDiskController(null); } - } - if (!hasSnapshot) { - deviceConfigSpecArray[i] = new VirtualDeviceConfigSpec(); - - VolumeObjectTO volumeTO = (VolumeObjectTO) vol.getData(); - DataStoreTO primaryStore = volumeTO.getDataStore(); - Map details = vol.getDetails(); - boolean managed = false; - String iScsiName = null; + if (DiskControllerType.getType(diskController) == DiskControllerType.ide) { + controllerKey = vmMo.getIDEControllerKey(ideUnitNumber); + if (vol.getType() == Volume.Type.DATADISK) { + // Could be result of flip due to user configured setting or "osdefault" for data disks + // Ensure maximum of 2 data volumes over IDE controller, 3 includeing root volume + if (vmMo.getNumberOfVirtualDisks() > 3) { + throw new CloudRuntimeException("Found more than 3 virtual disks attached to this VM [" + vmMo.getVmName() + "]. Unable to implement the disks over " + + diskController + " controller, as maximum number of devices supported over IDE controller is 4 includeing CDROM device."); + } + } + } else { + if (VmwareHelper.isReservedScsiDeviceNumber(scsiUnitNumber)) { + scsiUnitNumber++; + } - if (details != null) { - managed = Boolean.parseBoolean(details.get(DiskTO.MANAGED)); - iScsiName = details.get(DiskTO.IQN); + controllerKey = vmMo.getScsiDiskControllerKeyNoException(diskController, scsiUnitNumber); + if (controllerKey == -1) { + // This may happen for ROOT legacy VMs which doesn't have recommended disk controller when global configuration parameter 'vmware.root.disk.controller' is set to "osdefault" + // Retrieve existing controller and use. + Ternary vmScsiControllerInfo = vmMo.getScsiControllerInfo(); + DiskControllerType existingControllerType = vmScsiControllerInfo.third(); + controllerKey = vmMo.getScsiDiskControllerKeyNoException(existingControllerType.toString(), scsiUnitNumber); + } } + if (!hasSnapshot) { + deviceConfigSpecArray[i] = new VirtualDeviceConfigSpec(); + + VolumeObjectTO volumeTO = (VolumeObjectTO) vol.getData(); + DataStoreTO primaryStore = volumeTO.getDataStore(); + Map details = vol.getDetails(); + boolean managed = false; + String iScsiName = null; + + if (details != null) { + managed = Boolean.parseBoolean(details.get(DiskTO.MANAGED)); + iScsiName = details.get(DiskTO.IQN); + } - // if the storage is managed, iScsiName should not be null - String datastoreName = managed ? VmwareResource.getDatastoreName(iScsiName) : primaryStore.getUuid(); - Pair volumeDsDetails = dataStoresDetails.get(datastoreName); + // if the storage is managed, iScsiName should not be null + String datastoreName = managed ? VmwareResource.getDatastoreName(iScsiName) : primaryStore.getUuid(); + Pair volumeDsDetails = dataStoresDetails.get(datastoreName); - assert (volumeDsDetails != null); + assert (volumeDsDetails != null); - String[] diskChain = syncDiskChain(dcMo, vmMo, vmSpec, vol, matchingExistingDisk, dataStoresDetails); + String[] diskChain = syncDiskChain(dcMo, vmMo, vmSpec, vol, matchingExistingDisk, dataStoresDetails); - int deviceNumber = -1; - if (controllerKey == vmMo.getIDEControllerKey(ideUnitNumber)) { - deviceNumber = ideUnitNumber % VmwareHelper.MAX_ALLOWED_DEVICES_IDE_CONTROLLER; - ideUnitNumber++; - } else { - deviceNumber = scsiUnitNumber % VmwareHelper.MAX_ALLOWED_DEVICES_SCSI_CONTROLLER; - scsiUnitNumber++; - } + int deviceNumber = -1; + if (controllerKey == vmMo.getIDEControllerKey(ideUnitNumber)) { + deviceNumber = ideUnitNumber % VmwareHelper.MAX_ALLOWED_DEVICES_IDE_CONTROLLER; + ideUnitNumber++; + } else { + deviceNumber = scsiUnitNumber % VmwareHelper.MAX_ALLOWED_DEVICES_SCSI_CONTROLLER; + scsiUnitNumber++; + } - VirtualDevice device = VmwareHelper.prepareDiskDevice(vmMo, null, controllerKey, diskChain, volumeDsDetails.first(), deviceNumber, i + 1); + VirtualDevice device = VmwareHelper.prepareDiskDevice(vmMo, null, controllerKey, diskChain, volumeDsDetails.first(), deviceNumber, i + 1); - if (vol.getType() == Volume.Type.ROOT) - rootDiskTO = vol; - deviceConfigSpecArray[i].setDevice(device); - deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.ADD); + if (vol.getType() == Volume.Type.ROOT) + rootDiskTO = vol; + deviceConfigSpecArray[i].setDevice(device); + deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.ADD); - if (s_logger.isDebugEnabled()) - s_logger.debug("Prepare volume at new device " + _gson.toJson(device)); + if (s_logger.isDebugEnabled()) + s_logger.debug("Prepare volume at new device " + _gson.toJson(device)); - i++; - } else { - if (controllerKey == vmMo.getIDEControllerKey(ideUnitNumber)) - ideUnitNumber++; - else - scsiUnitNumber++; + i++; + } else { + if (controllerKey == vmMo.getIDEControllerKey(ideUnitNumber)) + ideUnitNumber++; + else + scsiUnitNumber++; + } } - } - // - // Setup USB devices - // - if (guestOsId.startsWith("darwin")) { //Mac OS - VirtualDevice[] devices = vmMo.getMatchedDevices(new Class[]{VirtualUSBController.class}); - if (devices.length == 0) { - s_logger.debug("No USB Controller device on VM Start. Add USB Controller device for Mac OS VM " + vmInternalCSName); + // + // Setup USB devices + // + if (guestOsId.startsWith("darwin")) { //Mac OS + VirtualDevice[] devices = vmMo.getMatchedDevices(new Class[]{VirtualUSBController.class}); + if (devices.length == 0) { + s_logger.debug("No USB Controller device on VM Start. Add USB Controller device for Mac OS VM " + vmInternalCSName); - //For Mac OS X systems, the EHCI+UHCI controller is enabled by default and is required for USB mouse and keyboard access. - VirtualDevice usbControllerDevice = VmwareHelper.prepareUSBControllerDevice(); - deviceConfigSpecArray[i] = new VirtualDeviceConfigSpec(); - deviceConfigSpecArray[i].setDevice(usbControllerDevice); - deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.ADD); + //For Mac OS X systems, the EHCI+UHCI controller is enabled by default and is required for USB mouse and keyboard access. + VirtualDevice usbControllerDevice = VmwareHelper.prepareUSBControllerDevice(); + deviceConfigSpecArray[i] = new VirtualDeviceConfigSpec(); + deviceConfigSpecArray[i].setDevice(usbControllerDevice); + deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.ADD); - if (s_logger.isDebugEnabled()) - s_logger.debug("Prepare USB controller at new device " + _gson.toJson(deviceConfigSpecArray[i])); + if (s_logger.isDebugEnabled()) + s_logger.debug("Prepare USB controller at new device " + _gson.toJson(deviceConfigSpecArray[i])); - i++; - } else { - s_logger.debug("USB Controller device exists on VM Start for Mac OS VM " + vmInternalCSName); + i++; + } else { + s_logger.debug("USB Controller device exists on VM Start for Mac OS VM " + vmInternalCSName); + } } } @@ -2201,9 +2223,7 @@ protected StartAnswer execute(StartCommand cmd) { } } - VirtualEthernetCardType nicDeviceType = VirtualEthernetCardType.valueOf(vmSpec.getDetails().get(VmDetailConstants.NIC_ADAPTER)); - if (s_logger.isDebugEnabled()) - s_logger.debug("VM " + vmInternalCSName + " will be started with NIC device type: " + nicDeviceType); + VirtualEthernetCardType nicDeviceType; NiciraNvpApiVersion.logNiciraApiVersion(); @@ -2211,6 +2231,14 @@ protected StartAnswer execute(StartCommand cmd) { for (NicTO nicTo : sortNicsByDeviceId(nics)) { s_logger.info("Prepare NIC device based on NicTO: " + _gson.toJson(nicTo)); + String adapterTypeStr = deployAsIs ? + deployAsIsInfo.getNicAdapterMap().get(nicTo.getDeviceId()) : + vmSpec.getDetails().get(VmDetailConstants.NIC_ADAPTER); + nicDeviceType = VirtualEthernetCardType.valueOf(adapterTypeStr); + + if (s_logger.isDebugEnabled()) { + s_logger.debug("VM " + vmInternalCSName + " will be started with NIC device type: " + nicDeviceType + " on NIC device " + nicTo.getDeviceId()); + } boolean configureVServiceInNexus = (nicTo.getType() == TrafficType.Guest) && (vmSpec.getDetails().containsKey("ConfigureVServiceInNexus")); VirtualMachine.Type vmType = cmd.getVirtualMachine().getType(); Pair networkInfo = prepareNetworkFromNicInfo(vmMo.getRunningHost(), nicTo, configureVServiceInNexus, vmType); @@ -2264,47 +2292,31 @@ protected StartAnswer execute(StartCommand cmd) { // pass boot arguments through machine.id & perform customized options to VMX ArrayList extraOptions = new ArrayList(); configBasicExtraOption(extraOptions, vmSpec); - configNvpExtraOption(extraOptions, vmSpec, nicUuidToDvSwitchUuid); - configCustomExtraOption(extraOptions, vmSpec); - - // config for NCC - VirtualMachine.Type vmType = cmd.getVirtualMachine().getType(); - if (vmType.equals(VirtualMachine.Type.NetScalerVm)) { - NicTO mgmtNic = vmSpec.getNics()[0]; - OptionValue option = new OptionValue(); - option.setKey("machine.id"); - option.setValue("ip=" + mgmtNic.getIp() + "&netmask=" + mgmtNic.getNetmask() + "&gateway=" + mgmtNic.getGateway()); - extraOptions.add(option); - } - - // config VNC - String keyboardLayout = null; - if (vmSpec.getDetails() != null) - keyboardLayout = vmSpec.getDetails().get(VmDetailConstants.KEYBOARD); - vmConfigSpec.getExtraConfig() - .addAll(Arrays.asList(configureVnc(extraOptions.toArray(new OptionValue[0]), hyperHost, vmInternalCSName, vmSpec.getVncPassword(), keyboardLayout))); - - // config video card - configureVideoCard(vmMo, vmSpec, vmConfigSpec); - - // Set OVF properties (if available) - Pair> ovfPropsMap = vmSpec.getOvfProperties(); - VmConfigInfo templateVappConfig = null; - List ovfProperties = null; - if (ovfPropsMap != null) { - String vmTemplate = ovfPropsMap.first(); - s_logger.info("Find VM template " + vmTemplate); - VirtualMachineMO vmTemplateMO = dcMo.findVm(vmTemplate); - templateVappConfig = vmTemplateMO.getConfigInfo().getVAppConfig(); - ovfProperties = ovfPropsMap.second(); - // Set OVF properties (if available) - if (CollectionUtils.isNotEmpty(ovfProperties)) { - s_logger.info("Copying OVF properties from template and setting them to the values the user provided"); - copyVAppConfigsFromTemplate(templateVappConfig, ovfProperties, vmConfigSpec); - } - } - - setBootOptions(vmSpec, bootMode, vmConfigSpec); + + if (deployAsIs) { + setDeployAsIsProperties(vmMo, deployAsIsInfo, vmConfigSpec); + configureVNC(vmSpec, extraOptions, vmConfigSpec, hyperHost, vmInternalCSName); + } else { + configNvpExtraOption(extraOptions, vmSpec, nicUuidToDvSwitchUuid); + configCustomExtraOption(extraOptions, vmSpec); + + // config for NCC + VirtualMachine.Type vmType = cmd.getVirtualMachine().getType(); + if (vmType.equals(VirtualMachine.Type.NetScalerVm)) { + NicTO mgmtNic = vmSpec.getNics()[0]; + OptionValue option = new OptionValue(); + option.setKey("machine.id"); + option.setValue("ip=" + mgmtNic.getIp() + "&netmask=" + mgmtNic.getNetmask() + "&gateway=" + mgmtNic.getGateway()); + extraOptions.add(option); + } + + configureVNC(vmSpec, extraOptions, vmConfigSpec, hyperHost, vmInternalCSName); + + // config video card + configureVideoCard(vmMo, vmSpec, vmConfigSpec); + + setBootOptions(vmSpec, bootMode, vmConfigSpec); + } // // Configure VM @@ -2319,7 +2331,7 @@ protected StartAnswer execute(StartCommand cmd) { // Resizing root disk only when explicit requested by user final Map vmDetails = cmd.getVirtualMachine().getDetails(); - if (rootDiskTO != null && !hasSnapshot && (vmDetails != null && vmDetails.containsKey(ApiConstants.ROOT_DISK_SIZE))) { + if (!deployAsIs && rootDiskTO != null && !hasSnapshot && (vmDetails != null && vmDetails.containsKey(ApiConstants.ROOT_DISK_SIZE))) { resizeRootDiskOnVMStart(vmMo, rootDiskTO, hyperHost, context); } @@ -2393,6 +2405,155 @@ protected StartAnswer execute(StartCommand cmd) { } } + /** + * Configure VNC + */ + private void configureVNC(VirtualMachineTO vmSpec, ArrayList extraOptions, VirtualMachineConfigSpec vmConfigSpec, VmwareHypervisorHost hyperHost, String vmInternalCSName) throws Exception { + String keyboardLayout = null; + if (vmSpec.getDetails() != null) + keyboardLayout = vmSpec.getDetails().get(VmDetailConstants.KEYBOARD); + vmConfigSpec.getExtraConfig() + .addAll(Arrays.asList(configureVnc(extraOptions.toArray(new OptionValue[0]), hyperHost, vmInternalCSName, vmSpec.getVncPassword(), keyboardLayout))); + + } + + private void ensureDiskControllersInternal(VirtualMachineMO vmMo, Boolean systemVm, + Pair controllerInfo, + DiskControllerType systemVmScsiControllerType, + int numScsiControllerForSystemVm, + int firstScsiControllerBusNum, boolean deployAsIs) throws Exception { + if (systemVm) { + ensureScsiDiskControllers(vmMo, systemVmScsiControllerType.toString(), numScsiControllerForSystemVm, firstScsiControllerBusNum); + } else if (!deployAsIs) { + ensureDiskControllers(vmMo, controllerInfo); + } + } + + private void tearDownVmDevices(VirtualMachineMO vmMo, boolean hasSnapshot, boolean deployAsIs) throws Exception { + if (deployAsIs) { + vmMo.tearDownDevices(new Class[]{VirtualEthernetCard.class}); + } else if (!hasSnapshot) { + vmMo.tearDownDevices(new Class[]{VirtualDisk.class, VirtualEthernetCard.class}); + } else { + vmMo.tearDownDevices(new Class[]{VirtualEthernetCard.class}); + } + } + + private String getGuestOsIdFromVmSpec(VirtualMachineTO vmSpec, boolean deployAsIs) { + String guestOsId = null; + if (!deployAsIs) { + guestOsId = translateGuestOsIdentifier(vmSpec.getArch(), vmSpec.getOs(), vmSpec.getPlatformEmulator()).value(); + } + return guestOsId; + } + + private Pair getControllerInfoFromVmSpec(VirtualMachineTO vmSpec, boolean deployAsIs) throws CloudRuntimeException { + String dataDiskController = vmSpec.getDetails().get(VmDetailConstants.DATA_DISK_CONTROLLER); + String rootDiskController = vmSpec.getDetails().get(VmDetailConstants.ROOT_DISK_CONTROLLER); + + // If root disk controller is scsi, then data disk controller would also be scsi instead of using 'osdefault' + // This helps avoid mix of different scsi subtype controllers in instance. + if (DiskControllerType.osdefault == DiskControllerType.getType(dataDiskController) && DiskControllerType.lsilogic == DiskControllerType.getType(rootDiskController)) { + dataDiskController = DiskControllerType.scsi.toString(); + } + + // Validate the controller types + dataDiskController = DiskControllerType.getType(dataDiskController).toString(); + rootDiskController = DiskControllerType.getType(rootDiskController).toString(); + + if (DiskControllerType.getType(rootDiskController) == DiskControllerType.none) { + throw new CloudRuntimeException("Invalid root disk controller detected : " + rootDiskController); + } + if (DiskControllerType.getType(dataDiskController) == DiskControllerType.none) { + throw new CloudRuntimeException("Invalid data disk controller detected : " + dataDiskController); + } + + return new Pair(rootDiskController, dataDiskController); + } + + private String getBootModeFromVmSpec(VirtualMachineTO vmSpec, boolean deployAsIs) { + String bootMode = null; + if (!deployAsIs) { + if (vmSpec.getDetails().containsKey(VmDetailConstants.BOOT_MODE)) { + bootMode = vmSpec.getDetails().get(VmDetailConstants.BOOT_MODE); + } + if (null == bootMode) { + bootMode = ApiConstants.BootType.BIOS.toString(); + } + } + return bootMode; + } + + /** + * Set OVF properties (if available) + */ + private void setDeployAsIsProperties(VirtualMachineMO vmMo, DeployAsIsInfoTO deployAsIsInfo, + VirtualMachineConfigSpec vmConfigSpec) throws Exception { + if (deployAsIsInfo != null) { + Map properties = deployAsIsInfo.getProperties(); + VmConfigInfo vAppConfig = vmMo.getConfigInfo().getVAppConfig(); + s_logger.info("Copying OVF properties to the values the user provided"); + setVAppPropertiesToConfigSpec(vAppConfig, properties, vmConfigSpec); + } + } + + /** + * Modify the specDisks information to match the cloned VM's disks (from vmMo VM) + */ + private void mapSpecDisksToClonedDisks(VirtualMachineMO vmMo, String vmInternalCSName, DiskTO[] specDisks) { + try { + s_logger.debug("Mapping spec disks information to cloned VM disks for VM " + vmInternalCSName); + if (vmMo != null && ArrayUtils.isNotEmpty(specDisks)) { + List vmDisks = vmMo.getVirtualDisks(); + List sortedDisks = Arrays.asList(sortVolumesByDeviceId(specDisks)) + .stream() + .filter(x -> x.getType() == Volume.Type.ROOT) + .collect(Collectors.toList()); + if (sortedDisks.size() != vmDisks.size()) { + s_logger.error("Different number of root disks spec vs cloned deploy-as-is VM disks: " + sortedDisks.size() + " - " + vmDisks.size()); + return; + } + for (int i = 0; i < sortedDisks.size(); i++) { + DiskTO specDisk = sortedDisks.get(i); + VirtualDisk vmDisk = vmDisks.get(i); + DataTO dataVolume = specDisk.getData(); + if (dataVolume instanceof VolumeObjectTO) { + VolumeObjectTO volumeObjectTO = (VolumeObjectTO) dataVolume; + if (!volumeObjectTO.getSize().equals(vmDisk.getCapacityInBytes())) { + s_logger.info("Mapped disk size is not the same as the cloned VM disk size: " + + volumeObjectTO.getSize() + " - " + vmDisk.getCapacityInBytes()); + } + VirtualDeviceBackingInfo backingInfo = vmDisk.getBacking(); + if (backingInfo instanceof VirtualDiskFlatVer2BackingInfo) { + VirtualDiskFlatVer2BackingInfo backing = (VirtualDiskFlatVer2BackingInfo) backingInfo; + String fileName = backing.getFileName(); + if (StringUtils.isNotBlank(fileName)) { + String[] fileNameParts = fileName.split(" "); + String datastoreUuid = fileNameParts[0].replace("[", "").replace("]", ""); + String relativePath = fileNameParts[1].split("/")[1].replace(".vmdk", ""); + String vmSpecDatastoreUuid = volumeObjectTO.getDataStore().getUuid().replaceAll("-", ""); + if (!datastoreUuid.equals(vmSpecDatastoreUuid)) { + s_logger.info("Mapped disk datastore UUID is not the same as the cloned VM datastore UUID: " + + datastoreUuid + " - " + vmSpecDatastoreUuid); + } + volumeObjectTO.setPath(relativePath); + specDisk.setPath(relativePath); + } else { + s_logger.error("Empty backing filename for volume " + volumeObjectTO.getName()); + } + } else { + s_logger.error("Could not get volume backing info for volume " + volumeObjectTO.getName()); + } + } + } + } + } catch (Exception e) { + String msg = "Error mapping deploy-as-is VM disks from cloned VM " + vmInternalCSName; + s_logger.error(msg, e); + throw new CloudRuntimeException(e); + } + } + private void setBootOptions(VirtualMachineTO vmSpec, String bootMode, VirtualMachineConfigSpec vmConfigSpec) { VirtualMachineBootOptions bootOptions = null; if (StringUtils.isNotBlank(bootMode) && !bootMode.equalsIgnoreCase("bios")) { @@ -2436,26 +2597,33 @@ protected List copyVAppConfigOvfSectionFromOVF(VmConfigInfo private Map> getOVFMap(List props) { Map> map = new HashMap<>(); for (OVFPropertyTO prop : props) { - Pair pair = new Pair<>(prop.getValue(), prop.isPassword()); + String value = getPropertyValue(prop); + Pair pair = new Pair<>(value, prop.isPassword()); map.put(prop.getKey(), pair); } return map; } + private String getPropertyValue(OVFPropertyTO prop) { + String type = prop.getType(); + String value = prop.getValue(); + if ("boolean".equalsIgnoreCase(type)) { + value = Boolean.parseBoolean(value) ? "True" : "False"; + } + return value; + } + /** * Set the properties section from existing vApp configuration and values set on ovfProperties */ - protected List copyVAppConfigPropertySectionFromOVF(VmConfigInfo vAppConfig, List ovfProperties) { + protected List copyVAppConfigPropertySectionFromOVF(VmConfigInfo vAppConfig, Map ovfProperties) { List productFromOvf = vAppConfig.getProperty(); List specs = new ArrayList<>(); - Map> ovfMap = getOVFMap(ovfProperties); for (VAppPropertyInfo info : productFromOvf) { VAppPropertySpec spec = new VAppPropertySpec(); - if (ovfMap.containsKey(info.getId())) { - Pair pair = ovfMap.get(info.getId()); - String value = pair.first(); - boolean isPassword = pair.second(); - info.setValue(isPassword ? DBEncryptionUtil.decrypt(value) : value); + if (ovfProperties.containsKey(info.getId())) { + String value = ovfProperties.get(info.getId()); + info.setValue(value); } spec.setInfo(info); spec.setOperation(ArrayUpdateOperation.ADD); @@ -2483,9 +2651,9 @@ protected List copyVAppConfigProductSectionFromOVF(VmConfigInfo * Set the vApp configuration to vmConfig spec, copying existing configuration from vAppConfig * and seting properties values from ovfProperties */ - protected void copyVAppConfigsFromTemplate(VmConfigInfo vAppConfig, - List ovfProperties, - VirtualMachineConfigSpec vmConfig) throws Exception { + protected void setVAppPropertiesToConfigSpec(VmConfigInfo vAppConfig, + Map ovfProperties, + VirtualMachineConfigSpec vmConfig) throws Exception { VmConfigSpec vmConfigSpec = new VmConfigSpec(); vmConfigSpec.getEula().addAll(vAppConfig.getEula()); vmConfigSpec.setInstallBootStopDelay(vAppConfig.getInstallBootStopDelay()); @@ -2702,43 +2870,51 @@ private String[] syncDiskChain(DatacenterMO dcMo, VirtualMachineMO vmMo, Virtual } DatastoreMO dsMo = volumeDsDetails.second(); + String datastoreDiskPath; - // we will honor vCenter's meta if it exists - if (diskInfo != null) { - // to deal with run-time upgrade to maintain the new datastore folder structure - String disks[] = diskInfo.getDiskChain(); - for (int i = 0; i < disks.length; i++) { - DatastoreFile file = new DatastoreFile(disks[i]); - if (!isManaged && file.getDir() != null && file.getDir().isEmpty()) { - s_logger.info("Perform run-time datastore folder upgrade. sync " + disks[i] + " to VM folder"); - disks[i] = VmwareStorageLayoutHelper.syncVolumeToVmDefaultFolder(dcMo, vmMo.getName(), dsMo, file.getFileBaseName(), VmwareManager.s_vmwareSearchExcludeFolder.value()); + if (dsMo.getDatastoreType().equalsIgnoreCase("VVOL")) { + datastoreDiskPath = VmwareStorageLayoutHelper.getLegacyDatastorePathFromVmdkFileName(dsMo, volumeTO.getPath() + ".vmdk"); + if (!dsMo.fileExists(datastoreDiskPath)) { + datastoreDiskPath = VmwareStorageLayoutHelper.getVmwareDatastorePathFromVmdkFileName(dsMo, vmMo.getName(), volumeTO.getPath() + ".vmdk"); + } + if (!dsMo.fileExists(datastoreDiskPath)) { + datastoreDiskPath = dsMo.searchFileInSubFolders(volumeTO.getPath() + ".vmdk", true, null); + } + } else { + // we will honor vCenter's meta if it exists + if (diskInfo != null) { + // to deal with run-time upgrade to maintain the new datastore folder structure + String disks[] = diskInfo.getDiskChain(); + for (int i = 0; i < disks.length; i++) { + DatastoreFile file = new DatastoreFile(disks[i]); + if (!isManaged && file.getDir() != null && file.getDir().isEmpty()) { + s_logger.info("Perform run-time datastore folder upgrade. sync " + disks[i] + " to VM folder"); + disks[i] = VmwareStorageLayoutHelper.syncVolumeToVmDefaultFolder(dcMo, vmMo.getName(), dsMo, file.getFileBaseName(), VmwareManager.s_vmwareSearchExcludeFolder.value()); + } } + return disks; } - return disks; - } - final String datastoreDiskPath; + if (isManaged) { + String vmdkPath = new DatastoreFile(volumeTO.getPath()).getFileBaseName(); - if (isManaged) { - String vmdkPath = new DatastoreFile(volumeTO.getPath()).getFileBaseName(); + if (volumeTO.getVolumeType() == Volume.Type.ROOT) { + if (vmdkPath == null) { + vmdkPath = volumeTO.getName(); + } - if (volumeTO.getVolumeType() == Volume.Type.ROOT) { - if (vmdkPath == null) { - vmdkPath = volumeTO.getName(); - } + datastoreDiskPath = VmwareStorageLayoutHelper.syncVolumeToVmDefaultFolder(dcMo, vmMo.getName(), dsMo, vmdkPath); + } else { + if (vmdkPath == null) { + vmdkPath = dsMo.getName(); + } - datastoreDiskPath = VmwareStorageLayoutHelper.syncVolumeToVmDefaultFolder(dcMo, vmMo.getName(), dsMo, vmdkPath); - } else { - if (vmdkPath == null) { - vmdkPath = dsMo.getName(); + datastoreDiskPath = dsMo.getDatastorePath(vmdkPath + VMDK_EXTENSION); } - - datastoreDiskPath = dsMo.getDatastorePath(vmdkPath + VMDK_EXTENSION); + } else { + datastoreDiskPath = VmwareStorageLayoutHelper.syncVolumeToVmDefaultFolder(dcMo, vmMo.getName(), dsMo, volumeTO.getPath(), VmwareManager.s_vmwareSearchExcludeFolder.value()); } - } else { - datastoreDiskPath = VmwareStorageLayoutHelper.syncVolumeToVmDefaultFolder(dcMo, vmMo.getName(), dsMo, volumeTO.getPath(), VmwareManager.s_vmwareSearchExcludeFolder.value()); } - if (!dsMo.fileExists(datastoreDiskPath)) { s_logger.warn("Volume " + volumeTO.getId() + " does not seem to exist on datastore, out of sync? path: " + datastoreDiskPath); } @@ -4384,7 +4560,8 @@ protected Answer execute(MigrateWithStorageCommand cmd) { s_logger.debug("Preparing spec for volume : " + volume.getName()); morDsAtTarget = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(tgtHyperHost, filerTo.getUuid()); - morDsAtSource = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(srcHyperHost, filerTo.getUuid()); + morDsAtSource = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(srcHyperHost, volume.getPoolUuid()); + if (morDsAtTarget == null) { String msg = "Unable to find the target datastore: " + filerTo.getUuid() + " on target host: " + tgtHyperHost.getHyperHostName() + " to execute MigrateWithStorageCommand"; @@ -4404,7 +4581,7 @@ protected Answer execute(MigrateWithStorageCommand cmd) { // If datastore is NFS and target datastore is not already mounted on source host then mount the datastore. if (filerTo.getType().equals(StoragePoolType.NetworkFilesystem)) { if (morDsAtSource == null) { - morDsAtSource = srcHyperHost.mountDatastore(false, tgtDsHost, tgtDsPort, tgtDsPath, tgtDsName); + morDsAtSource = srcHyperHost.mountDatastore(false, tgtDsHost, tgtDsPort, tgtDsPath, tgtDsName, true); if (morDsAtSource == null) { throw new Exception("Unable to mount NFS datastore " + tgtDsHost + ":/" + tgtDsPath + " on " + _hostName); } @@ -4412,9 +4589,8 @@ protected Answer execute(MigrateWithStorageCommand cmd) { s_logger.debug("Mounted datastore " + tgtDsHost + ":/" + tgtDsPath + " on " + _hostName); } } - // If datastore is VMFS and target datastore is not mounted or accessible to source host then fail migration. - if (filerTo.getType().equals(StoragePoolType.VMFS)) { + if (filerTo.getType().equals(StoragePoolType.VMFS) || filerTo.getType().equals(StoragePoolType.PreSetup)) { if (morDsAtSource == null) { s_logger.warn( "If host version is below 5.1, then target VMFS datastore(s) need to manually mounted on source host for a successful live storage migration."); @@ -4433,6 +4609,7 @@ protected Answer execute(MigrateWithStorageCommand cmd) { if (volume.getType() == Volume.Type.ROOT) { relocateSpec.setDatastore(morTgtDatastore); } + diskLocator = new VirtualMachineRelocateSpecDiskLocator(); diskLocator.setDatastore(morDsAtSource); Pair diskInfo = getVirtualDiskInfo(vmMo, appendFileType(volume.getPath(), VMDK_EXTENSION)); @@ -4457,8 +4634,9 @@ protected Answer execute(MigrateWithStorageCommand cmd) { diskLocators.add(diskLocator); } } - - relocateSpec.getDisk().addAll(diskLocators); + if (srcHyperHost.getHyperHostCluster().equals(tgtHyperHost.getHyperHostCluster())) { + relocateSpec.getDisk().addAll(diskLocators); + } // Prepare network at target before migration NicTO[] nics = vmTo.getNics(); @@ -4576,7 +4754,9 @@ private Answer migrateVolume(MigrateVolumeCommand cmd) { VmwareHypervisorHost hyperHost = getHyperHost(getServiceContext()); VirtualMachineMO vmMo = null; DatastoreMO dsMo = null; + DatastoreMO destinationDsMo = null; ManagedObjectReference morSourceDS = null; + ManagedObjectReference morDestintionDS = null; String vmdkDataStorePath = null; String vmName = null; @@ -4584,15 +4764,20 @@ private Answer migrateVolume(MigrateVolumeCommand cmd) { // OfflineVmwareMigration: we need to refactor the worker vm creation out for use in migration methods as well as here // OfflineVmwareMigration: this method is 100 lines and needs refactorring anyway // we need to spawn a worker VM to attach the volume to and move it - vmName = getWorkerName(getServiceContext(), cmd, 0); + morSourceDS = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, cmd.getSourcePool().getUuid()); + dsMo = new DatastoreMO(hyperHost.getContext(), morSourceDS); + morDestintionDS = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, cmd.getTargetPool().getUuid()); + destinationDsMo = new DatastoreMO(hyperHost.getContext(), morDestintionDS); + vmName = getWorkerName(getServiceContext(), cmd, 0, dsMo); + + if (destinationDsMo.getDatastoreType().equalsIgnoreCase("VVOL")) + vmName = getWorkerName(getServiceContext(), cmd, 0, destinationDsMo); // OfflineVmwareMigration: refactor for re-use // OfflineVmwareMigration: 1. find data(store) // OfflineVmwareMigration: more robust would be to find the store given the volume as it might have been moved out of band or due to error // example: DatastoreMO existingVmDsMo = new DatastoreMO(dcMo.getContext(), dcMo.findDatastore(fileInDatastore.getDatastoreName())); - morSourceDS = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, cmd.getSourcePool().getUuid()); - dsMo = new DatastoreMO(hyperHost.getContext(), morSourceDS); s_logger.info("Create worker VM " + vmName); // OfflineVmwareMigration: 2. create the worker with access to the data(store) vmMo = HypervisorHostHelper.createWorkerVM(hyperHost, dsMo, vmName); @@ -4901,27 +5086,71 @@ protected Answer execute(ModifyStoragePoolCommand cmd) { VmwareHypervisorHost hyperHost = getHyperHost(getServiceContext()); StorageFilerTO pool = cmd.getPool(); - if (pool.getType() != StoragePoolType.NetworkFilesystem && pool.getType() != StoragePoolType.VMFS) { + if (pool.getType() != StoragePoolType.NetworkFilesystem && pool.getType() != StoragePoolType.VMFS && pool.getType() != StoragePoolType.PreSetup && pool.getType() != StoragePoolType.DatastoreCluster) { throw new Exception("Unsupported storage pool type " + pool.getType()); } ManagedObjectReference morDatastore = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, pool.getUuid()); if (morDatastore == null) { - morDatastore = hyperHost.mountDatastore(pool.getType() == StoragePoolType.VMFS, pool.getHost(), pool.getPort(), pool.getPath(), pool.getUuid().replace("-", "")); + morDatastore = hyperHost.mountDatastore((pool.getType() == StoragePoolType.VMFS || pool.getType() == StoragePoolType.PreSetup || pool.getType() == StoragePoolType.DatastoreCluster), pool.getHost(), pool.getPort(), pool.getPath(), pool.getUuid().replace("-", ""), true); } assert (morDatastore != null); - DatastoreSummary summary = new DatastoreMO(getServiceContext(), morDatastore).getSummary(); + DatastoreMO dsMo = new DatastoreMO(getServiceContext(), morDatastore); + HypervisorHostHelper.createBaseFolder(dsMo, hyperHost, pool.getType()); + + long capacity = 0; + long available = 0; + List childDatastoresModifyStoragePoolAnswers = new ArrayList<>(); + if (pool.getType() == StoragePoolType.DatastoreCluster) { + StoragepodMO datastoreClusterMo = new StoragepodMO(getServiceContext(), morDatastore); + StoragePodSummary dsClusterSummary = datastoreClusterMo.getDatastoreClusterSummary(); + capacity = dsClusterSummary.getCapacity(); + available = dsClusterSummary.getFreeSpace(); + + List childDatastoreMors = datastoreClusterMo.getDatastoresInDatastoreCluster(); + for (ManagedObjectReference childDsMor : childDatastoreMors) { + DatastoreMO childDsMo = new DatastoreMO(getServiceContext(), childDsMor); + + Map tInfo = new HashMap<>(); + DatastoreSummary summary = childDsMo.getDatastoreSummary();; + ModifyStoragePoolAnswer answer = new ModifyStoragePoolAnswer(cmd, summary.getCapacity(), summary.getFreeSpace(), tInfo); + StoragePoolInfo poolInfo = answer.getPoolInfo(); + poolInfo.setName(summary.getName()); + String datastoreClusterPath = pool.getPath(); + int pathstartPosition = datastoreClusterPath.lastIndexOf('/'); + String datacenterName = datastoreClusterPath.substring(0, pathstartPosition+1); + String childPath = datacenterName + summary.getName(); + poolInfo.setHostPath(childPath); + String uuid = UUID.nameUUIDFromBytes(((pool.getHost() + childPath)).getBytes()).toString(); + poolInfo.setUuid(uuid); + poolInfo.setLocalPath(cmd.LOCAL_PATH_PREFIX + File.separator + uuid); + + answer.setPoolInfo(poolInfo); + answer.setPoolType(summary.getType()); + answer.setLocalDatastoreName(morDatastore.getValue()); + + childDsMo.setCustomFieldValue(CustomFieldConstants.CLOUD_UUID, uuid); + HypervisorHostHelper.createBaseFolderInDatastore(childDsMo, hyperHost); + + childDatastoresModifyStoragePoolAnswers.add(answer); + } + } else { + HypervisorHostHelper.createBaseFolderInDatastore(dsMo, hyperHost); - long capacity = summary.getCapacity(); - long available = summary.getFreeSpace(); + DatastoreSummary summary = dsMo.getDatastoreSummary(); + capacity = summary.getCapacity(); + available = summary.getFreeSpace(); + } Map tInfo = new HashMap<>(); ModifyStoragePoolAnswer answer = new ModifyStoragePoolAnswer(cmd, capacity, available, tInfo); + answer.setDatastoreClusterChildren(childDatastoresModifyStoragePoolAnswers); - if (cmd.getAdd() && pool.getType() == StoragePoolType.VMFS) { + if (cmd.getAdd() && (pool.getType() == StoragePoolType.VMFS || pool.getType() == StoragePoolType.PreSetup) && pool.getType() != StoragePoolType.DatastoreCluster) { + answer.setPoolType(dsMo.getDatastoreType()); answer.setLocalDatastoreName(morDatastore.getValue()); } @@ -5084,7 +5313,7 @@ public synchronized ManagedObjectReference prepareSecondaryDatastoreOnHost(Strin URI uri = new URI(storeUrl); VmwareHypervisorHost hyperHost = getHyperHost(getServiceContext()); - ManagedObjectReference morDatastore = hyperHost.mountDatastore(false, uri.getHost(), 0, uri.getPath(), storeName.replace("-", "")); + ManagedObjectReference morDatastore = hyperHost.mountDatastore(false, uri.getHost(), 0, uri.getPath(), storeName.replace("-", ""), false); if (morDatastore == null) throw new Exception("Unable to mount secondary storage on host. storeUrl: " + storeUrl); @@ -5096,7 +5325,7 @@ public synchronized ManagedObjectReference prepareSecondaryDatastoreOnSpecificHo String storeName = getSecondaryDatastoreUUID(storeUrl); URI uri = new URI(storeUrl); - ManagedObjectReference morDatastore = hyperHost.mountDatastore(false, uri.getHost(), 0, uri.getPath(), storeName.replace("-", "")); + ManagedObjectReference morDatastore = hyperHost.mountDatastore(false, uri.getHost(), 0, uri.getPath(), storeName.replace("-", ""), false); if (morDatastore == null) throw new Exception("Unable to mount secondary storage on host. storeUrl: " + storeUrl); @@ -5301,12 +5530,20 @@ protected Answer execute(GetStorageStatsCommand cmd) { ManagedObjectReference morDs = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, cmd.getStorageId()); if (morDs != null) { - DatastoreMO datastoreMo = new DatastoreMO(context, morDs); - DatastoreSummary summary = datastoreMo.getSummary(); - assert (summary != null); + long capacity = 0; + long free = 0; + if (cmd.getPooltype() == StoragePoolType.DatastoreCluster) { + StoragepodMO datastoreClusterMo = new StoragepodMO(getServiceContext(), morDs); + StoragePodSummary summary = datastoreClusterMo.getDatastoreClusterSummary(); + capacity = summary.getCapacity(); + free = summary.getFreeSpace(); + } else { + DatastoreMO datastoreMo = new DatastoreMO(context, morDs); + DatastoreSummary summary = datastoreMo.getDatastoreSummary(); + capacity = summary.getCapacity(); + free = summary.getFreeSpace(); + } - long capacity = summary.getCapacity(); - long free = summary.getFreeSpace(); long used = capacity - free; if (s_logger.isDebugEnabled()) { @@ -5314,7 +5551,7 @@ protected Answer execute(GetStorageStatsCommand cmd) { + ", capacity: " + toHumanReadableSize(capacity) + ", free: " + toHumanReadableSize(free) + ", used: " + toHumanReadableSize(used)); } - if (summary.getCapacity() <= 0) { + if (capacity <= 0) { s_logger.warn("Something is wrong with vSphere NFS datastore, rebooting ESX(ESXi) host should help"); } @@ -5836,7 +6073,7 @@ private List initializeLocalStorage() { dsMo.setCustomFieldValue(CustomFieldConstants.CLOUD_UUID, poolUuid); } - DatastoreSummary dsSummary = dsMo.getSummary(); + DatastoreSummary dsSummary = dsMo.getDatastoreSummary(); String address = hostMo.getHostName(); StoragePoolInfo pInfo = new StoragePoolInfo(poolUuid, address, dsMo.getMor().getValue(), "", StoragePoolType.VMFS, dsSummary.getCapacity(), dsSummary.getFreeSpace()); @@ -6552,6 +6789,8 @@ public boolean configure(String name, Map params) throws Configu CustomFieldsManagerMO cfmMo = new CustomFieldsManagerMO(context, context.getServiceContent().getCustomFieldsManager()); cfmMo.ensureCustomFieldDef("Datastore", CustomFieldConstants.CLOUD_UUID); + cfmMo.ensureCustomFieldDef("StoragePod", CustomFieldConstants.CLOUD_UUID); + if (_publicTrafficInfo != null && _publicTrafficInfo.getVirtualSwitchType() != VirtualSwitchType.StandardVirtualSwitch || _guestTrafficInfo != null && _guestTrafficInfo.getVirtualSwitchType() != VirtualSwitchType.StandardVirtualSwitch) { cfmMo.ensureCustomFieldDef("DistributedVirtualPortgroup", CustomFieldConstants.CLOUD_GC_DVP); @@ -6729,9 +6968,12 @@ public VmwareHypervisorHost getHyperHost(VmwareContext context, Command cmd) { @Override @DB - public String getWorkerName(VmwareContext context, Command cmd, int workerSequence) { + public String getWorkerName(VmwareContext context, Command cmd, int workerSequence, DatastoreMO dsMo) throws Exception { VmwareManager mgr = context.getStockObject(VmwareManager.CONTEXT_STOCK_NAME); String vmName = mgr.composeWorkerName(); + if (dsMo!= null && dsMo.getDatastoreType().equalsIgnoreCase("VVOL")) { + vmName = CustomFieldConstants.CLOUD_UUID + "-" + vmName; + } assert (cmd != null); context.getStockObject(VmwareManager.CONTEXT_STOCK_NAME); @@ -6921,14 +7163,8 @@ private List getUnmanageInstanceDisks(VirtualMachineMO instanceDisk.setDatastorePath(dsInfo.getNas().getRemotePath()); instanceDisk.setDatastoreType(dsInfo.getNas().getType()); } - } else if (info instanceof VmfsDatastoreInfo) { - VmfsDatastoreInfo dsInfo = (VmfsDatastoreInfo) info; - instanceDisk.setDatastoreName(dsInfo.getVmfs().getName()); - instanceDisk.setDatastoreType(dsInfo.getVmfs().getType()); } else { - String msg = String.format("Unmanaged instance disk: %s is on unsupported datastore %s", instanceDisk.getDiskId(), info.getClass().getSimpleName()); - s_logger.error(msg); - throw new Exception(msg); + instanceDisk.setDatastoreName(info.getName()); } } s_logger.info(vmMo.getName() + " " + disk.getDeviceInfo().getLabel() + " " + disk.getDeviceInfo().getSummary() + " " + disk.getDiskObjectId() + " " + disk.getCapacityInKB() + " " + instanceDisk.getController()); @@ -7185,4 +7421,18 @@ private Answer execute(PrepareUnmanageVMInstanceCommand cmd) { return new PrepareUnmanageVMInstanceAnswer(cmd, true, "OK"); } + + private Answer execute(ValidateVcenterDetailsCommand cmd) { + if (s_logger.isInfoEnabled()) { + s_logger.info("Executing resource ValidateVcenterDetailsCommand " + _gson.toJson(cmd)); + } + String vCenterServerAddress = cmd.getvCenterServerAddress(); + VmwareContext context = getServiceContext(); + + if (vCenterServerAddress.equals(context.getServerAddress())) { + return new Answer(cmd, true, "success"); + } else { + return new Answer(cmd, false, "Provided vCenter server address is invalid"); + } + } } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareSecondaryStorageResourceHandler.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareSecondaryStorageResourceHandler.java index b58253d66506..936ca15a4a5d 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareSecondaryStorageResourceHandler.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareSecondaryStorageResourceHandler.java @@ -41,6 +41,7 @@ import com.cloud.hypervisor.vmware.manager.VmwareStorageManagerImpl; import com.cloud.hypervisor.vmware.manager.VmwareStorageMount; import com.cloud.hypervisor.vmware.mo.ClusterMO; +import com.cloud.hypervisor.vmware.mo.DatastoreMO; import com.cloud.hypervisor.vmware.mo.HostMO; import com.cloud.hypervisor.vmware.mo.VmwareHostType; import com.cloud.hypervisor.vmware.mo.VmwareHypervisorHost; @@ -297,7 +298,7 @@ public VmwareHypervisorHost getHyperHost(VmwareContext context, Command cmd) { } @Override - public String getWorkerName(VmwareContext context, Command cmd, int workerSequence) { + public String getWorkerName(VmwareContext context, Command cmd, int workerSequence, DatastoreMO dsMo) { assert (cmd.getContextParam("worker") != null); assert (workerSequence < 2); diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageLayoutHelper.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageLayoutHelper.java index 9b2acbc5179e..6c66183240ca 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageLayoutHelper.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageLayoutHelper.java @@ -16,13 +16,19 @@ // under the License. package com.cloud.storage.resource; +import java.util.ArrayList; +import java.util.Arrays; import java.util.List; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; import org.apache.log4j.Logger; import com.cloud.hypervisor.vmware.mo.DatacenterMO; import com.cloud.hypervisor.vmware.mo.DatastoreFile; import com.cloud.hypervisor.vmware.mo.DatastoreMO; +import com.cloud.hypervisor.vmware.mo.HypervisorHostHelper; + import com.cloud.utils.Pair; /** @@ -30,32 +36,93 @@ * To provide helper methods to handle storage layout in one place * */ -public class VmwareStorageLayoutHelper { +public class VmwareStorageLayoutHelper implements Configurable { private static final Logger s_logger = Logger.getLogger(VmwareStorageLayoutHelper.class); + static final ConfigKey VsphereLinkedCloneExtensions = new ConfigKey("Hidden", String.class, + "vsphere.linked.clone.extensions", "delta.vmdk,sesparse.vmdk", + "Comma separated list of linked clone disk formats allowed to handle storage in VMware", true); + + public static String[] getVmdkFilePairDatastorePath(DatastoreMO dsMo, String vmName, String vmdkName, VmwareStorageLayoutType layoutType, boolean linkedVmdk) throws Exception { - String[] filePair = new String[2]; + int i = 0; + String[] vSphereLinkedCloneExtensions = VsphereLinkedCloneExtensions.value().trim().split("\\s*,\\s*"); + String[] fileNames; + if (linkedVmdk) + fileNames = new String[vSphereLinkedCloneExtensions.length + 1]; + else + fileNames = new String[2]; + switch (layoutType) { case VMWARE: assert (vmName != null && !vmName.isEmpty()); - filePair[0] = getVmwareDatastorePathFromVmdkFileName(dsMo, vmName, vmdkName + ".vmdk"); + fileNames[i] = getVmwareDatastorePathFromVmdkFileName(dsMo, vmName, vmdkName + ".vmdk"); - if (linkedVmdk) - filePair[1] = getVmwareDatastorePathFromVmdkFileName(dsMo, vmName, vmdkName + "-delta.vmdk"); + if (linkedVmdk) { + for (int j=0 ; j < vSphereLinkedCloneExtensions.length; j++) { + fileNames[++i] = getVmwareDatastorePathFromVmdkFileName(dsMo, vmName, String.format("%s-%s",vmdkName, vSphereLinkedCloneExtensions[j])); + } + } else - filePair[1] = getVmwareDatastorePathFromVmdkFileName(dsMo, vmName, vmdkName + "-flat.vmdk"); - return filePair; + fileNames[i+1] = getVmwareDatastorePathFromVmdkFileName(dsMo, vmName, vmdkName + "-flat.vmdk"); + return fileNames; case CLOUDSTACK_LEGACY: - filePair[0] = getLegacyDatastorePathFromVmdkFileName(dsMo, vmdkName + ".vmdk"); + fileNames[i] = getDatastorePathBaseFolderFromVmdkFileName(dsMo, vmdkName + ".vmdk"); - if (linkedVmdk) - filePair[1] = getLegacyDatastorePathFromVmdkFileName(dsMo, vmdkName + "-delta.vmdk"); - else - filePair[1] = getLegacyDatastorePathFromVmdkFileName(dsMo, vmdkName + "-flat.vmdk"); - return filePair; + if (linkedVmdk) { + for (int j=0 ; j < vSphereLinkedCloneExtensions.length; j++) { + fileNames[++i] = getDatastorePathBaseFolderFromVmdkFileName(dsMo, String.format("%s-%s",vmdkName, vSphereLinkedCloneExtensions[j])); + } + } else + fileNames[i+1] = getDatastorePathBaseFolderFromVmdkFileName(dsMo, vmdkName + "-flat.vmdk"); + return fileNames; + + default: + assert (false); + break; + } + + assert (false); + return null; + } + + public static String[] getVmdkFilePairManagedDatastorePath(DatastoreMO dsMo, String vmName, String vmdkName, VmwareStorageLayoutType layoutType, boolean linkedVmdk) + throws Exception { + + int i = 0; + String[] vSphereLinkedCloneExtensions = VsphereLinkedCloneExtensions.value().trim().split("\\s*,\\s*"); + String[] fileNames; + if (linkedVmdk) + fileNames = new String[vSphereLinkedCloneExtensions.length + 1]; + else + fileNames = new String[2]; + + switch (layoutType) { + case VMWARE: + assert (vmName != null && !vmName.isEmpty()); + fileNames[i] = getVmwareDatastorePathFromVmdkFileName(dsMo, vmName, vmdkName + ".vmdk"); + + if (linkedVmdk) { + for (int j=0 ; j < vSphereLinkedCloneExtensions.length; j++) { + fileNames[++i] = getVmwareDatastorePathFromVmdkFileName(dsMo, vmName, String.format("%s-%s",vmdkName, vSphereLinkedCloneExtensions[j])); + } + } else + fileNames[i+1] = getVmwareDatastorePathFromVmdkFileName(dsMo, vmName, vmdkName + "-flat.vmdk"); + return fileNames; + + case CLOUDSTACK_LEGACY: + fileNames[i] = getDeprecatedLegacyDatastorePathFromVmdkFileName(dsMo, vmdkName + ".vmdk"); + + if (linkedVmdk) { + for (int j=0 ; j < vSphereLinkedCloneExtensions.length; j++) { + fileNames[++i] = getDeprecatedLegacyDatastorePathFromVmdkFileName(dsMo, String.format("%s-%s",vmdkName, vSphereLinkedCloneExtensions[j])); + } + } else + fileNames[i+1] = getDeprecatedLegacyDatastorePathFromVmdkFileName(dsMo, vmdkName + "-flat.vmdk"); + return fileNames; default: assert (false); @@ -121,16 +188,20 @@ public static String syncVolumeToVmDefaultFolder(DatacenterMO dcMo, String vmNam syncVolumeToRootFolder(dcMo, ds, vmdkName, vmName, excludeFolders); } - if (ds.fileExists(vmdkFullCloneModeLegacyPair[1])) { - s_logger.info("sync " + vmdkFullCloneModeLegacyPair[1] + "->" + vmdkFullCloneModePair[1]); + for (int i=1; i" + vmdkFullCloneModePair[i]); - ds.moveDatastoreFile(vmdkFullCloneModeLegacyPair[1], dcMo.getMor(), ds.getMor(), vmdkFullCloneModePair[1], dcMo.getMor(), true); + ds.moveDatastoreFile(vmdkFullCloneModeLegacyPair[i], dcMo.getMor(), ds.getMor(), vmdkFullCloneModePair[i], dcMo.getMor(), true); + } } - if (ds.fileExists(vmdkLinkedCloneModeLegacyPair[1])) { - s_logger.info("sync " + vmdkLinkedCloneModeLegacyPair[1] + "->" + vmdkLinkedCloneModePair[1]); + for (int i=1; i" + vmdkLinkedCloneModePair[i]); - ds.moveDatastoreFile(vmdkLinkedCloneModeLegacyPair[1], dcMo.getMor(), ds.getMor(), vmdkLinkedCloneModePair[1], dcMo.getMor(), true); + ds.moveDatastoreFile(vmdkLinkedCloneModeLegacyPair[i], dcMo.getMor(), ds.getMor(), vmdkLinkedCloneModePair[i], dcMo.getMor(), true); + } } if (ds.fileExists(vmdkLinkedCloneModeLegacyPair[0])) { @@ -157,24 +228,22 @@ public static void syncVolumeToRootFolder(DatacenterMO dcMo, DatastoreMO ds, Str } DatastoreFile srcDsFile = new DatastoreFile(fileDsFullPath); - String companionFilePath = srcDsFile.getCompanionPath(vmdkName + "-flat.vmdk"); - if (ds.fileExists(companionFilePath)) { - String targetPath = getLegacyDatastorePathFromVmdkFileName(ds, vmdkName + "-flat.vmdk"); - - s_logger.info("Fixup folder-synchronization. move " + companionFilePath + " -> " + targetPath); - ds.moveDatastoreFile(companionFilePath, dcMo.getMor(), ds.getMor(), targetPath, dcMo.getMor(), true); - } - companionFilePath = srcDsFile.getCompanionPath(vmdkName + "-delta.vmdk"); - if (ds.fileExists(companionFilePath)) { - String targetPath = getLegacyDatastorePathFromVmdkFileName(ds, vmdkName + "-delta.vmdk"); + List vSphereFileExtensions = new ArrayList<>(Arrays.asList(VsphereLinkedCloneExtensions.value().trim().split("\\s*,\\s*"))); + // add flat file format to the above list + vSphereFileExtensions.add("flat.vmdk"); + for (String linkedCloneExtension : vSphereFileExtensions) { + String companionFilePath = srcDsFile.getCompanionPath(String.format("%s-%s",vmdkName, linkedCloneExtension)); + if (ds.fileExists(companionFilePath)) { + String targetPath = getDatastorePathBaseFolderFromVmdkFileName(ds, String.format("%s-%s",vmdkName, linkedCloneExtension)); - s_logger.info("Fixup folder-synchronization. move " + companionFilePath + " -> " + targetPath); - ds.moveDatastoreFile(companionFilePath, dcMo.getMor(), ds.getMor(), targetPath, dcMo.getMor(), true); + s_logger.info("Fixup folder-synchronization. move " + companionFilePath + " -> " + targetPath); + ds.moveDatastoreFile(companionFilePath, dcMo.getMor(), ds.getMor(), targetPath, dcMo.getMor(), true); + } } // move the identity VMDK file the last - String targetPath = getLegacyDatastorePathFromVmdkFileName(ds, vmdkName + ".vmdk"); + String targetPath = getDatastorePathBaseFolderFromVmdkFileName(ds, vmdkName + ".vmdk"); s_logger.info("Fixup folder-synchronization. move " + fileDsFullPath + " -> " + targetPath); ds.moveDatastoreFile(fileDsFullPath, dcMo.getMor(), ds.getMor(), targetPath, dcMo.getMor(), true); @@ -193,24 +262,22 @@ public static void moveVolumeToRootFolder(DatacenterMO dcMo, List detach s_logger.info("Check if we need to move " + fileFullDsPath + " to its root location"); DatastoreMO dsMo = new DatastoreMO(dcMo.getContext(), dcMo.findDatastore(file.getDatastoreName())); - if (dsMo.getMor() != null) { - DatastoreFile targetFile = new DatastoreFile(file.getDatastoreName(), file.getFileName()); + if (dsMo.getMor() != null && !dsMo.getDatastoreType().equalsIgnoreCase("VVOL")) { + DatastoreFile targetFile = new DatastoreFile(file.getDatastoreName(), HypervisorHostHelper.VSPHERE_DATASTORE_BASE_FOLDER, file.getFileName()); if (!targetFile.getPath().equalsIgnoreCase(file.getPath())) { s_logger.info("Move " + file.getPath() + " -> " + targetFile.getPath()); dsMo.moveDatastoreFile(file.getPath(), dcMo.getMor(), dsMo.getMor(), targetFile.getPath(), dcMo.getMor(), true); - String pairSrcFilePath = file.getCompanionPath(file.getFileBaseName() + "-flat.vmdk"); - String pairTargetFilePath = targetFile.getCompanionPath(file.getFileBaseName() + "-flat.vmdk"); - if (dsMo.fileExists(pairSrcFilePath)) { - s_logger.info("Move " + pairSrcFilePath + " -> " + pairTargetFilePath); - dsMo.moveDatastoreFile(pairSrcFilePath, dcMo.getMor(), dsMo.getMor(), pairTargetFilePath, dcMo.getMor(), true); - } - - pairSrcFilePath = file.getCompanionPath(file.getFileBaseName() + "-delta.vmdk"); - pairTargetFilePath = targetFile.getCompanionPath(file.getFileBaseName() + "-delta.vmdk"); - if (dsMo.fileExists(pairSrcFilePath)) { - s_logger.info("Move " + pairSrcFilePath + " -> " + pairTargetFilePath); - dsMo.moveDatastoreFile(pairSrcFilePath, dcMo.getMor(), dsMo.getMor(), pairTargetFilePath, dcMo.getMor(), true); + List vSphereFileExtensions = new ArrayList<>(Arrays.asList(VsphereLinkedCloneExtensions.value().trim().split("\\s*,\\s*"))); + // add flat file format to the above list + vSphereFileExtensions.add("flat.vmdk"); + for (String linkedCloneExtension : vSphereFileExtensions) { + String pairSrcFilePath = file.getCompanionPath(String.format("%s-%s", file.getFileBaseName(), linkedCloneExtension)); + String pairTargetFilePath = targetFile.getCompanionPath(String.format("%s-%s", file.getFileBaseName(), linkedCloneExtension)); + if (dsMo.fileExists(pairSrcFilePath)) { + s_logger.info("Move " + pairSrcFilePath + " -> " + pairTargetFilePath); + dsMo.moveDatastoreFile(pairSrcFilePath, dcMo.getMor(), dsMo.getMor(), pairTargetFilePath, dcMo.getMor(), true); + } } } } else { @@ -287,32 +354,49 @@ public static void deleteVolumeVmdkFiles(DatastoreMO dsMo, String volumeName, Da s_logger.warn("Unable to locate VMDK file: " + fileName); } - fileName = volumeName + "-flat.vmdk"; - fileFullPath = getLegacyDatastorePathFromVmdkFileName(dsMo, fileName); - if (!dsMo.fileExists(fileFullPath)) - fileFullPath = dsMo.searchFileInSubFolders(fileName, false, excludeFolders); - if (fileFullPath != null) { - dsMo.deleteFile(fileFullPath, dcMo.getMor(), true, excludeFolders); - } else { - s_logger.warn("Unable to locate VMDK file: " + fileName); + List vSphereFileExtensions = new ArrayList<>(Arrays.asList(VsphereLinkedCloneExtensions.value().trim().split("\\s*,\\s*"))); + vSphereFileExtensions.add("flat.vmdk"); + for (String linkedCloneExtension : vSphereFileExtensions) { + fileFullPath = getLegacyDatastorePathFromVmdkFileName(dsMo, String.format("%s-%s", volumeName, linkedCloneExtension)); + if (!dsMo.fileExists(fileFullPath)) + fileFullPath = dsMo.searchFileInSubFolders(String.format("%s-%s", volumeName, linkedCloneExtension), false, excludeFolders); + if (fileFullPath != null) { + dsMo.deleteFile(fileFullPath, dcMo.getMor(), true, excludeFolders); + } else { + s_logger.warn("Unable to locate VMDK file: " + String.format("%s-%s", volumeName, linkedCloneExtension)); + } } + } - fileName = volumeName + "-delta.vmdk"; - fileFullPath = getLegacyDatastorePathFromVmdkFileName(dsMo, fileName); - if (!dsMo.fileExists(fileFullPath)) - fileFullPath = dsMo.searchFileInSubFolders(fileName, false, excludeFolders); - if (fileFullPath != null) { - dsMo.deleteFile(fileFullPath, dcMo.getMor(), true, excludeFolders); - } else { - s_logger.warn("Unable to locate VMDK file: " + fileName); + //This method call is for the volumes which actually exists + public static String getLegacyDatastorePathFromVmdkFileName(DatastoreMO dsMo, String vmdkFileName) throws Exception { + String vmdkDatastorePath = String.format("[%s] %s/%s", dsMo.getName(), HypervisorHostHelper.VSPHERE_DATASTORE_BASE_FOLDER, vmdkFileName); + if (!dsMo.fileExists(vmdkDatastorePath)) { + vmdkDatastorePath = getDeprecatedLegacyDatastorePathFromVmdkFileName(dsMo, vmdkFileName); } + return vmdkDatastorePath; } - public static String getLegacyDatastorePathFromVmdkFileName(DatastoreMO dsMo, String vmdkFileName) throws Exception { + //This method call is for the volumes to be created or can also be for volumes already exists + public static String getDatastorePathBaseFolderFromVmdkFileName(DatastoreMO dsMo, String vmdkFileName) throws Exception { + return String.format("[%s] %s/%s", dsMo.getName(), HypervisorHostHelper.VSPHERE_DATASTORE_BASE_FOLDER, vmdkFileName); + } + + public static String getDeprecatedLegacyDatastorePathFromVmdkFileName(DatastoreMO dsMo, String vmdkFileName) throws Exception { return String.format("[%s] %s", dsMo.getName(), vmdkFileName); } public static String getVmwareDatastorePathFromVmdkFileName(DatastoreMO dsMo, String vmName, String vmdkFileName) throws Exception { return String.format("[%s] %s/%s", dsMo.getName(), vmName, vmdkFileName); } + + @Override + public String getConfigComponentName() { + return VmwareStorageLayoutHelper.class.getSimpleName(); + } + + @Override + public ConfigKey[] getConfigKeys() { + return new ConfigKey[] {VsphereLinkedCloneExtensions}; + } } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java index e5fae17ed515..96061786269f 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java @@ -34,9 +34,14 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import com.cloud.hypervisor.vmware.mo.VirtualStorageObjectManagerMO; +import com.vmware.vim25.BaseConfigInfoDiskFileBackingInfo; +import com.vmware.vim25.VStorageObject; +import com.vmware.vim25.VirtualDiskType; import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand; import org.apache.cloudstack.storage.command.AttachAnswer; import org.apache.cloudstack.storage.command.AttachCommand; +import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplainceCommand; import org.apache.cloudstack.storage.command.CopyCmdAnswer; import org.apache.cloudstack.storage.command.CopyCommand; import org.apache.cloudstack.storage.command.CreateObjectAnswer; @@ -394,7 +399,7 @@ private String cleanUpDatastore(Command cmd, HostDatastoreSystemMO hostDatastore // If vmName is not null, then move all VMDK files out of this folder to the root folder and then delete the folder named vmName. if (vmName != null) { - String workerVmName = hostService.getWorkerName(context, cmd, 0); + String workerVmName = hostService.getWorkerName(context, cmd, 0, dsMo); VirtualMachineMO vmMo = HypervisorHostHelper.createWorkerVM(hyperHost, dsMo, workerVmName); @@ -479,9 +484,9 @@ private String getOVFFilePath(String srcOVAFileName) { private Pair copyTemplateFromSecondaryToPrimary(VmwareHypervisorHost hyperHost, DatastoreMO datastoreMo, String secondaryStorageUrl, String templatePathAtSecondaryStorage, String templateName, String templateUuid, - boolean createSnapshot, String nfsVersion) throws Exception { + boolean createSnapshot, String nfsVersion, String configuration) throws Exception { s_logger.info("Executing copyTemplateFromSecondaryToPrimary. secondaryStorage: " + secondaryStorageUrl + ", templatePathAtSecondaryStorage: " + - templatePathAtSecondaryStorage + ", templateName: " + templateName); + templatePathAtSecondaryStorage + ", templateName: " + templateName + ", configuration: " + configuration); String secondaryMountPoint = mountService.getMountPoint(secondaryStorageUrl, nfsVersion); s_logger.info("Secondary storage mount point: " + secondaryMountPoint); @@ -512,11 +517,16 @@ private Pair copyTemplateFromSecondaryToPrimary(VmwareHy throw new Exception(msg); } - String vmName = templateUuid; - hyperHost.importVmFromOVF(srcFileName, vmName, datastoreMo, "thin"); + if (datastoreMo.getDatastoreType().equalsIgnoreCase("VVOL")) { + templateUuid = CustomFieldConstants.CLOUD_UUID + "-" + templateUuid; + } - VirtualMachineMO vmMo = hyperHost.findVmOnHyperHost(vmName); VmConfigInfo vAppConfig; + if (s_logger.isDebugEnabled()) { + s_logger.debug(String.format("Deploying OVF template %s with configuration %s", templateName, configuration)); + } + hyperHost.importVmFromOVF(srcFileName, templateUuid, datastoreMo, "thin", configuration); + VirtualMachineMO vmMo = hyperHost.findVmOnHyperHost(templateUuid); if (vmMo == null) { String msg = "Failed to import OVA template. secondaryStorage: " + secondaryStorageUrl + ", templatePathAtSecondaryStorage: " + templatePathAtSecondaryStorage + @@ -572,6 +582,7 @@ public Answer copyTemplateToPrimaryStorage(CopyCommand cmd) { DataTO destData = cmd.getDestTO(); DataStoreTO destStore = destData.getDataStore(); DataStoreTO primaryStore = destStore; + String configurationId = ((TemplateObjectTO) destData).getDeployAsIsConfiguration(); String secondaryStorageUrl = nfsImageStore.getUrl(); @@ -629,51 +640,56 @@ public Answer copyTemplateToPrimaryStorage(CopyCommand cmd) { try { String storageUuid = managed ? managedStoragePoolName : primaryStore.getUuid(); - String templateUuidName = deriveTemplateUuidOnHost(hyperHost, storageUuid, templateInfo.second()); + + // Generate a new template uuid if the template is marked as deploy-as-is, + // as it supports multiple configurations + String templateUuidName = template.isDeployAsIs() ? + UUID.randomUUID().toString() : + deriveTemplateUuidOnHost(hyperHost, storageUuid, templateInfo.second()); + DatacenterMO dcMo = new DatacenterMO(context, hyperHost.getHyperHostDatacenter()); VirtualMachineMO templateMo = VmwareHelper.pickOneVmOnRunningHost(dcMo.findVmByNameAndLabel(templateUuidName), true); Pair vmInfo = null; + final ManagedObjectReference morDs; + if (managed) { + morDs = prepareManagedDatastore(context, hyperHost, null, managedStoragePoolName, storageHost, storagePort, + chapInitiatorUsername, chapInitiatorSecret, chapTargetUsername, chapTargetSecret); + } + else { + morDs = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, storageUuid); + } + assert (morDs != null); + dsMo = new DatastoreMO(context, morDs); + if (templateMo == null) { if (s_logger.isInfoEnabled()) { s_logger.info("Template " + templateInfo.second() + " is not setup yet. Set up template from secondary storage with uuid name: " + templateUuidName); } - final ManagedObjectReference morDs; - - if (managed) { - morDs = prepareManagedDatastore(context, hyperHost, null, managedStoragePoolName, storageHost, storagePort, - chapInitiatorUsername, chapInitiatorSecret, chapTargetUsername, chapTargetSecret); - } - else { - morDs = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, storageUuid); - } - - assert (morDs != null); - - dsMo = new DatastoreMO(context, morDs); - if (managed) { vmInfo = copyTemplateFromSecondaryToPrimary(hyperHost, dsMo, secondaryStorageUrl, templateInfo.first(), templateInfo.second(), - managedStoragePoolRootVolumeName, false, _nfsVersion); + managedStoragePoolRootVolumeName, false, _nfsVersion, configurationId); VirtualMachineMO vmMo = vmInfo.first(); vmMo.unregisterVm(); - String[] vmwareLayoutFilePair = VmwareStorageLayoutHelper.getVmdkFilePairDatastorePath(dsMo, managedStoragePoolRootVolumeName, + String[] vmwareLayoutFilePair = VmwareStorageLayoutHelper.getVmdkFilePairManagedDatastorePath(dsMo, managedStoragePoolRootVolumeName, managedStoragePoolRootVolumeName, VmwareStorageLayoutType.VMWARE, false); - String[] legacyCloudStackLayoutFilePair = VmwareStorageLayoutHelper.getVmdkFilePairDatastorePath(dsMo, null, + String[] legacyCloudStackLayoutFilePair = VmwareStorageLayoutHelper.getVmdkFilePairManagedDatastorePath(dsMo, null, managedStoragePoolRootVolumeName, VmwareStorageLayoutType.CLOUDSTACK_LEGACY, false); dsMo.moveDatastoreFile(vmwareLayoutFilePair[0], dcMo.getMor(), dsMo.getMor(), legacyCloudStackLayoutFilePair[0], dcMo.getMor(), true); - dsMo.moveDatastoreFile(vmwareLayoutFilePair[1], dcMo.getMor(), dsMo.getMor(), legacyCloudStackLayoutFilePair[1], dcMo.getMor(), true); + for (int i=1; i template.getSize() ? true : _fullCloneFlag; - } - if (!_fullCloneFlag) { - createVMLinkedClone(vmTemplate, dcMo, vmdkName, morDatastore, morPool); } else { - createVMFullClone(vmTemplate, dcMo, dsMo, vmdkName, morDatastore, morPool); - } - - vmMo = new ClusterMO(context, morCluster).findVmOnHyperHost(vmdkName); - assert (vmMo != null); - - vmdkFileBaseName = vmMo.getVmdkFileBaseNames().get(0); - s_logger.info("Move volume out of volume-wrapper VM " + vmdkFileBaseName); - String[] vmwareLayoutFilePair = VmwareStorageLayoutHelper.getVmdkFilePairDatastorePath(dsMo, vmdkName, vmdkFileBaseName, VmwareStorageLayoutType.VMWARE, !_fullCloneFlag); - String[] legacyCloudStackLayoutFilePair = VmwareStorageLayoutHelper.getVmdkFilePairDatastorePath(dsMo, vmdkName, vmdkFileBaseName, VmwareStorageLayoutType.CLOUDSTACK_LEGACY, !_fullCloneFlag); - - dsMo.moveDatastoreFile(vmwareLayoutFilePair[0], dcMo.getMor(), dsMo.getMor(), legacyCloudStackLayoutFilePair[0], dcMo.getMor(), true); - dsMo.moveDatastoreFile(vmwareLayoutFilePair[1], dcMo.getMor(), dsMo.getMor(), legacyCloudStackLayoutFilePair[1], dcMo.getMor(), true); - - s_logger.info("detach disks from volume-wrapper VM " + vmdkName); - vmMo.detachAllDisks(); - - s_logger.info("destroy volume-wrapper VM " + vmdkName); - vmMo.destroy(); - - String srcFile = dsMo.getDatastorePath(vmdkName, true); - - dsMo.deleteFile(srcFile, dcMo.getMor(), true, searchExcludedFolders); - - if (dsMo.folderExists(String.format("[%s]", dsMo.getName()), vmdkName)) { - dsMo.deleteFolder(srcFile, dcMo.getMor()); + String templatePath = template.getPath(); + VirtualMachineMO vmTemplate = VmwareHelper.pickOneVmOnRunningHost(dcMo.findVmByNameAndLabel(templatePath), true); + if (vmTemplate == null) { + s_logger.warn("Template host in vSphere is not in connected state, request template reload"); + return new CopyCmdAnswer("Template host in vSphere is not in connected state, request template reload"); + } + if (dsMo.getDatastoreType().equalsIgnoreCase("VVOL") && volume.getVolumeType() == Volume.Type.ROOT) { + vmdkFileBaseName = cloneVMwithVMname(context, hyperHost, template, vmTemplate, volume, dcMo, dsMo); + } else { + vmdkFileBaseName = createVMFolderWithVMName(context, hyperHost, template, vmTemplate, volume, dcMo, dsMo, searchExcludedFolders); + } } - } - // restoreVM - move the new ROOT disk into corresponding VM folder - VirtualMachineMO restoreVmMo = dcMo.findVm(volume.getVmName()); - if (restoreVmMo != null) { - String vmNameInVcenter = restoreVmMo.getName(); // VM folder name in datastore will be VM's name in vCenter. - if (dsMo.folderExists(String.format("[%s]", dsMo.getName()), vmNameInVcenter)) { - VmwareStorageLayoutHelper.syncVolumeToVmDefaultFolder(dcMo, vmNameInVcenter, dsMo, vmdkFileBaseName, searchExcludedFolders); + // restoreVM - move the new ROOT disk into corresponding VM folder + VirtualMachineMO restoreVmMo = dcMo.findVm(volume.getVmName()); + if (restoreVmMo != null) { + String vmNameInVcenter = restoreVmMo.getName(); // VM folder name in datastore will be VM's name in vCenter. + if (dsMo.folderExists(String.format("[%s]", dsMo.getName()), vmNameInVcenter)) { + VmwareStorageLayoutHelper.syncVolumeToVmDefaultFolder(dcMo, vmNameInVcenter, dsMo, vmdkFileBaseName, searchExcludedFolders); + } } } VolumeObjectTO newVol = new VolumeObjectTO(); newVol.setPath(vmdkFileBaseName); - if (template.getSize() != null){ + if (template.isDeployAsIs()) { + newVol.setSize(volume.getSize()); + } else if (template.getSize() != null) { newVol.setSize(template.getSize()); - } - else { + } else { newVol.setSize(volume.getSize()); } return new CopyCmdAnswer(newVol); @@ -891,6 +886,92 @@ public Answer cloneVolumeFromBaseTemplate(CopyCommand cmd) { } } + private String cloneVMwithVMname(VmwareContext context, VmwareHypervisorHost hyperHost, TemplateObjectTO template, + VirtualMachineMO vmTemplate, VolumeObjectTO volume, DatacenterMO dcMo, DatastoreMO dsMo) throws Exception { + ManagedObjectReference morDatastore = dsMo.getMor(); + ManagedObjectReference morPool = hyperHost.getHyperHostOwnerResourcePool(); + ManagedObjectReference morCluster = hyperHost.getHyperHostCluster(); + if (template.getSize() != null) { + _fullCloneFlag = volume.getSize() > template.getSize() ? true : _fullCloneFlag; + } + if (!_fullCloneFlag) { + createVMLinkedClone(vmTemplate, dcMo, volume.getVmName(), morDatastore, morPool); + } else { + createVMFullClone(vmTemplate, dcMo, dsMo, volume.getVmName(), morDatastore, morPool); + } + + VirtualMachineMO vmMo = new ClusterMO(context, morCluster).findVmOnHyperHost(volume.getVmName()); + assert (vmMo != null); + + return vmMo.getVmdkFileBaseNames().get(0); + } + + private String createVMFolderWithVMName(VmwareContext context, VmwareHypervisorHost hyperHost, TemplateObjectTO template, + VirtualMachineMO vmTemplate, VolumeObjectTO volume, DatacenterMO dcMo, DatastoreMO dsMo, + String searchExcludedFolders) throws Exception { + ManagedObjectReference morDatastore = dsMo.getMor(); + ManagedObjectReference morPool = hyperHost.getHyperHostOwnerResourcePool(); + ManagedObjectReference morCluster = hyperHost.getHyperHostCluster(); + String vmdkName = volume.getName(); + if (template.getSize() != null){ + _fullCloneFlag = volume.getSize() > template.getSize() ? true : _fullCloneFlag; + } + if (!_fullCloneFlag) { + createVMLinkedClone(vmTemplate, dcMo, vmdkName, morDatastore, morPool); + } else { + createVMFullClone(vmTemplate, dcMo, dsMo,vmdkName, morDatastore, morPool); + } + + VirtualMachineMO vmMo = new ClusterMO(context, morCluster).findVmOnHyperHost(vmdkName); + assert (vmMo != null); + + String vmdkFileBaseName = vmMo.getVmdkFileBaseNames().get(0); + s_logger.info("Move volume out of volume-wrapper VM " + vmdkFileBaseName); + String[] vmwareLayoutFilePair = VmwareStorageLayoutHelper.getVmdkFilePairDatastorePath(dsMo, vmdkName, vmdkFileBaseName, VmwareStorageLayoutType.VMWARE, !_fullCloneFlag); + String[] legacyCloudStackLayoutFilePair = VmwareStorageLayoutHelper.getVmdkFilePairDatastorePath(dsMo, vmdkName, vmdkFileBaseName, VmwareStorageLayoutType.CLOUDSTACK_LEGACY, !_fullCloneFlag); + + for (int i=0; i template.getSize() || _fullCloneFlag; + } + if (!_fullCloneFlag) { + createVMLinkedClone(vmMo, dcMo, cloneName, morDatastore, morPool); + } else { + createVMFullClone(vmMo, dcMo, dsMo, cloneName, morDatastore, morPool); + } + } + private Pair copyVolumeFromSecStorage(VmwareHypervisorHost hyperHost, String srcVolumePath, DatastoreMO dsMo, String secStorageUrl, long wait, String nfsVersion) throws Exception { String volumeFolder; @@ -905,7 +986,7 @@ private Pair copyVolumeFromSecStorage(VmwareHypervisorHost hyper volumeName = srcVolumePath.substring(index + 1); } - String newVolume = VmwareHelper.getVCenterSafeUuid(); + String newVolume = VmwareHelper.getVCenterSafeUuid(dsMo); restoreVolumeFromSecStorage(hyperHost, dsMo, newVolume, secStorageUrl, volumeFolder, volumeName, wait, nfsVersion); return new Pair<>(volumeFolder, newVolume); @@ -944,7 +1025,7 @@ public Answer copyVolumeFromImageCacheToPrimary(CopyCommand cmd) { if (morDatastore == null) { URI uri = new URI(destStore.getUrl()); - morDatastore = hyperHost.mountDatastore(false, uri.getHost(), 0, uri.getPath(), destStore.getUuid().replace("-", "")); + morDatastore = hyperHost.mountDatastore(false, uri.getHost(), 0, uri.getPath(), destStore.getUuid().replace("-", ""), true); if (morDatastore == null) { throw new Exception("Unable to mount storage pool on host. storeUrl: " + uri.getHost() + ":/" + uri.getPath()); @@ -1008,11 +1089,9 @@ private Pair copyVolumeToSecStorage(VmwareHostService hostServic workerVm.attachDisk(new String[] {datastoreVolumePath}, morDs); vmMo = workerVm; clonedWorkerVMNeeded = false; - } else { - vmMo.createSnapshot(exportName, "Temporary snapshot for copy-volume command", false, false); } - exportVolumeToSecondaryStorage(vmMo, volumePath, secStorageUrl, destVolumePath, exportName, hostService.getWorkerName(hyperHost.getContext(), cmd, 1), _nfsVersion, clonedWorkerVMNeeded); + exportVolumeToSecondaryStorage(hyperHost.getContext(), vmMo, hyperHost, volumePath, secStorageUrl, destVolumePath, exportName, hostService.getWorkerName(hyperHost.getContext(), cmd, 1, null), _nfsVersion, clonedWorkerVMNeeded); return new Pair<>(destVolumePath, exportName); } finally { @@ -1043,7 +1122,7 @@ public Answer copyVolumeFromPrimaryToSecondary(CopyCommand cmd) { result = copyVolumeToSecStorage(hostService, hyperHost, cmd, vmName, primaryStorage.getUuid(), srcVolume.getPath(), destVolume.getPath(), destStore.getUrl(), - hostService.getWorkerName(context, cmd, 0)); + hostService.getWorkerName(context, cmd, 0, null)); VolumeObjectTO newVolume = new VolumeObjectTO(); newVolume.setPath(result.first() + File.separator + result.second()); return new CopyCmdAnswer(newVolume); @@ -1097,7 +1176,7 @@ private void postCreatePrivateTemplate(String installFullPath, long templateId, } } - private Ternary createTemplateFromVolume(VirtualMachineMO vmMo, String installPath, long templateId, String templateUniqueName, + private Ternary createTemplateFromVolume(VmwareContext context, VirtualMachineMO vmMo, VmwareHypervisorHost hyperHost, String installPath, long templateId, String templateUniqueName, String secStorageUrl, String volumePath, String workerVmName, String nfsVersion) throws Exception { String secondaryMountPoint = mountService.getMountPoint(secStorageUrl, nfsVersion); @@ -1124,6 +1203,12 @@ private Ternary createTemplateFromVolume(VirtualMachineMO vm throw new Exception(msg); } + DatacenterMO dcMo = new DatacenterMO(context, hyperHost.getHyperHostDatacenter()); + ManagedObjectReference morPool = hyperHost.getHyperHostOwnerResourcePool(); + vmMo.createFullCloneWithSpecificDisk(templateUniqueName, dcMo.getVmFolder(), morPool, VmwareHelper.getDiskDeviceDatastore(volumeDeviceInfo.first()), volumeDeviceInfo); + clonedVm = dcMo.findVm(templateUniqueName); + + /* FR41 THIS IS OLD way of creating template using snapshot if (!vmMo.createSnapshot(templateUniqueName, "Temporary snapshot for template creation", false, false)) { String msg = "Unable to take snapshot for creating template from volume. volume path: " + volumePath; s_logger.error(msg); @@ -1134,7 +1219,7 @@ private Ternary createTemplateFromVolume(VirtualMachineMO vm Pair cloneResult = vmMo.cloneFromCurrentSnapshot(workerVmName, 0, 4, volumeDeviceInfo.second(), VmwareHelper.getDiskDeviceDatastore(volumeDeviceInfo.first())); clonedVm = cloneResult.first(); - + * */ clonedVm.exportVm(secondaryMountPoint + "/" + installPath, templateUniqueName, false, false); // Get VMDK filename @@ -1167,8 +1252,6 @@ private Ternary createTemplateFromVolume(VirtualMachineMO vm clonedVm.detachAllDisks(); clonedVm.destroy(); } - - vmMo.removeSnapshot(templateUniqueName, false); } } @@ -1215,9 +1298,8 @@ public Answer createTemplateFromVolume(CopyCommand cmd) { } Ternary result = - createTemplateFromVolume(vmMo, template.getPath(), template.getId(), template.getName(), secondaryStoragePoolURL, volumePath, - hostService.getWorkerName(context, cmd, 0), _nfsVersion); - + createTemplateFromVolume(context, vmMo, hyperHost, template.getPath(), template.getId(), template.getName(), secondaryStoragePoolURL, volumePath, + hostService.getWorkerName(context, cmd, 0, null), _nfsVersion); TemplateObjectTO newTemplate = new TemplateObjectTO(); newTemplate.setPath(result.first()); newTemplate.setFormat(ImageFormat.OVA); @@ -1525,10 +1607,9 @@ private void exportManagedStorageSnapshotToTemplate(CopyCommand cmd, String inst VmwareContext context = hostService.getServiceContext(cmd); VmwareHypervisorHost hyperHost = hostService.getHyperHost(context, null); - String workerVMName = hostService.getWorkerName(context, cmd, 0); - ManagedObjectReference dsMor = hyperHost.findDatastoreByName(dsFile.getDatastoreName()); DatastoreMO dsMo = new DatastoreMO(context, dsMor); + String workerVMName = hostService.getWorkerName(context, cmd, 0, dsMo); VirtualMachineMO workerVM = HypervisorHostHelper.createWorkerVM(hyperHost, dsMo, workerVMName); @@ -1668,7 +1749,7 @@ public Answer createTemplateFromSnapshot(CopyCommand cmd) { } // return Pair - private Pair exportVolumeToSecondaryStorage(VirtualMachineMO vmMo, String volumePath, String secStorageUrl, String secStorageDir, + private Pair exportVolumeToSecondaryStorage(VmwareContext context, VirtualMachineMO vmMo, VmwareHypervisorHost hyperHost, String volumePath, String secStorageUrl, String secStorageDir, String exportName, String workerVmName, String nfsVersion, boolean clonedWorkerVMNeeded) throws Exception { String secondaryMountPoint = mountService.getMountPoint(secStorageUrl, nfsVersion); @@ -1699,13 +1780,19 @@ private Pair exportVolumeToSecondaryStorage(VirtualMachineMO v String disks[] = vmMo.getCurrentSnapshotDiskChainDatastorePaths(diskDevice); if (clonedWorkerVMNeeded) { // 4 MB is the minimum requirement for VM memory in VMware - Pair cloneResult = - vmMo.cloneFromCurrentSnapshot(workerVmName, 0, 4, diskDevice, VmwareHelper.getDiskDeviceDatastore(volumeDeviceInfo.first())); - clonedVm = cloneResult.first(); - clonedVm.exportVm(exportPath, exportName, false, false); - } else { - vmMo.exportVm(exportPath, exportName, false, false); + DatacenterMO dcMo = new DatacenterMO(context, hyperHost.getHyperHostDatacenter()); + ManagedObjectReference morPool = hyperHost.getHyperHostOwnerResourcePool(); + vmMo.createFullCloneWithSpecificDisk(exportName, dcMo.getVmFolder(), morPool, VmwareHelper.getDiskDeviceDatastore(volumeDeviceInfo.first()), volumeDeviceInfo); + clonedVm = dcMo.findVm(exportName); + if (clonedVm == null) { + String msg = "Failed to clone VM. volume path: " + volumePath; + s_logger.error(msg); + throw new Exception(msg); + } + vmMo = clonedVm; } + vmMo.exportVm(exportPath, exportName, false, false); + return new Pair<>(diskDevice, disks); } finally { if (clonedVm != null) { @@ -1716,12 +1803,12 @@ private Pair exportVolumeToSecondaryStorage(VirtualMachineMO v } // Ternary - private Ternary backupSnapshotToSecondaryStorage(VirtualMachineMO vmMo, String installPath, String volumePath, String snapshotUuid, + private Ternary backupSnapshotToSecondaryStorage(VmwareContext context, VirtualMachineMO vmMo, VmwareHypervisorHost hypervisorHost, String installPath, String volumePath, String snapshotUuid, String secStorageUrl, String prevSnapshotUuid, String prevBackupUuid, String workerVmName, String nfsVersion) throws Exception { String backupUuid = UUID.randomUUID().toString(); - Pair snapshotInfo = exportVolumeToSecondaryStorage(vmMo, volumePath, secStorageUrl, installPath, backupUuid, workerVmName, nfsVersion, true); + Pair snapshotInfo = exportVolumeToSecondaryStorage(context, vmMo, hypervisorHost, volumePath, secStorageUrl, installPath, backupUuid, workerVmName, nfsVersion, true); return new Ternary<>(backupUuid, snapshotInfo.first(), snapshotInfo.second()); } @@ -1776,27 +1863,23 @@ public Answer backupSnapshot(CopyCommand cmd) { } if(vmMo == null) { dsMo = new DatastoreMO(hyperHost.getContext(), morDs); - workerVMName = hostService.getWorkerName(context, cmd, 0); + workerVMName = hostService.getWorkerName(context, cmd, 0, dsMo); vmMo = HypervisorHostHelper.createWorkerVM(hyperHost, dsMo, workerVMName); if (vmMo == null) { throw new Exception("Failed to find the newly create or relocated VM. vmName: " + workerVMName); } workerVm = vmMo; // attach volume to worker VM - String datastoreVolumePath = dsMo.getDatastorePath(volumePath + ".vmdk"); + String datastoreVolumePath = VmwareStorageLayoutHelper.getLegacyDatastorePathFromVmdkFileName(dsMo, volumePath + ".vmdk"); vmMo.attachDisk(new String[] { datastoreVolumePath }, morDs); } else { s_logger.info("Using owner VM " + vmName + " for snapshot operation"); hasOwnerVm = true; } - if (!vmMo.createSnapshot(snapshotUuid, "Snapshot taken for " + srcSnapshot.getName(), false, false)) { - throw new Exception("Failed to take snapshot " + srcSnapshot.getName() + " on vm: " + vmName); - } - backupResult = - backupSnapshotToSecondaryStorage(vmMo, destSnapshot.getPath(), srcSnapshot.getVolume().getPath(), snapshotUuid, secondaryStorageUrl, - prevSnapshotUuid, prevBackupUuid, hostService.getWorkerName(context, cmd, 1), _nfsVersion); + backupSnapshotToSecondaryStorage(context, vmMo, hyperHost, destSnapshot.getPath(), srcSnapshot.getVolume().getPath(), snapshotUuid, secondaryStorageUrl, + prevSnapshotUuid, prevBackupUuid, hostService.getWorkerName(context, cmd, 1, null), _nfsVersion); snapshotBackupUuid = backupResult.first(); success = (snapshotBackupUuid != null); @@ -1876,7 +1959,7 @@ public Answer backupSnapshot(CopyCommand cmd) { } } } else { - s_logger.error("Can not find the snapshot we just used ?!"); + s_logger.info("No snapshots created to be deleted!"); } } @@ -1978,17 +2061,29 @@ private Answer attachVolume(Command cmd, DiskTO disk, boolean isAttach, boolean if (isManaged) { datastoreVolumePath = dsMo.getDatastorePath((vmdkPath != null ? vmdkPath : dsMo.getName()) + ".vmdk"); } else { - datastoreVolumePath = VmwareStorageLayoutHelper.syncVolumeToVmDefaultFolder(dsMo.getOwnerDatacenter().first(), vmName, dsMo, volumeTO.getPath(), VmwareManager.s_vmwareSearchExcludeFolder.value()); + if (dsMo.getDatastoreType().equalsIgnoreCase("VVOL")) { + datastoreVolumePath = VmwareStorageLayoutHelper.getLegacyDatastorePathFromVmdkFileName(dsMo, volumeTO.getPath() + ".vmdk"); + if (!dsMo.fileExists(datastoreVolumePath)) { + datastoreVolumePath = VmwareStorageLayoutHelper.getVmwareDatastorePathFromVmdkFileName(dsMo, vmName, volumeTO.getPath() + ".vmdk"); + } + if (!dsMo.fileExists(datastoreVolumePath)) { + datastoreVolumePath = dsMo.searchFileInSubFolders(volumeTO.getPath() + ".vmdk", true, null); + } + } else { + datastoreVolumePath = VmwareStorageLayoutHelper.syncVolumeToVmDefaultFolder(dsMo.getOwnerDatacenter().first(), vmName, dsMo, volumeTO.getPath(), VmwareManager.s_vmwareSearchExcludeFolder.value()); + } } } else { if (isManaged) { datastoreVolumePath = dsMo.getDatastorePath((vmdkPath != null ? vmdkPath : dsMo.getName()) + ".vmdk"); } else { datastoreVolumePath = VmwareStorageLayoutHelper.getLegacyDatastorePathFromVmdkFileName(dsMo, volumeTO.getPath() + ".vmdk"); - if (!dsMo.fileExists(datastoreVolumePath)) { datastoreVolumePath = VmwareStorageLayoutHelper.getVmwareDatastorePathFromVmdkFileName(dsMo, vmName, volumeTO.getPath() + ".vmdk"); } + if (!dsMo.fileExists(datastoreVolumePath)) { + datastoreVolumePath = dsMo.searchFileInSubFolders(volumeTO.getPath() + ".vmdk", true, null); + } } } @@ -2019,7 +2114,9 @@ private Answer attachVolume(Command cmd, DiskTO disk, boolean isAttach, boolean if (isManaged) { handleDatastoreAndVmdkDetachManaged(cmd, diskUuid, iScsiName, storageHost, storagePort); } else { - VmwareStorageLayoutHelper.syncVolumeToRootFolder(dsMo.getOwnerDatacenter().first(), dsMo, volumeTO.getPath(), vmName, VmwareManager.s_vmwareSearchExcludeFolder.value()); + if (!dsMo.getDatastoreType().equalsIgnoreCase("VVOL")) { + VmwareStorageLayoutHelper.syncVolumeToRootFolder(dsMo.getOwnerDatacenter().first(), dsMo, volumeTO.getPath(), vmName, VmwareManager.s_vmwareSearchExcludeFolder.value()); + } } } @@ -2102,7 +2199,7 @@ private synchronized ManagedObjectReference prepareSecondaryDatastoreOnHost(Stri URI uri = new URI(storeUrl); VmwareHypervisorHost hyperHost = hostService.getHyperHost(hostService.getServiceContext(null), null); - ManagedObjectReference morDatastore = hyperHost.mountDatastore(false, uri.getHost(), 0, uri.getPath(), storeName.replace("-", "")); + ManagedObjectReference morDatastore = hyperHost.mountDatastore(false, uri.getHost(), 0, uri.getPath(), storeName.replace("-", ""), false); if (morDatastore == null) { throw new Exception("Unable to mount secondary storage on host. storeUrl: " + storeUrl); @@ -2227,37 +2324,49 @@ public Answer createVolume(CreateObjectCommand cmd) { String volumeUuid = UUID.randomUUID().toString().replace("-", ""); String volumeDatastorePath = dsMo.getDatastorePath(volumeUuid + ".vmdk"); - String dummyVmName = hostService.getWorkerName(context, cmd, 0); - try { - s_logger.info("Create worker VM " + dummyVmName); - vmMo = HypervisorHostHelper.createWorkerVM(hyperHost, dsMo, dummyVmName); - if (vmMo == null) { - throw new Exception("Unable to create a dummy VM for volume creation"); - } + VolumeObjectTO newVol = new VolumeObjectTO(); - synchronized (this) { - try { - vmMo.createDisk(volumeDatastorePath, (int)(volume.getSize() / (1024L * 1024L)), morDatastore, vmMo.getScsiDeviceControllerKey()); - vmMo.detachDisk(volumeDatastorePath, false); + try { + VirtualStorageObjectManagerMO vStorageObjectManagerMO = new VirtualStorageObjectManagerMO(context); + VStorageObject virtualDisk = vStorageObjectManagerMO.createDisk(morDatastore, VirtualDiskType.THIN, volume.getSize(), volumeDatastorePath, volumeUuid); + DatastoreFile file = new DatastoreFile(((BaseConfigInfoDiskFileBackingInfo)virtualDisk.getConfig().getBacking()).getFilePath()); + newVol.setPath(file.getFileBaseName()); + newVol.setSize(volume.getSize()); + } catch (Exception e) { + s_logger.debug("Create disk using vStorageObject manager failed due to exception " + e.getMessage() + ", retying using worker VM"); + String dummyVmName = hostService.getWorkerName(context, cmd, 0, dsMo); + try { + s_logger.info("Create worker VM " + dummyVmName); + vmMo = HypervisorHostHelper.createWorkerVM(hyperHost, dsMo, dummyVmName); + if (vmMo == null) { + throw new Exception("Unable to create a dummy VM for volume creation"); } - catch (Exception e) { - s_logger.error("Deleting file " + volumeDatastorePath + " due to error: " + e.getMessage()); - VmwareStorageLayoutHelper.deleteVolumeVmdkFiles(dsMo, volumeUuid, dcMo, VmwareManager.s_vmwareSearchExcludeFolder.value()); - throw new CloudRuntimeException("Unable to create volume due to: " + e.getMessage()); + + synchronized (this) { + try { + vmMo.createDisk(volumeDatastorePath, (int)(volume.getSize() / (1024L * 1024L)), morDatastore, vmMo.getScsiDeviceControllerKey()); + vmMo.detachDisk(volumeDatastorePath, false); + } + catch (Exception e1) { + s_logger.error("Deleting file " + volumeDatastorePath + " due to error: " + e1.getMessage()); + VmwareStorageLayoutHelper.deleteVolumeVmdkFiles(dsMo, volumeUuid, dcMo, VmwareManager.s_vmwareSearchExcludeFolder.value()); + throw new CloudRuntimeException("Unable to create volume due to: " + e1.getMessage()); + } } - } - VolumeObjectTO newVol = new VolumeObjectTO(); - newVol.setPath(volumeUuid); - newVol.setSize(volume.getSize()); - return new CreateObjectAnswer(newVol); - } finally { - s_logger.info("Destroy dummy VM after volume creation"); - if (vmMo != null) { - vmMo.detachAllDisks(); - vmMo.destroy(); + newVol = new VolumeObjectTO(); + newVol.setPath(volumeUuid); + newVol.setSize(volume.getSize()); + return new CreateObjectAnswer(newVol); + } finally { + s_logger.info("Destroy dummy VM after volume creation"); + if (vmMo != null) { + vmMo.detachAllDisks(); + vmMo.destroy(); + } } } + return new CreateObjectAnswer(newVol); } catch (Throwable e) { if (e instanceof RemoteException) { s_logger.warn("Encounter remote exception to vCenter, invalidate VMware session context"); @@ -2911,7 +3020,7 @@ private void createVmdk(Command cmd, DatastoreMO dsMo, String vmdkDatastorePath, VmwareContext context = hostService.getServiceContext(null); VmwareHypervisorHost hyperHost = hostService.getHyperHost(context, null); - String dummyVmName = hostService.getWorkerName(context, cmd, 0); + String dummyVmName = hostService.getWorkerName(context, cmd, 0, dsMo); VirtualMachineMO vmMo = HypervisorHostHelper.createWorkerVM(hyperHost, dsMo, dummyVmName); @@ -2919,7 +3028,7 @@ private void createVmdk(Command cmd, DatastoreMO dsMo, String vmdkDatastorePath, throw new Exception("Unable to create a dummy VM for volume creation"); } - Long volumeSizeToUse = volumeSize < dsMo.getSummary().getFreeSpace() ? volumeSize : dsMo.getSummary().getFreeSpace(); + Long volumeSizeToUse = volumeSize < dsMo.getDatastoreSummary().getFreeSpace() ? volumeSize : dsMo.getDatastoreSummary().getFreeSpace(); vmMo.createDisk(vmdkDatastorePath, getMBsFromBytes(volumeSizeToUse), dsMo.getMor(), vmMo.getScsiDeviceControllerKey()); vmMo.detachDisk(vmdkDatastorePath, false); @@ -3429,13 +3538,13 @@ private Long restoreVolumeFromSecStorage(VmwareHypervisorHost hyperHost, Datasto VirtualMachineMO clonedVm = null; try { - hyperHost.importVmFromOVF(srcOVFFileName, newVolumeName, primaryDsMo, "thin"); + hyperHost.importVmFromOVF(srcOVFFileName, newVolumeName, primaryDsMo, "thin", null); clonedVm = hyperHost.findVmOnHyperHost(newVolumeName); if (clonedVm == null) { throw new Exception("Unable to create container VM for volume creation"); } - clonedVm.moveAllVmDiskFiles(primaryDsMo, "", false); + clonedVm.moveAllVmDiskFiles(primaryDsMo, HypervisorHostHelper.VSPHERE_DATASTORE_BASE_FOLDER, false); clonedVm.detachAllDisks(); return _storage.getSize(srcOVFFileName); } finally { @@ -3467,7 +3576,6 @@ public Answer createVolumeFromSnapshot(CopyCommand cmd) { String backupPath = backedUpSnapshotUuid.substring(0, index); backedUpSnapshotUuid = backedUpSnapshotUuid.substring(index + 1); String details; - String newVolumeName = VmwareHelper.getVCenterSafeUuid(); VmwareContext context = hostService.getServiceContext(cmd); try { @@ -3479,13 +3587,14 @@ public Answer createVolumeFromSnapshot(CopyCommand cmd) { throw new Exception(msg); } + DatastoreMO primaryDsMo = new DatastoreMO(hyperHost.getContext(), morPrimaryDs); + String newVolumeName = VmwareHelper.getVCenterSafeUuid(primaryDsMo); // strip off the extension since restoreVolumeFromSecStorage internally will append suffix there. if (backedUpSnapshotUuid.endsWith(".ova")){ backedUpSnapshotUuid = backedUpSnapshotUuid.replace(".ova", ""); } else if (backedUpSnapshotUuid.endsWith(".ovf")){ backedUpSnapshotUuid = backedUpSnapshotUuid.replace(".ovf", ""); } - DatastoreMO primaryDsMo = new DatastoreMO(hyperHost.getContext(), morPrimaryDs); restoreVolumeFromSecStorage(hyperHost, primaryDsMo, newVolumeName, secondaryStorageUrl, backupPath, backedUpSnapshotUuid, (long)cmd.getWait() * 1000, _nfsVersion); VolumeObjectTO newVol = new VolumeObjectTO(); @@ -3555,8 +3664,76 @@ public Answer handleDownloadTemplateToPrimaryStorage(DirectDownloadCommand cmd) return null; } + @Override + public Answer CheckDataStoreStoragePolicyComplaince(CheckDataStoreStoragePolicyComplainceCommand cmd) { + String primaryStorageNameLabel = cmd.getStoragePool().getUuid(); + String storagePolicyId = cmd.getStoragePolicyId(); + VmwareContext context = hostService.getServiceContext(cmd); + try { + VmwareHypervisorHost hyperHost = hostService.getHyperHost(context, cmd); + ManagedObjectReference morPrimaryDs = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, primaryStorageNameLabel); + if (morPrimaryDs == null) { + String msg = "Unable to find datastore: " + primaryStorageNameLabel; + s_logger.error(msg); + throw new Exception(msg); + } + + DatastoreMO primaryDsMo = new DatastoreMO(hyperHost.getContext(), morPrimaryDs); + boolean isDatastoreStoragePolicyComplaint = primaryDsMo.isDatastoreStoragePolicyComplaint(storagePolicyId); + + String failedMessage = String.format("DataStore %s is not complaince with storage policy id %s", primaryStorageNameLabel, storagePolicyId); + if (!isDatastoreStoragePolicyComplaint) + return new Answer(cmd, isDatastoreStoragePolicyComplaint, failedMessage); + else + return new Answer(cmd, isDatastoreStoragePolicyComplaint, null); + } catch (Throwable e) { + if (e instanceof RemoteException) { + hostService.invalidateServiceContext(context); + } + String details = String.format("Exception while checking if datastore %s is storage policy %s complaince : %s", primaryStorageNameLabel, storagePolicyId, VmwareHelper.getExceptionMessage(e)); + s_logger.error(details, e); + return new Answer(cmd, false, details); + } + } + @Override public Answer copyVolumeFromPrimaryToPrimary(CopyCommand cmd) { return null; } + + /** + * Return the cloned VM from the template + */ + public VirtualMachineMO cloneVMFromTemplate(String templateName, String cloneName, String templatePrimaryStoreUuid) { + try { + VmwareContext context = hostService.getServiceContext(null); + VmwareHypervisorHost hyperHost = hostService.getHyperHost(context, null); + DatacenterMO dcMo = new DatacenterMO(context, hyperHost.getHyperHostDatacenter()); + VirtualMachineMO templateMo = dcMo.findVm(templateName); + if (templateMo == null) { + throw new CloudRuntimeException(String.format("Unable to find template %s in vSphere", templateName)); + } + ManagedObjectReference morDatastore = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, templatePrimaryStoreUuid); + DatastoreMO dsMo = new DatastoreMO(context, morDatastore); + ManagedObjectReference morPool = hyperHost.getHyperHostOwnerResourcePool(); + if (morDatastore == null) { + throw new CloudRuntimeException("Unable to find datastore in vSphere"); + } + s_logger.info("Cloning VM " + cloneName + " from template " + templatePrimaryStoreUuid); + if (!_fullCloneFlag) { + createVMLinkedClone(templateMo, dcMo, cloneName, morDatastore, morPool); + } else { + createVMFullClone(templateMo, dcMo, dsMo, cloneName, morDatastore, morPool); + } + VirtualMachineMO vm = dcMo.findVm(cloneName); + if (vm == null) { + throw new CloudRuntimeException("Unable to get the cloned VM " + cloneName); + } + return vm; + } catch (Throwable e) { + String msg = "Error cloning VM from template in primary storage: %s" + e.getMessage(); + s_logger.error(msg, e); + throw new CloudRuntimeException(msg, e); + } + } } diff --git a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ImportVsphereStoragePoliciesCmd.java b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ImportVsphereStoragePoliciesCmd.java new file mode 100644 index 000000000000..ea5bacfb1f42 --- /dev/null +++ b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ImportVsphereStoragePoliciesCmd.java @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.admin.zone; + +import com.cloud.dc.DataCenter; +import com.cloud.dc.VsphereStoragePolicy; +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.hypervisor.vmware.VmwareDatacenterService; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.VsphereStoragePoliciesResponse; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.ZoneResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.log4j.Logger; + +import javax.inject.Inject; +import java.util.ArrayList; +import java.util.List; + +@APICommand(name = ImportVsphereStoragePoliciesCmd.APINAME, description = "Import vSphere storage policies", + responseObject = VsphereStoragePoliciesResponse.class, + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, + authorized = {RoleType.Admin}) +public class ImportVsphereStoragePoliciesCmd extends BaseCmd { + + public static final Logger LOGGER = Logger.getLogger(ImportVsphereStoragePoliciesCmd.class.getName()); + + public static final String APINAME = "importVsphereStoragePolicies"; + + @Inject + public VmwareDatacenterService _vmwareDatacenterService; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.ZONE_ID, type = CommandType.UUID, entityType = ZoneResponse.class, + description = "ID of the zone") + private Long zoneId; + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { + final DataCenter dataCenter = _resourceService.getZone(getZoneId()); + if (dataCenter == null) { + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Unable to find zone by ID: " + getZoneId()); + } + + List storagePolicies = _vmwareDatacenterService.importVsphereStoragePolicies(this); + final ListResponse responseList = new ListResponse<>(); + final List storagePoliciesResponseList = new ArrayList<>(); + for (VsphereStoragePolicy storagePolicy : storagePolicies) { + final VsphereStoragePoliciesResponse storagePoliciesResponse = new VsphereStoragePoliciesResponse(); + storagePoliciesResponse.setZoneId(dataCenter.getUuid()); + storagePoliciesResponse.setId(storagePolicy.getUuid()); + storagePoliciesResponse.setName(storagePolicy.getName()); + storagePoliciesResponse.setPolicyId(storagePolicy.getPolicyId()); + storagePoliciesResponse.setDescription(storagePolicy.getDescription()); + storagePoliciesResponse.setObjectName("StoragePolicy"); + + storagePoliciesResponseList.add(storagePoliciesResponse); + } + responseList.setResponses(storagePoliciesResponseList); + responseList.setResponseName(getCommandName()); + setResponseObject(responseList); + } + + @Override + public String getCommandName() { + return APINAME.toLowerCase() + BaseCmd.RESPONSE_SUFFIX; + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccountId(); + } + + public Long getZoneId() { + return zoneId; + } + +} diff --git a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVsphereStoragePoliciesCmd.java b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVsphereStoragePoliciesCmd.java new file mode 100644 index 000000000000..90e8e8805485 --- /dev/null +++ b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVsphereStoragePoliciesCmd.java @@ -0,0 +1,109 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.admin.zone; + +import com.cloud.dc.DataCenter; +import com.cloud.dc.VsphereStoragePolicy; +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.hypervisor.vmware.VmwareDatacenterService; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.VsphereStoragePoliciesResponse; +import org.apache.cloudstack.api.response.ZoneResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.log4j.Logger; + +import javax.inject.Inject; +import java.util.ArrayList; +import java.util.List; + +@APICommand(name = ListVsphereStoragePoliciesCmd.APINAME, description = "List vSphere storage policies", + responseObject = VsphereStoragePoliciesResponse.class, + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, + authorized = {RoleType.Admin}) +public class ListVsphereStoragePoliciesCmd extends BaseCmd { + + public static final Logger LOGGER = Logger.getLogger(ListVsphereStoragePoliciesCmd.class.getName()); + + public static final String APINAME = "listVsphereStoragePolicies"; + + @Inject + public VmwareDatacenterService _vmwareDatacenterService; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.ZONE_ID, type = CommandType.UUID, entityType = ZoneResponse.class, + description = "ID of the zone") + private Long zoneId; + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { + final DataCenter dataCenter = _resourceService.getZone(getZoneId()); + if (dataCenter == null) { + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Unable to find zone by ID: " + getZoneId()); + } + + List storagePolicies = _vmwareDatacenterService.listVsphereStoragePolicies(this); + final ListResponse responseList = new ListResponse<>(); + final List storagePoliciesResponseList = new ArrayList<>(); + for (VsphereStoragePolicy storagePolicy : storagePolicies) { + final VsphereStoragePoliciesResponse storagePoliciesResponse = new VsphereStoragePoliciesResponse(); + storagePoliciesResponse.setZoneId(dataCenter.getUuid()); + storagePoliciesResponse.setId(storagePolicy.getUuid()); + storagePoliciesResponse.setName(storagePolicy.getName()); + storagePoliciesResponse.setPolicyId(storagePolicy.getPolicyId()); + storagePoliciesResponse.setDescription(storagePolicy.getDescription()); + storagePoliciesResponse.setObjectName("StoragePolicy"); + + storagePoliciesResponseList.add(storagePoliciesResponse); + } + responseList.setResponses(storagePoliciesResponseList); + responseList.setResponseName(getCommandName()); + setResponseObject(responseList); + } + + @Override + public String getCommandName() { + return APINAME.toLowerCase() + BaseCmd.RESPONSE_SUFFIX; + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccountId(); + } + + public Long getZoneId() { + return zoneId; + } +} diff --git a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/storage/motion/VmwareStorageMotionStrategy.java b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/storage/motion/VmwareStorageMotionStrategy.java index 2463e75c01d3..5a7b4c4ca677 100644 --- a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/storage/motion/VmwareStorageMotionStrategy.java +++ b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/storage/motion/VmwareStorageMotionStrategy.java @@ -132,7 +132,10 @@ private boolean isIntraCluster(DataObject srcData, DataObject destData) { StoragePool srcPool = storagePoolDao.findById(srcStore.getId()); DataStore destStore = destData.getDataStore(); StoragePool destPool = storagePoolDao.findById(destStore.getId()); - return srcPool.getClusterId().equals(destPool.getClusterId()); + if (srcPool.getClusterId() != null && destPool.getClusterId() != null) { + return srcPool.getClusterId().equals(destPool.getClusterId()); + } + return false; } /** diff --git a/plugins/hypervisors/vmware/src/main/resources/META-INF/cloudstack/core/spring-vmware-core-context.xml b/plugins/hypervisors/vmware/src/main/resources/META-INF/cloudstack/core/spring-vmware-core-context.xml index 3af2d1ac31fe..a2d8314bfb88 100644 --- a/plugins/hypervisors/vmware/src/main/resources/META-INF/cloudstack/core/spring-vmware-core-context.xml +++ b/plugins/hypervisors/vmware/src/main/resources/META-INF/cloudstack/core/spring-vmware-core-context.xml @@ -37,7 +37,6 @@ class="com.cloud.hypervisor.vmware.dao.VmwareDatacenterZoneMapDaoImpl" /> - \ No newline at end of file diff --git a/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/vmware/VmwareDatacenterApiUnitTest.java b/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/vmware/VmwareDatacenterApiUnitTest.java index eb041396459b..1249f6ce0047 100644 --- a/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/vmware/VmwareDatacenterApiUnitTest.java +++ b/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/vmware/VmwareDatacenterApiUnitTest.java @@ -31,6 +31,7 @@ import com.cloud.dc.dao.ClusterVSMMapDao; import com.cloud.dc.dao.DataCenterDao; import com.cloud.dc.dao.HostPodDao; +import com.cloud.dc.dao.VsphereStoragePolicyDao; import com.cloud.event.dao.EventDao; import com.cloud.exception.DiscoveryException; import com.cloud.exception.InvalidParameterValueException; @@ -486,6 +487,11 @@ public TemplateManager templateManager() { return Mockito.mock(TemplateManager.class); } + @Bean + public VsphereStoragePolicyDao vsphereStoragePolicyDao() { + return Mockito.mock(VsphereStoragePolicyDao.class); + } + public static class Library implements TypeFilter { @Override diff --git a/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImplTest.java b/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImplTest.java index 8aa92f7d9c4d..80677e9d3ba3 100644 --- a/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImplTest.java +++ b/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImplTest.java @@ -105,6 +105,7 @@ public void updateVmwareDatacenterNormalUpdate() { Mockito.lenient().doReturn(hostDetails).when(hostDetailsDao).findDetails(Mockito.anyLong()); Mockito.doReturn("some-old-guid").when(hostDetails).get("guid"); Mockito.doReturn(hostDetails).when(hostDetailsDao).findDetails(Mockito.anyLong()); + Mockito.doReturn(null).when(vmwareManager).importVsphereStoragePoliciesInternal(Mockito.anyLong(), Mockito.anyLong()); final VmwareDatacenter vmwareDatacenter = vmwareManager.updateVmwareDatacenter(updateVmwareDcCmd); diff --git a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java index d987f28ca947..d530969941b9 100644 --- a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java +++ b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java @@ -35,6 +35,7 @@ import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand; import org.apache.cloudstack.storage.command.AttachAnswer; import org.apache.cloudstack.storage.command.AttachCommand; +import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplainceCommand; import org.apache.cloudstack.storage.command.CopyCmdAnswer; import org.apache.cloudstack.storage.command.CopyCommand; import org.apache.cloudstack.storage.command.CreateObjectAnswer; @@ -215,6 +216,12 @@ public Answer copyVolumeFromPrimaryToPrimary(CopyCommand cmd) { return null; } + @Override + public Answer CheckDataStoreStoragePolicyComplaince(CheckDataStoreStoragePolicyComplainceCommand cmd) { + s_logger.info("'CheckDataStoreStoragePolicyComplainceCommand' not applicable used for XenServerStorageProcessor"); + return new Answer(cmd,false,"Not applicable used for XenServerStorageProcessor"); + } + @Override public AttachAnswer attachIso(final AttachCommand cmd) { final DiskTO disk = cmd.getDisk(); diff --git a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/Xenserver625StorageProcessor.java b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/Xenserver625StorageProcessor.java index 184013e42a4b..9b54e0bc1f79 100644 --- a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/Xenserver625StorageProcessor.java +++ b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/Xenserver625StorageProcessor.java @@ -27,6 +27,7 @@ import java.util.Set; import java.util.UUID; +import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplainceCommand; import org.apache.cloudstack.storage.command.CopyCmdAnswer; import org.apache.cloudstack.storage.command.CopyCommand; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; @@ -912,6 +913,12 @@ public Answer createVolumeFromSnapshot(final CopyCommand cmd) { return new CopyCmdAnswer(details); } + @Override + public Answer CheckDataStoreStoragePolicyComplaince(CheckDataStoreStoragePolicyComplainceCommand cmd) { + s_logger.info("'CheckDataStoreStoragePolicyComplainceCommand' not applicable used for XenServerStorageProcessor"); + return new Answer(cmd,false,"Not applicable used for XenServerStorageProcessor"); + } + @Override public Answer copyVolumeFromPrimaryToSecondary(final CopyCommand cmd) { final Connection conn = hypervisorResource.getConnection(); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index 37e1ecd4b8a1..b05c782d1e80 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -537,7 +537,7 @@ private DeployDestination plan(final long nodesCount, final DataCenter zone, fin } if (capacityManager.checkIfHostHasCapacity(h.getId(), cpu_requested * reserved, ram_requested * reserved, false, cpuOvercommitRatio, memoryOvercommitRatio, true)) { if (LOGGER.isDebugEnabled()) { - LOGGER.debug(String.format("Found host ID: %s for with enough capacity, CPU=%d RAM=%d", h.getUuid(), cpu_requested * reserved, toHumanReadableSize(ram_requested * reserved))); + LOGGER.debug(String.format("Found host ID: %s for with enough capacity, CPU=%d RAM=%s", h.getUuid(), cpu_requested * reserved, toHumanReadableSize(ram_requested * reserved))); } hostEntry.setValue(new Pair(h, reserved)); suitable_host_found = true; @@ -558,7 +558,7 @@ private DeployDestination plan(final long nodesCount, final DataCenter zone, fin } return new DeployDestination(zone, null, planCluster, null); } - String msg = String.format("Cannot find enough capacity for Kubernetes cluster(requested cpu=%d memory=%d) with offering ID: %s", + String msg = String.format("Cannot find enough capacity for Kubernetes cluster(requested cpu=%d memory=%s) with offering ID: %s", cpu_requested * nodesCount, toHumanReadableSize(ram_requested * nodesCount), offering.getUuid()); LOGGER.warn(msg); throw new InsufficientServerCapacityException(msg, DataCenter.class, zone.getId()); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java index e274f5e1a438..d8d41bdf7ac6 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java @@ -231,7 +231,7 @@ protected DeployDestination plan(final long nodesCount, final DataCenter zone, f } if (capacityManager.checkIfHostHasCapacity(h.getId(), cpu_requested * reserved, ram_requested * reserved, false, cpuOvercommitRatio, memoryOvercommitRatio, true)) { if (LOGGER.isDebugEnabled()) { - LOGGER.debug(String.format("Found host ID: %s for with enough capacity, CPU=%d RAM=%d", h.getUuid(), cpu_requested * reserved, toHumanReadableSize(ram_requested * reserved))); + LOGGER.debug(String.format("Found host ID: %s for with enough capacity, CPU=%d RAM=%s", h.getUuid(), cpu_requested * reserved, toHumanReadableSize(ram_requested * reserved))); } hostEntry.setValue(new Pair(h, reserved)); suitable_host_found = true; @@ -251,7 +251,7 @@ protected DeployDestination plan(final long nodesCount, final DataCenter zone, f } return new DeployDestination(zone, null, null, null); } - String msg = String.format("Cannot find enough capacity for Kubernetes cluster(requested cpu=%d memory=%d) with offering ID: %s and hypervisor: %s", + String msg = String.format("Cannot find enough capacity for Kubernetes cluster(requested cpu=%d memory=%s) with offering ID: %s and hypervisor: %s", cpu_requested * nodesCount, toHumanReadableSize(ram_requested * nodesCount), offering.getUuid(), clusterTemplate.getHypervisorType().toString()); LOGGER.warn(msg); diff --git a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java index 92c128b27fe1..7a1fb0c5af8a 100644 --- a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java +++ b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java @@ -220,7 +220,7 @@ private void addStorageMetrics(final List metricsList, final long dcId, fi metricsList.add(new ItemPool(zoneName, zoneUuid, poolName, poolPath, "primary", poolFactor, TOTAL, totalCapacity)); } - for (final ImageStore imageStore : imageStoreDao.findByScope(new ZoneScope(dcId))) { + for (final ImageStore imageStore : imageStoreDao.findByZone(new ZoneScope(dcId), null)) { final StorageStats stats = ApiDBUtils.getSecondaryStorageStatistics(imageStore.getId()); metricsList.add(new ItemPool(zoneName, zoneUuid, imageStore.getName(), imageStore.getUrl(), "secondary", null, USED, stats != null ? stats.getByteUsed() : 0)); metricsList.add(new ItemPool(zoneName, zoneUuid, imageStore.getName(), imageStore.getUrl(), "secondary", null, TOTAL, stats != null ? stats.getCapacityBytes() : 0)); diff --git a/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/driver/DateraPrimaryDataStoreDriver.java b/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/driver/DateraPrimaryDataStoreDriver.java index 497960d1c232..fa1f3d4a6467 100644 --- a/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/driver/DateraPrimaryDataStoreDriver.java +++ b/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/driver/DateraPrimaryDataStoreDriver.java @@ -955,7 +955,7 @@ private DateraObject.AppInstance createDateraClone(DateraObject.DateraConnection } else if (dataType == DataObjectType.TEMPLATE) { s_logger.debug("Clone volume from a template"); - VMTemplateStoragePoolVO templatePoolRef = tmpltPoolDao.findByPoolTemplate(storagePoolId, dataObjectId); + VMTemplateStoragePoolVO templatePoolRef = tmpltPoolDao.findByPoolTemplate(storagePoolId, dataObjectId, null); if (templatePoolRef != null) { baseAppInstanceName = templatePoolRef.getLocalDownloadPath(); @@ -1114,7 +1114,7 @@ public String createTemplateVolume(TemplateInfo templateInfo, long storagePoolId iqn = appInstance.getIqn(); VMTemplateStoragePoolVO templatePoolRef = tmpltPoolDao.findByPoolTemplate(storagePoolId, - templateInfo.getId()); + templateInfo.getId(), null); templatePoolRef.setInstallPath(DateraUtil.generateIqnPath(iqn)); templatePoolRef.setLocalDownloadPath(appInstance.getName()); @@ -1505,7 +1505,7 @@ private void deleteTemplate(TemplateInfo templateInfo, long storagePoolId) DateraUtil.deleteAppInstance(conn, appInstanceName); VMTemplateStoragePoolVO templatePoolRef = tmpltPoolDao.findByPoolTemplate(storagePoolId, - templateInfo.getId()); + templateInfo.getId(), null); tmpltPoolDao.remove(templatePoolRef.getId()); diff --git a/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java b/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java index bdaeddca4c02..068b26fe70a3 100644 --- a/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java +++ b/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java @@ -18,36 +18,12 @@ */ package org.apache.cloudstack.storage.datastore.lifecycle; -import java.io.UnsupportedEncodingException; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URLDecoder; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.UUID; - -import javax.inject.Inject; - -import org.apache.log4j.Logger; - -import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope; -import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; -import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; -import org.apache.cloudstack.engine.subsystem.api.storage.HostScope; -import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreInfo; -import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreLifeCycle; -import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreParameters; -import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; -import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; -import org.apache.cloudstack.storage.volume.datastore.PrimaryDataStoreHelper; - import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; import com.cloud.agent.api.CreateStoragePoolCommand; import com.cloud.agent.api.DeleteStoragePoolCommand; import com.cloud.agent.api.StoragePoolInfo; +import com.cloud.agent.api.ValidateVcenterDetailsCommand; import com.cloud.alert.AlertManager; import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.StorageConflictException; @@ -77,6 +53,29 @@ import com.cloud.vm.dao.SecondaryStorageVmDao; import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.VMInstanceDao; +import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.engine.subsystem.api.storage.HostScope; +import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreLifeCycle; +import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreParameters; +import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.storage.volume.datastore.PrimaryDataStoreHelper; +import org.apache.commons.lang.StringUtils; +import org.apache.log4j.Logger; + +import javax.inject.Inject; +import java.io.UnsupportedEncodingException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URLDecoder; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.UUID; public class CloudStackPrimaryDataStoreLifeCycleImpl implements PrimaryDataStoreLifeCycle { private static final Logger s_logger = Logger.getLogger(CloudStackPrimaryDataStoreLifeCycleImpl.class); @@ -133,6 +132,7 @@ public DataStore initialize(Map dsInfos) { Long zoneId = (Long)dsInfos.get("zoneId"); String url = (String)dsInfos.get("url"); String providerName = (String)dsInfos.get("providerName"); + String hypervisorType = (String)dsInfos.get("hypervisorType"); if (clusterId != null && podId == null) { throw new InvalidParameterValueException("Cluster id requires pod id"); } @@ -250,10 +250,21 @@ public DataStore initialize(Map dsInfos) { parameters.setPath(hostPath.replaceFirst("/", "")); parameters.setUserInfo(userInfo); } else if (scheme.equalsIgnoreCase("PreSetup")) { + if (StringUtils.isNotBlank(hypervisorType) && HypervisorType.getType(hypervisorType).equals(HypervisorType.VMware)) { + validateVcenterDetails(zoneId, podId, clusterId,storageHost); + } parameters.setType(StoragePoolType.PreSetup); parameters.setHost(storageHost); parameters.setPort(0); parameters.setPath(hostPath); + } else if (scheme.equalsIgnoreCase("DatastoreCluster")) { + if (StringUtils.isNotBlank(hypervisorType) && HypervisorType.getType(hypervisorType).equals(HypervisorType.VMware)) { + validateVcenterDetails(zoneId, podId, clusterId,storageHost); + } + parameters.setType(StoragePoolType.DatastoreCluster); + parameters.setHost(storageHost); + parameters.setPort(0); + parameters.setPath(hostPath); } else if (scheme.equalsIgnoreCase("iscsi")) { String[] tokens = hostPath.split("/"); int lun = NumbersUtil.parseInt(tokens[tokens.length - 1], -1); @@ -327,7 +338,7 @@ public DataStore initialize(Map dsInfos) { uuid = (String)existingUuid; } else if (scheme.equalsIgnoreCase("sharedmountpoint") || scheme.equalsIgnoreCase("clvm")) { uuid = UUID.randomUUID().toString(); - } else if (scheme.equalsIgnoreCase("PreSetup")) { + } else if (scheme.equalsIgnoreCase("PreSetup") && !(StringUtils.isNotBlank(hypervisorType) && HypervisorType.getType(hypervisorType).equals(HypervisorType.VMware))) { uuid = hostPath.replace("/", ""); } else { uuid = UUID.nameUUIDFromBytes((storageHost + hostPath).getBytes()).toString(); @@ -353,12 +364,39 @@ public DataStore initialize(Map dsInfos) { return dataStoreHelper.createPrimaryDataStore(parameters); } + private void validateVcenterDetails(Long zoneId, Long podId, Long clusterId, String storageHost) { + + List allHosts = + _resourceMgr.listAllUpHosts(Host.Type.Routing, clusterId, podId, zoneId); + if (allHosts.isEmpty()) { + throw new CloudRuntimeException("No host up to associate a storage pool with in zone: " + zoneId + " pod: " + podId + " cluster: " + clusterId); + } + + boolean success = false; + for (HostVO h : allHosts) { + ValidateVcenterDetailsCommand cmd = new ValidateVcenterDetailsCommand(storageHost); + final Answer answer = agentMgr.easySend(h.getId(), cmd); + if (answer != null && answer.getResult()) { + s_logger.info("Successfully validated vCenter details provided"); + return; + } else { + if (answer != null) { + throw new InvalidParameterValueException("Provided vCenter server details does not match with the existing vCenter in zone id: " + zoneId); + } else { + String msg = "Can not validate vCenter through host " + h.getId() + " due to ValidateVcenterDetailsCommand returns null"; + s_logger.warn(msg); + } + } + } + throw new CloudRuntimeException("Could not validate vCenter details through any of the hosts with in zone: " + zoneId + ", pod: " + podId + ", cluster: " + clusterId); + } + protected boolean createStoragePool(long hostId, StoragePool pool) { s_logger.debug("creating pool " + pool.getName() + " on host " + hostId); if (pool.getPoolType() != StoragePoolType.NetworkFilesystem && pool.getPoolType() != StoragePoolType.Filesystem && pool.getPoolType() != StoragePoolType.IscsiLUN && pool.getPoolType() != StoragePoolType.Iscsi && pool.getPoolType() != StoragePoolType.VMFS && - pool.getPoolType() != StoragePoolType.SharedMountPoint && pool.getPoolType() != StoragePoolType.PreSetup && pool.getPoolType() != StoragePoolType.OCFS2 && + pool.getPoolType() != StoragePoolType.SharedMountPoint && pool.getPoolType() != StoragePoolType.PreSetup && pool.getPoolType() != StoragePoolType.DatastoreCluster && pool.getPoolType() != StoragePoolType.OCFS2 && pool.getPoolType() != StoragePoolType.RBD && pool.getPoolType() != StoragePoolType.CLVM && pool.getPoolType() != StoragePoolType.SMB && pool.getPoolType() != StoragePoolType.Gluster) { s_logger.warn(" Doesn't support storage pool type " + pool.getPoolType()); diff --git a/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/driver/SolidFirePrimaryDataStoreDriver.java b/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/driver/SolidFirePrimaryDataStoreDriver.java index 22e4e952b3ef..0651f2ea8560 100644 --- a/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/driver/SolidFirePrimaryDataStoreDriver.java +++ b/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/driver/SolidFirePrimaryDataStoreDriver.java @@ -709,7 +709,7 @@ private SolidFireUtil.SolidFireVolume createClone(SolidFireUtil.SolidFireConnect sfVolumeId = Long.parseLong(snapshotDetails.getValue()); } else if (dataObjectType == DataObjectType.TEMPLATE) { // get the cached template on this storage - VMTemplateStoragePoolVO templatePoolRef = vmTemplatePoolDao.findByPoolTemplate(storagePoolId, dataObjectId); + VMTemplateStoragePoolVO templatePoolRef = vmTemplatePoolDao.findByPoolTemplate(storagePoolId, dataObjectId, null); if (templatePoolRef != null) { sfVolumeId = Long.parseLong(templatePoolRef.getLocalDownloadPath()); @@ -1135,7 +1135,7 @@ private String createTemplateVolume(TemplateInfo templateInfo, long storagePoolI String iqn = sfVolume.getIqn(); - VMTemplateStoragePoolVO templatePoolRef = vmTemplatePoolDao.findByPoolTemplate(storagePoolId, templateInfo.getId()); + VMTemplateStoragePoolVO templatePoolRef = vmTemplatePoolDao.findByPoolTemplate(storagePoolId, templateInfo.getId(), null); templatePoolRef.setInstallPath(iqn); templatePoolRef.setLocalDownloadPath(Long.toString(sfVolume.getId())); diff --git a/pom.xml b/pom.xml index 262960cad8fa..c040d9667251 100644 --- a/pom.xml +++ b/pom.xml @@ -98,6 +98,7 @@ 0.5 2.6 2.7.0 + 3.6.1 diff --git a/server/pom.xml b/server/pom.xml index 7cd0dd1460d3..e3d98dff6108 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -89,6 +89,11 @@ commons-codec commons-codec + + org.apache.commons + commons-math3 + ${cs.commons-math3.version} + org.apache.cloudstack cloud-utils diff --git a/server/src/main/java/com/cloud/acl/AffinityGroupAccessChecker.java b/server/src/main/java/com/cloud/acl/AffinityGroupAccessChecker.java index 6106c7268e13..3a648cdcbf0a 100644 --- a/server/src/main/java/com/cloud/acl/AffinityGroupAccessChecker.java +++ b/server/src/main/java/com/cloud/acl/AffinityGroupAccessChecker.java @@ -80,8 +80,8 @@ public boolean checkAccess(Account caller, ControlledEntity entity, AccessType a //check if the group belongs to a project User user = CallContext.current().getCallingUser(); ProjectVO project = _projectDao.findByProjectAccountId(group.getAccountId()); - ProjectAccount userProjectAccount = _projectAccountDao.findByProjectIdUserId(project.getId(), user.getAccountId(), user.getId()); if (project != null) { + ProjectAccount userProjectAccount = _projectAccountDao.findByProjectIdUserId(project.getId(), user.getAccountId(), user.getId()); if (userProjectAccount != null) { if (AccessType.ModifyProject.equals(accessType) && _projectAccountDao.canUserModifyProject(project.getId(), user.getAccountId(), user.getId())) { return true; diff --git a/server/src/main/java/com/cloud/acl/DomainChecker.java b/server/src/main/java/com/cloud/acl/DomainChecker.java index 5fc2b343be9e..24b6b2a42b42 100644 --- a/server/src/main/java/com/cloud/acl/DomainChecker.java +++ b/server/src/main/java/com/cloud/acl/DomainChecker.java @@ -61,6 +61,7 @@ import com.cloud.user.User; import com.cloud.user.dao.AccountDao; import com.cloud.utils.component.AdapterBase; +import com.cloud.utils.exception.CloudRuntimeException; @Component public class DomainChecker extends AdapterBase implements SecurityChecker { @@ -199,6 +200,9 @@ public boolean checkAccess(Account caller, ControlledEntity entity, AccessType a private boolean checkOperationPermitted(Account caller, ControlledEntity entity) { User user = CallContext.current().getCallingUser(); Project project = projectDao.findByProjectAccountId(entity.getAccountId()); + if (project == null) { + throw new CloudRuntimeException("Unable to find project to which the entity belongs to"); + } ProjectAccount projectUser = _projectAccountDao.findByProjectIdUserId(project.getId(), user.getAccountId(), user.getId()); String apiCommandName = CallContext.current().getApiName(); diff --git a/server/src/main/java/com/cloud/api/ApiDBUtils.java b/server/src/main/java/com/cloud/api/ApiDBUtils.java index e6b307757956..a2433ab634d6 100644 --- a/server/src/main/java/com/cloud/api/ApiDBUtils.java +++ b/server/src/main/java/com/cloud/api/ApiDBUtils.java @@ -2004,16 +2004,16 @@ public static TemplateResponse newTemplateUpdateResponse(TemplateJoinVO vr) { return s_templateJoinDao.newUpdateResponse(vr); } - public static TemplateResponse newTemplateResponse(ResponseView view, TemplateJoinVO vr) { - return s_templateJoinDao.newTemplateResponse(view, vr); + public static TemplateResponse newTemplateResponse(EnumSet detailsView, ResponseView view, TemplateJoinVO vr) { + return s_templateJoinDao.newTemplateResponse(detailsView, view, vr); } public static TemplateResponse newIsoResponse(TemplateJoinVO vr) { return s_templateJoinDao.newIsoResponse(vr); } - public static TemplateResponse fillTemplateDetails(ResponseView view, TemplateResponse vrData, TemplateJoinVO vr) { - return s_templateJoinDao.setTemplateResponse(view, vrData, vr); + public static TemplateResponse fillTemplateDetails(EnumSet detailsView, ResponseView view, TemplateResponse vrData, TemplateJoinVO vr) { + return s_templateJoinDao.setTemplateResponse(detailsView, view, vrData, vr); } public static List newTemplateView(VirtualMachineTemplate vr) { diff --git a/server/src/main/java/com/cloud/api/ApiResponseHelper.java b/server/src/main/java/com/cloud/api/ApiResponseHelper.java index 201ea1c4eff1..51a5436fba54 100644 --- a/server/src/main/java/com/cloud/api/ApiResponseHelper.java +++ b/server/src/main/java/com/cloud/api/ApiResponseHelper.java @@ -1564,7 +1564,7 @@ public List createTemplateResponses(ResponseView view, Virtual tvo = ApiDBUtils.newTemplateView(result, zoneId, readyOnly); } - return ViewResponseHelper.createTemplateResponse(view, tvo.toArray(new TemplateJoinVO[tvo.size()])); + return ViewResponseHelper.createTemplateResponse(EnumSet.of(DomainDetails.all), view, tvo.toArray(new TemplateJoinVO[tvo.size()])); } @Override @@ -1581,7 +1581,7 @@ public List createTemplateResponses(ResponseView view, Virtual tvo.addAll(ApiDBUtils.newTemplateView(result, zoneId, readyOnly)); } } - return ViewResponseHelper.createTemplateResponse(view, tvo.toArray(new TemplateJoinVO[tvo.size()])); + return ViewResponseHelper.createTemplateResponse(EnumSet.of(DomainDetails.all), view, tvo.toArray(new TemplateJoinVO[tvo.size()])); } @Override @@ -2409,6 +2409,7 @@ public NetworkACLItemResponse createNetworkACLItemResponse(NetworkACLItem aclIte NetworkACL acl = ApiDBUtils.findByNetworkACLId(aclItem.getAclId()); if (acl != null) { response.setAclId(acl.getUuid()); + response.setAclName(acl.getName()); } //set tag information @@ -3003,6 +3004,7 @@ public PrivateGatewayResponse createPrivateGatewayResponse(PrivateGateway result NetworkACL acl = ApiDBUtils.findByNetworkACLId(result.getNetworkACLId()); if (acl != null) { response.setAclId(acl.getUuid()); + response.setAclName(acl.getName()); } response.setObjectName("privategateway"); diff --git a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java index f13b7953e336..055708062645 100644 --- a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java +++ b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java @@ -31,6 +31,7 @@ import javax.inject.Inject; +import com.cloud.storage.dao.VMTemplateDetailsDao; import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.affinity.AffinityGroupDomainMapVO; import org.apache.cloudstack.affinity.AffinityGroupResponse; @@ -71,7 +72,6 @@ import org.apache.cloudstack.api.command.user.resource.ListDetailOptionsCmd; import org.apache.cloudstack.api.command.user.securitygroup.ListSecurityGroupsCmd; import org.apache.cloudstack.api.command.user.tag.ListTagsCmd; -import org.apache.cloudstack.api.command.user.template.ListTemplateOVFProperties; import org.apache.cloudstack.api.command.user.template.ListTemplatesCmd; import org.apache.cloudstack.api.command.user.vm.ListVMsCmd; import org.apache.cloudstack.api.command.user.vmgroup.ListVMGroupsCmd; @@ -101,7 +101,6 @@ import org.apache.cloudstack.api.response.ServiceOfferingResponse; import org.apache.cloudstack.api.response.StoragePoolResponse; import org.apache.cloudstack.api.response.StorageTagResponse; -import org.apache.cloudstack.api.response.TemplateOVFPropertyResponse; import org.apache.cloudstack.api.response.TemplateResponse; import org.apache.cloudstack.api.response.UserResponse; import org.apache.cloudstack.api.response.UserVmResponse; @@ -118,11 +117,13 @@ import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.query.QueryService; import org.apache.cloudstack.resourcedetail.dao.DiskOfferingDetailsDao; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; -import com.cloud.agent.api.storage.OVFProperty; import com.cloud.api.query.dao.AccountJoinDao; import com.cloud.api.query.dao.AffinityGroupJoinDao; import com.cloud.api.query.dao.AsyncJobJoinDao; @@ -211,11 +212,9 @@ import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.Storage.TemplateType; import com.cloud.storage.StoragePoolTagVO; -import com.cloud.storage.TemplateOVFPropertyVO; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.Volume; import com.cloud.storage.dao.StoragePoolTagsDao; -import com.cloud.storage.dao.TemplateOVFPropertiesDao; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.tags.ResourceTagVO; import com.cloud.tags.dao.ResourceTagDao; @@ -402,7 +401,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q ManagementServerHostDao managementServerHostDao; @Inject - TemplateOVFPropertiesDao templateOVFPropertiesDao; + VMTemplateDetailsDao vmTemplateDetailsDao; @Inject public VpcVirtualNetworkApplianceService routerService; @@ -413,11 +412,18 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q @Inject private RouterHealthCheckResultDao routerHealthCheckResultDao; + @Inject + private TemplateDataStoreDao templateDataStoreDao; + + @Inject + private PrimaryDataStoreDao _storagePoolDao; + @Inject private ProjectInvitationDao projectInvitationDao; @Inject private UserDao userDao; + /* * (non-Javadoc) * @@ -960,7 +966,12 @@ private Pair, Integer> searchForUserVMsInternal(ListVMsCmd cm } if (storageId != null) { - sb.and("poolId", sb.entity().getPoolId(), SearchCriteria.Op.EQ); + StoragePoolVO poolVO = _storagePoolDao.findById((Long) storageId); + if (poolVO.getPoolType() == Storage.StoragePoolType.DatastoreCluster) { + sb.and("poolId", sb.entity().getPoolId(), SearchCriteria.Op.IN); + } else { + sb.and("poolId", sb.entity().getPoolId(), SearchCriteria.Op.EQ); + } } if (affinityGroupId != null) { @@ -1086,7 +1097,14 @@ private Pair, Integer> searchForUserVMsInternal(ListVMsCmd cm } if (storageId != null) { - sc.setParameters("poolId", storageId); + StoragePoolVO poolVO = _storagePoolDao.findById((Long) storageId); + if (poolVO.getPoolType() == Storage.StoragePoolType.DatastoreCluster) { + List childDatastores = _storagePoolDao.listChildStoragePoolsInDatastoreCluster((Long) storageId); + List childDatastoreIds = childDatastores.stream().map(mo -> mo.getId()).collect(Collectors.toList()); + sc.setParameters("poolId", childDatastoreIds.toArray()); + } else { + sc.setParameters("poolId", storageId); + } } } @@ -1913,7 +1931,14 @@ private Pair, Integer> searchForVolumesInternal(ListVolumesCm sb.and("instanceId", sb.entity().getVmId(), SearchCriteria.Op.EQ); sb.and("dataCenterId", sb.entity().getDataCenterId(), SearchCriteria.Op.EQ); sb.and("podId", sb.entity().getPodId(), SearchCriteria.Op.EQ); - sb.and("storageId", sb.entity().getPoolUuid(), SearchCriteria.Op.EQ); + if (storageId != null) { + StoragePoolVO poolVO = _storagePoolDao.findByUuid(storageId); + if (poolVO.getPoolType() == Storage.StoragePoolType.DatastoreCluster) { + sb.and("storageId", sb.entity().getPoolUuid(), SearchCriteria.Op.IN); + } else { + sb.and("storageId", sb.entity().getPoolUuid(), SearchCriteria.Op.EQ); + } + } sb.and("diskOfferingId", sb.entity().getDiskOfferingId(), SearchCriteria.Op.EQ); sb.and("display", sb.entity().isDisplayVolume(), SearchCriteria.Op.EQ); sb.and("state", sb.entity().getState(), SearchCriteria.Op.EQ); @@ -1982,7 +2007,14 @@ private Pair, Integer> searchForVolumesInternal(ListVolumesCm } if (storageId != null) { - sc.setParameters("storageId", storageId); + StoragePoolVO poolVO = _storagePoolDao.findByUuid(storageId); + if (poolVO.getPoolType() == Storage.StoragePoolType.DatastoreCluster) { + List childDatastores = _storagePoolDao.listChildStoragePoolsInDatastoreCluster(poolVO.getId()); + List childDatastoreIds = childDatastores.stream().map(mo -> mo.getUuid()).collect(Collectors.toList()); + sc.setParameters("storageId", childDatastoreIds.toArray()); + } else { + sc.setParameters("storageId", storageId); + } } if (clusterId != null) { @@ -2373,6 +2405,7 @@ private Pair, Integer> searchForStoragePoolsInternal(Lis sb.and("clusterId", sb.entity().getClusterId(), SearchCriteria.Op.EQ); sb.and("hostAddress", sb.entity().getHostAddress(), SearchCriteria.Op.EQ); sb.and("scope", sb.entity().getScope(), SearchCriteria.Op.EQ); + sb.and("parent", sb.entity().getParent(), Op.EQ); SearchCriteria sc = sb.create(); @@ -2410,6 +2443,7 @@ private Pair, Integer> searchForStoragePoolsInternal(Lis if (scopeType != null) { sc.setParameters("scope", scopeType.toString()); } + sc.setParameters("parent", 0); // search Pool details by ids Pair, Integer> uniquePoolPair = _poolJoinDao.searchAndCount(sc, searchFilter); @@ -2861,6 +2895,9 @@ private Pair, Integer> searchForServiceOfferingsInte ServiceOfferingVO currentVmOffering = null; Boolean isRecursive = cmd.isRecursive(); Long zoneId = cmd.getZoneId(); + Integer cpuNumber = cmd.getCpuNumber(); + Integer memory = cmd.getMemory(); + Integer cpuSpeed = cmd.getCpuSpeed(); SearchCriteria sc = _srvOfferingJoinDao.createSearchCriteria(); if (!_accountMgr.isRootAdmin(caller.getId()) && isSystem) { @@ -2912,35 +2949,7 @@ private Pair, Integer> searchForServiceOfferingsInte if (caller.getType() == Account.ACCOUNT_TYPE_NORMAL) { throw new InvalidParameterValueException("Only ROOT admins and Domain admins can list service offerings with isrecursive=true"); } - }/* else { // domain + all ancestors - // find all domain Id up to root domain for this account - List domainIds = new ArrayList(); - DomainVO domainRecord; - if (vmId != null) { - UserVmVO vmInstance = _userVmDao.findById(vmId); - domainRecord = _domainDao.findById(vmInstance.getDomainId()); - if (domainRecord == null) { - s_logger.error("Could not find the domainId for vmId:" + vmId); - throw new CloudAuthenticationException("Could not find the domainId for vmId:" + vmId); - } - } else { - domainRecord = _domainDao.findById(caller.getDomainId()); - if (domainRecord == null) { - s_logger.error("Could not find the domainId for account:" + caller.getAccountName()); - throw new CloudAuthenticationException("Could not find the domainId for account:" + caller.getAccountName()); - } - } - domainIds.add(domainRecord.getId()); - while (domainRecord.getParent() != null) { - domainRecord = _domainDao.findById(domainRecord.getParent()); - domainIds.add(domainRecord.getId()); - } - - SearchCriteria spc = _srvOfferingJoinDao.createSearchCriteria(); - spc.addOr("domainId", SearchCriteria.Op.IN, domainIds.toArray()); - spc.addOr("domainId", SearchCriteria.Op.NULL); // include public offering as well - sc.addAnd("domainId", SearchCriteria.Op.SC, spc); - }*/ + } } else { // for root users if (caller.getDomainId() != 1 && isSystem) { // NON ROOT admin @@ -2987,6 +2996,37 @@ private Pair, Integer> searchForServiceOfferingsInte sc.addAnd("zoneId", SearchCriteria.Op.SC, zoneSC); } + if (cpuNumber != null) { + SearchCriteria cpuConstraintSearchCriteria = _srvOfferingJoinDao.createSearchCriteria(); + cpuConstraintSearchCriteria.addAnd("minCpu", Op.LTEQ, cpuNumber); + cpuConstraintSearchCriteria.addAnd("maxCpu", Op.GTEQ, cpuNumber); + + SearchCriteria cpuSearchCriteria = _srvOfferingJoinDao.createSearchCriteria(); + cpuSearchCriteria.addOr("minCpu", Op.NULL); + cpuSearchCriteria.addOr("constraints", Op.SC, cpuConstraintSearchCriteria); + + sc.addAnd("cpuConstraints", SearchCriteria.Op.SC, cpuSearchCriteria); + } + + if (memory != null) { + SearchCriteria memoryConstraintSearchCriteria = _srvOfferingJoinDao.createSearchCriteria(); + memoryConstraintSearchCriteria.addAnd("minMemory", Op.LTEQ, memory); + memoryConstraintSearchCriteria.addAnd("maxMemory", Op.GTEQ, memory); + + SearchCriteria memSearchCriteria = _srvOfferingJoinDao.createSearchCriteria(); + memSearchCriteria.addOr("minMemory", Op.NULL); + memSearchCriteria.addOr("memconstraints", Op.SC, memoryConstraintSearchCriteria); + + sc.addAnd("memoryConstraints", SearchCriteria.Op.SC, memSearchCriteria); + } + + if (cpuSpeed != null) { + SearchCriteria cpuSpeedSearchCriteria = _srvOfferingJoinDao.createSearchCriteria(); + cpuSpeedSearchCriteria.addOr("speed", Op.NULL); + cpuSpeedSearchCriteria.addOr("speed", Op.EQ, cpuSpeed); + sc.addAnd("cpuspeedconstraints", SearchCriteria.Op.SC, cpuSpeedSearchCriteria); + } + Pair, Integer> result = _srvOfferingJoinDao.searchAndCount(sc, searchFilter); //Couldn't figure out a smart way to filter offerings based on tags in sql so doing it in Java. @@ -3267,7 +3307,7 @@ public ListResponse listTemplates(ListTemplatesCmd cmd) { respView = ResponseView.Full; } - List templateResponses = ViewResponseHelper.createTemplateResponse(respView, result.first().toArray(new TemplateJoinVO[result.first().size()])); + List templateResponses = ViewResponseHelper.createTemplateResponse(cmd.getDetails(), respView, result.first().toArray(new TemplateJoinVO[result.first().size()])); response.setResponses(templateResponses, result.second()); return response; } @@ -4037,29 +4077,6 @@ protected ManagementServerResponse createManagementServerResponse(ManagementServ return mgmtResponse; } - @Override - public ListResponse listTemplateOVFProperties(ListTemplateOVFProperties cmd) { - ListResponse response = new ListResponse<>(); - List result = new ArrayList<>(); - Long templateId = cmd.getTemplateId(); - List ovfProperties = templateOVFPropertiesDao.listByTemplateId(templateId); - for (OVFProperty property : ovfProperties) { - TemplateOVFPropertyResponse propertyResponse = new TemplateOVFPropertyResponse(); - propertyResponse.setKey(property.getKey()); - propertyResponse.setType(property.getType()); - propertyResponse.setValue(property.getValue()); - propertyResponse.setQualifiers(property.getQualifiers()); - propertyResponse.setUserConfigurable(property.isUserConfigurable()); - propertyResponse.setLabel(property.getLabel()); - propertyResponse.setDescription(property.getDescription()); - propertyResponse.setPassword(property.isPassword()); - propertyResponse.setObjectName("ovfproperty"); - result.add(propertyResponse); - } - response.setResponses(result); - return response; - } - @Override public List listRouterHealthChecks(GetRouterHealthCheckResultsCmd cmd) { s_logger.info("Executing health check command " + cmd); diff --git a/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java b/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java index 88a7639ac914..d91fa7bf480e 100644 --- a/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java +++ b/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java @@ -26,6 +26,7 @@ import java.util.List; import java.util.Map; +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.affinity.AffinityGroupResponse; import org.apache.cloudstack.api.ApiConstants.DomainDetails; import org.apache.cloudstack.api.ApiConstants.HostDetails; @@ -583,17 +584,17 @@ public static List createDataCenterResponse(ResponseView view, Boo return respList; } - public static List createTemplateResponse(ResponseView view, TemplateJoinVO... templates) { + public static List createTemplateResponse(EnumSet detailsView, ResponseView view, TemplateJoinVO... templates) { LinkedHashMap vrDataList = new LinkedHashMap(); for (TemplateJoinVO vr : templates) { TemplateResponse vrData = vrDataList.get(vr.getTempZonePair()); if (vrData == null) { // first time encountering this volume - vrData = ApiDBUtils.newTemplateResponse(view, vr); + vrData = ApiDBUtils.newTemplateResponse(detailsView, view, vr); } else{ // update tags - vrData = ApiDBUtils.fillTemplateDetails(view, vrData, vr); + vrData = ApiDBUtils.fillTemplateDetails(detailsView, view, vrData, vr); } vrDataList.put(vr.getTempZonePair(), vrData); } @@ -609,7 +610,7 @@ public static List createTemplateUpdateResponse(ResponseView v vrData = ApiDBUtils.newTemplateUpdateResponse(vr); } else { // update tags - vrData = ApiDBUtils.fillTemplateDetails(view, vrData, vr); + vrData = ApiDBUtils.fillTemplateDetails(EnumSet.of(DomainDetails.all), view, vrData, vr); } vrDataList.put(vr.getId(), vrData); } @@ -625,7 +626,7 @@ public static List createIsoResponse(ResponseView view, Templa vrData = ApiDBUtils.newIsoResponse(vr); } else { // update tags - vrData = ApiDBUtils.fillTemplateDetails(view, vrData, vr); + vrData = ApiDBUtils.fillTemplateDetails(EnumSet.of(DomainDetails.all), view, vrData, vr); } vrDataList.put(vr.getTempZonePair(), vrData); } diff --git a/server/src/main/java/com/cloud/api/query/dao/ImageStoreJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/ImageStoreJoinDaoImpl.java index 2389b57cf4fe..b91398de9b41 100644 --- a/server/src/main/java/com/cloud/api/query/dao/ImageStoreJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/ImageStoreJoinDaoImpl.java @@ -67,6 +67,7 @@ public ImageStoreResponse newImageStoreResponse(ImageStoreJoinVO ids) { osResponse.setName(ids.getName()); osResponse.setProviderName(ids.getProviderName()); osResponse.setProtocol(ids.getProtocol()); + osResponse.setReadonly(ids.isReadonly()); String url = ids.getUrl(); //if store is type cifs, remove the password if(ids.getProtocol().equals("cifs".toString())) { diff --git a/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDaoImpl.java index b8b312bd267b..a248d71e72c6 100644 --- a/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDaoImpl.java @@ -32,6 +32,8 @@ import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreDriver; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailVO; +import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -52,6 +54,9 @@ public class StoragePoolJoinDaoImpl extends GenericDaoBase spSearch; private final SearchBuilder spIdSearch; @@ -94,6 +99,10 @@ public StoragePoolResponse newStoragePoolResponse(StoragePoolJoinVO pool) { poolResponse.setHypervisor(pool.getHypervisor().toString()); } + StoragePoolDetailVO poolType = storagePoolDetailsDao.findDetail(pool.getId(), "pool_type"); + if (poolType != null) { + poolResponse.setType(poolType.getValue()); + } long allocatedSize = pool.getUsedCapacity() + pool.getReservedCapacity(); poolResponse.setDiskSizeTotal(pool.getCapacityBytes()); poolResponse.setDiskSizeAllocated(allocatedSize); diff --git a/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDao.java b/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDao.java index c9d7eba48b2f..58cb886594fd 100644 --- a/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDao.java +++ b/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDao.java @@ -16,8 +16,10 @@ // under the License. package com.cloud.api.query.dao; +import java.util.EnumSet; import java.util.List; +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ResponseObject.ResponseView; import org.apache.cloudstack.api.response.TemplateResponse; @@ -30,13 +32,13 @@ public interface TemplateJoinDao extends GenericDao { - TemplateResponse newTemplateResponse(ResponseView view, TemplateJoinVO tmpl); + TemplateResponse newTemplateResponse(EnumSet detailsView, ResponseView view, TemplateJoinVO tmpl); TemplateResponse newIsoResponse(TemplateJoinVO tmpl); TemplateResponse newUpdateResponse(TemplateJoinVO tmpl); - TemplateResponse setTemplateResponse(ResponseView view, TemplateResponse tmplData, TemplateJoinVO tmpl); + TemplateResponse setTemplateResponse(EnumSet detailsView, ResponseView view, TemplateResponse tmplData, TemplateJoinVO tmpl); List newTemplateView(VirtualMachineTemplate tmpl); diff --git a/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDaoImpl.java index 61a101743c91..ddc069bf5beb 100644 --- a/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDaoImpl.java @@ -17,6 +17,7 @@ package com.cloud.api.query.dao; import java.util.ArrayList; +import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -25,6 +26,10 @@ import javax.inject.Inject; +import com.cloud.deployasis.DeployAsIsConstants; +import com.cloud.deployasis.TemplateDeployAsIsDetailVO; +import com.cloud.deployasis.dao.TemplateDeployAsIsDetailsDao; +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.utils.security.DigestHelper; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -36,6 +41,9 @@ import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateState; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import com.cloud.api.ApiDBUtils; import com.cloud.api.ApiResponseHelper; @@ -70,7 +78,13 @@ public class TemplateJoinDaoImpl extends GenericDaoBaseWithTagInformation tmpltIdPairSearch; @@ -108,6 +122,7 @@ protected TemplateJoinDaoImpl() { activeTmpltSearch.and("store_id", activeTmpltSearch.entity().getDataStoreId(), SearchCriteria.Op.EQ); activeTmpltSearch.and("type", activeTmpltSearch.entity().getTemplateType(), SearchCriteria.Op.EQ); activeTmpltSearch.and("templateState", activeTmpltSearch.entity().getTemplateState(), SearchCriteria.Op.EQ); + activeTmpltSearch.and("public", activeTmpltSearch.entity().isPublicTemplate(), SearchCriteria.Op.EQ); activeTmpltSearch.done(); // select distinct pair (template_id, zone_id) @@ -140,8 +155,20 @@ private String getTemplateStatus(TemplateJoinVO template) { } @Override - public TemplateResponse newTemplateResponse(ResponseView view, TemplateJoinVO template) { + public TemplateResponse newTemplateResponse(EnumSet detailsView, ResponseView view, TemplateJoinVO template) { + List templatesInStore = _templateStoreDao.listByTemplateNotBypassed(template.getId()); + List> downloadProgressDetails = new ArrayList(); + HashMap downloadDetailInImageStores = null; + for (TemplateDataStoreVO templateInStore : templatesInStore) { + downloadDetailInImageStores = new HashMap<>(); + downloadDetailInImageStores.put("datastore", dataStoreDao.findById(templateInStore.getDataStoreId()).getName()); + downloadDetailInImageStores.put("downloadPercent", Integer.toString(templateInStore.getDownloadPercent())); + downloadDetailInImageStores.put("downloadState", (templateInStore.getDownloadState() != null ? templateInStore.getDownloadState().toString() : "")); + downloadProgressDetails.add(downloadDetailInImageStores); + } + TemplateResponse templateResponse = new TemplateResponse(); + templateResponse.setDownloadProgress(downloadProgressDetails); templateResponse.setId(template.getUuid()); templateResponse.setName(template.getName()); templateResponse.setDisplayText(template.getDisplayText()); @@ -212,8 +239,12 @@ public TemplateResponse newTemplateResponse(ResponseView view, TemplateJoinVO te } // set details map - Map details = _templateDetailsDao.listDetailsKeyPairs(template.getId()); - templateResponse.setDetails(details); + if (detailsView.contains(ApiConstants.DomainDetails.all)) { + Map details = _templateDetailsDao.listDetailsKeyPairs(template.getId()); + templateResponse.setDetails(details); + + setDeployAsIsDetails(template, templateResponse); + } // update tag information long tag_id = template.getTagId(); @@ -222,6 +253,7 @@ public TemplateResponse newTemplateResponse(ResponseView view, TemplateJoinVO te } templateResponse.setDirectDownload(template.isDirectDownload()); + templateResponse.setDeployAsIs(template.isDeployAsIs()); templateResponse.setRequiresHvm(template.isRequiresHvm()); //set template children disks @@ -244,6 +276,19 @@ public TemplateResponse newTemplateResponse(ResponseView view, TemplateJoinVO te return templateResponse; } + private void setDeployAsIsDetails(TemplateJoinVO template, TemplateResponse templateResponse) { + if (template.isDeployAsIs()) { + List deployAsIsDetails = templateDeployAsIsDetailsDao.listDetails(template.getId()); + for (TemplateDeployAsIsDetailVO deployAsIsDetailVO : deployAsIsDetails) { + if (deployAsIsDetailVO.getName().startsWith(DeployAsIsConstants.HARDWARE_ITEM_PREFIX)) { + //Do not list hardware items + continue; + } + templateResponse.addDeployAsIsDetail(deployAsIsDetailVO.getName(), deployAsIsDetailVO.getValue()); + } + } + } + //TODO: This is to keep compatibility with 4.1 API, where updateTemplateCmd and updateIsoCmd will return a simpler TemplateResponse // compared to listTemplates and listIsos. @Override @@ -289,16 +334,13 @@ public TemplateResponse newUpdateResponse(TemplateJoinVO result) { } @Override - public TemplateResponse setTemplateResponse(ResponseView view, TemplateResponse templateResponse, TemplateJoinVO template) { - - // update details map - if (template.getDetailName() != null) { - Map details = templateResponse.getDetails(); - if (details == null) { - details = new HashMap<>(); + public TemplateResponse setTemplateResponse(EnumSet detailsView, ResponseView view, TemplateResponse templateResponse, TemplateJoinVO template) { + if (detailsView.contains(ApiConstants.DomainDetails.all)) { + // update details map + String key = template.getDetailName(); + if (key != null) { + templateResponse.addDetail(key, template.getDetailValue()); } - details.put(template.getDetailName(), template.getDetailValue()); - templateResponse.setDetails(details); } // update tag information diff --git a/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDaoImpl.java index 80c433ad71ca..8e489f89eca0 100644 --- a/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDaoImpl.java @@ -322,8 +322,8 @@ public UserVmResponse newUserVmResponse(ResponseView view, String objectName, Us if (vmDetails != null) { Map resourceDetails = new HashMap(); for (UserVmDetailVO userVmDetailVO : vmDetails) { - if (!userVmDetailVO.getName().startsWith(ApiConstants.OVF_PROPERTIES) || - (UserVmManager.DisplayVMOVFProperties.value() && userVmDetailVO.getName().startsWith(ApiConstants.OVF_PROPERTIES))) { + if (!userVmDetailVO.getName().startsWith(ApiConstants.PROPERTIES) || + (UserVmManager.DisplayVMOVFProperties.value() && userVmDetailVO.getName().startsWith(ApiConstants.PROPERTIES))) { resourceDetails.put(userVmDetailVO.getName(), userVmDetailVO.getValue()); } if ((ApiConstants.BootType.UEFI.toString()).equalsIgnoreCase(userVmDetailVO.getName())) { diff --git a/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDaoImpl.java index c7977356379e..2ee0ca1b1f45 100644 --- a/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDaoImpl.java @@ -24,6 +24,8 @@ import org.apache.cloudstack.api.ResponseObject.ResponseView; import org.apache.cloudstack.api.response.VolumeResponse; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -51,6 +53,8 @@ public class VolumeJoinDaoImpl extends GenericDaoBaseWithTagInformation volSearch; @@ -220,6 +224,14 @@ public VolumeResponse newVolumeResponse(ResponseView view, VolumeJoinVO volume) String poolName = (poolId == null) ? "none" : volume.getPoolName(); volResponse.setStoragePoolName(poolName); volResponse.setStoragePoolId(volume.getPoolUuid()); + if (poolId != null) { + StoragePoolVO poolVO = primaryDataStoreDao.findById(poolId); + if (poolVO != null && poolVO.getParent() != 0L) { + StoragePoolVO datastoreClusterVO = primaryDataStoreDao.findById(poolVO.getParent()); + volResponse.setStoragePoolName(datastoreClusterVO.getName()); + volResponse.setStoragePoolId(datastoreClusterVO.getUuid()); + } + } } volResponse.setAttached(volume.getAttached()); diff --git a/server/src/main/java/com/cloud/api/query/vo/ImageStoreJoinVO.java b/server/src/main/java/com/cloud/api/query/vo/ImageStoreJoinVO.java index 244f89ec3c2b..bcc73cb47bf5 100644 --- a/server/src/main/java/com/cloud/api/query/vo/ImageStoreJoinVO.java +++ b/server/src/main/java/com/cloud/api/query/vo/ImageStoreJoinVO.java @@ -67,6 +67,9 @@ public class ImageStoreJoinVO extends BaseViewVO implements InternalIdentity, Id @Enumerated(value = EnumType.STRING) private DataStoreRole role; + @Column(name = "readonly") + private boolean readonly = false; + @Column(name = "data_center_id") private long zoneId; @@ -128,4 +131,8 @@ public DataStoreRole getRole() { public Date getRemoved() { return removed; } + + public boolean isReadonly() { + return readonly; + } } diff --git a/server/src/main/java/com/cloud/api/query/vo/ServiceOfferingJoinVO.java b/server/src/main/java/com/cloud/api/query/vo/ServiceOfferingJoinVO.java index 4f8932ad4cfb..a3e1cb147234 100644 --- a/server/src/main/java/com/cloud/api/query/vo/ServiceOfferingJoinVO.java +++ b/server/src/main/java/com/cloud/api/query/vo/ServiceOfferingJoinVO.java @@ -175,6 +175,18 @@ public class ServiceOfferingJoinVO extends BaseViewVO implements InternalIdentit @Column(name = "cache_mode") String cacheMode; + @Column(name = "min_cpu") + Integer minCpu; + + @Column(name = "max_cpu") + Integer maxCpu; + + @Column(name = "min_memory") + Integer minMemory; + + @Column(name = "max_memory") + Integer maxMemory; + public ServiceOfferingJoinVO() { } @@ -356,4 +368,20 @@ public boolean isDynamic() { public String getCacheMode() { return cacheMode; } + + public Integer getMinCpu() { + return minCpu; + } + + public Integer getMaxCpu() { + return maxCpu; + } + + public Integer getMinMemory() { + return minMemory; + } + + public Integer getMaxMemory() { + return maxMemory; + } } diff --git a/server/src/main/java/com/cloud/api/query/vo/StoragePoolJoinVO.java b/server/src/main/java/com/cloud/api/query/vo/StoragePoolJoinVO.java index 565e290bd704..1831aaafac97 100644 --- a/server/src/main/java/com/cloud/api/query/vo/StoragePoolJoinVO.java +++ b/server/src/main/java/com/cloud/api/query/vo/StoragePoolJoinVO.java @@ -139,6 +139,9 @@ public class StoragePoolJoinVO extends BaseViewVO implements InternalIdentity, I @Column(name = "storage_provider_name") private String storageProviderName; + @Column(name = "parent") + private Long parent; + /** * @return the scope */ @@ -263,4 +266,8 @@ public int getJobStatus() { public String getStorageProviderName() { return storageProviderName; } + + public Long getParent() { + return parent; + } } diff --git a/server/src/main/java/com/cloud/api/query/vo/TemplateJoinVO.java b/server/src/main/java/com/cloud/api/query/vo/TemplateJoinVO.java index 25e3b0b5ff50..91bb76336ccc 100644 --- a/server/src/main/java/com/cloud/api/query/vo/TemplateJoinVO.java +++ b/server/src/main/java/com/cloud/api/query/vo/TemplateJoinVO.java @@ -231,6 +231,9 @@ public class TemplateJoinVO extends BaseViewWithTagInformationVO implements Cont @Column(name = "direct_download") private boolean directDownload; + @Column(name = "deploy_as_is") + private boolean deployAsIs; + public TemplateJoinVO() { } @@ -490,6 +493,10 @@ public boolean isDirectDownload() { return directDownload; } + public boolean isDeployAsIs() { + return deployAsIs; + } + public Object getParentTemplateId() { return parentTemplateId; } diff --git a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java index 7e9c9d39c2b1..0bb7eb525523 100755 --- a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java +++ b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java @@ -39,6 +39,8 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.dc.dao.VsphereStoragePolicyDao; +import com.cloud.storage.Storage; import org.apache.cloudstack.acl.SecurityChecker; import org.apache.cloudstack.affinity.AffinityGroup; import org.apache.cloudstack.affinity.AffinityGroupService; @@ -386,6 +388,9 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati IndirectAgentLB _indirectAgentLB; @Inject private VMTemplateZoneDao templateZoneDao; + @Inject + VsphereStoragePolicyDao vsphereStoragePolicyDao; + // FIXME - why don't we have interface for DataCenterLinkLocalIpAddressDao? @Inject @@ -593,6 +598,12 @@ public String updateConfiguration(final long userId, final String name, final St } _storagePoolDetailsDao.addDetail(resourceId, name, value, true); + if (pool.getPoolType() == Storage.StoragePoolType.DatastoreCluster) { + List childDataStores = _storagePoolDao.listChildStoragePoolsInDatastoreCluster(resourceId); + for (StoragePoolVO childDataStore: childDataStores) { + _storagePoolDetailsDao.addDetail(childDataStore.getId(), name, value, true); + } + } break; @@ -1708,7 +1719,7 @@ protected void checkIfZoneIsDeletable(final long zoneId) { } //check if there are any secondary stores attached to the zone - if(!_imageStoreDao.findByScope(new ZoneScope(zoneId)).isEmpty()) { + if(!_imageStoreDao.findByZone(new ZoneScope(zoneId), null).isEmpty()) { throw new CloudRuntimeException(errorMsg + "there are Secondary storages in this zone"); } @@ -2444,6 +2455,13 @@ public ServiceOffering createServiceOffering(final CreateServiceOfferingCmd cmd) } } + final Long storagePolicyId = cmd.getStoragePolicy(); + if (storagePolicyId != null) { + if (vsphereStoragePolicyDao.findById(storagePolicyId) == null) { + throw new InvalidParameterValueException("Please specify a valid vSphere storage policy id"); + } + } + return createServiceOffering(userId, cmd.isSystem(), vmType, cmd.getServiceOfferingName(), cpuNumber, memory, cpuSpeed, cmd.getDisplayText(), cmd.getProvisioningType(), localStorageRequired, offerHA, limitCpuUse, volatileVm, cmd.getTags(), cmd.getDomainIds(), cmd.getZoneIds(), cmd.getHostTag(), cmd.getNetworkRate(), cmd.getDeploymentPlanner(), details, isCustomizedIops, cmd.getMinIops(), cmd.getMaxIops(), @@ -2451,7 +2469,7 @@ public ServiceOffering createServiceOffering(final CreateServiceOfferingCmd cmd) cmd.getBytesWriteRate(), cmd.getBytesWriteRateMax(), cmd.getBytesWriteRateMaxLength(), cmd.getIopsReadRate(), cmd.getIopsReadRateMax(), cmd.getIopsReadRateMaxLength(), cmd.getIopsWriteRate(), cmd.getIopsWriteRateMax(), cmd.getIopsWriteRateMaxLength(), - cmd.getHypervisorSnapshotReserve(), cmd.getCacheMode()); + cmd.getHypervisorSnapshotReserve(), cmd.getCacheMode(), storagePolicyId); } protected ServiceOfferingVO createServiceOffering(final long userId, final boolean isSystem, final VirtualMachine.Type vmType, @@ -2462,7 +2480,7 @@ protected ServiceOfferingVO createServiceOffering(final long userId, final boole Long bytesWriteRate, Long bytesWriteRateMax, Long bytesWriteRateMaxLength, Long iopsReadRate, Long iopsReadRateMax, Long iopsReadRateMaxLength, Long iopsWriteRate, Long iopsWriteRateMax, Long iopsWriteRateMaxLength, - final Integer hypervisorSnapshotReserve, String cacheMode) { + final Integer hypervisorSnapshotReserve, String cacheMode, final Long storagePolicyID) { // Filter child domains when both parent and child domains are present List filteredDomainIds = filterChildSubDomains(domainIds); @@ -2602,6 +2620,10 @@ protected ServiceOfferingVO createServiceOffering(final long userId, final boole } } + if (storagePolicyID != null) { + detailsVO.add(new ServiceOfferingDetailsVO(offering.getId(), ApiConstants.STORAGE_POLICY, String.valueOf(storagePolicyID), false)); + } + if ((offering = _serviceOfferingDao.persist(offering)) != null) { for (Long domainId : filteredDomainIds) { detailsVO.add(new ServiceOfferingDetailsVO(offering.getId(), ApiConstants.DOMAIN_ID, String.valueOf(domainId), false)); @@ -2820,7 +2842,7 @@ protected DiskOfferingVO createDiskOffering(final Long userId, final List Long bytesWriteRate, Long bytesWriteRateMax, Long bytesWriteRateMaxLength, Long iopsReadRate, Long iopsReadRateMax, Long iopsReadRateMaxLength, Long iopsWriteRate, Long iopsWriteRateMax, Long iopsWriteRateMaxLength, - final Integer hypervisorSnapshotReserve, String cacheMode) { + final Integer hypervisorSnapshotReserve, String cacheMode, final Long storagePolicyID) { long diskSize = 0;// special case for custom disk offerings if (numGibibytes != null && numGibibytes <= 0) { throw new InvalidParameterValueException("Please specify a disk size of at least 1 Gb."); @@ -2948,6 +2970,9 @@ protected DiskOfferingVO createDiskOffering(final Long userId, final List detailsVO.add(new DiskOfferingDetailVO(offering.getId(), ApiConstants.ZONE_ID, String.valueOf(zoneId), false)); } } + if (storagePolicyID != null) { + detailsVO.add(new DiskOfferingDetailVO(offering.getId(), ApiConstants.STORAGE_POLICY, String.valueOf(storagePolicyID), false)); + } if (!detailsVO.isEmpty()) { diskOfferingDetailsDao.saveDetails(detailsVO); } @@ -2969,6 +2994,7 @@ public DiskOffering createDiskOffering(final CreateDiskOfferingCmd cmd) { final String tags = cmd.getTags(); final List domainIds = cmd.getDomainIds(); final List zoneIds = cmd.getZoneIds(); + final Long storagePolicyId = cmd.getStoragePolicy(); // check if valid domain if (CollectionUtils.isNotEmpty(domainIds)) { @@ -3008,6 +3034,12 @@ public DiskOffering createDiskOffering(final CreateDiskOfferingCmd cmd) { } } + if (storagePolicyId != null) { + if (vsphereStoragePolicyDao.findById(storagePolicyId) == null) { + throw new InvalidParameterValueException("Please specify a valid vSphere storage policy id"); + } + } + final Boolean isCustomizedIops = cmd.isCustomizedIops(); final Long minIops = cmd.getMinIops(); final Long maxIops = cmd.getMaxIops(); @@ -3038,7 +3070,7 @@ public DiskOffering createDiskOffering(final CreateDiskOfferingCmd cmd) { localStorageRequired, isDisplayOfferingEnabled, isCustomizedIops, minIops, maxIops, bytesReadRate, bytesReadRateMax, bytesReadRateMaxLength, bytesWriteRate, bytesWriteRateMax, bytesWriteRateMaxLength, iopsReadRate, iopsReadRateMax, iopsReadRateMaxLength, iopsWriteRate, iopsWriteRateMax, iopsWriteRateMaxLength, - hypervisorSnapshotReserve, cacheMode); + hypervisorSnapshotReserve, cacheMode, storagePolicyId); } /** diff --git a/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java b/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java index c1acb45fd4fc..72d6c41a78dd 100644 --- a/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java +++ b/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java @@ -30,7 +30,10 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.dao.VMTemplateDao; import com.cloud.utils.StringUtils; +import com.cloud.exception.StorageUnavailableException; import com.cloud.utils.db.Filter; import com.cloud.utils.fsm.StateMachine2; @@ -167,6 +170,8 @@ public class DeploymentPlanningManagerImpl extends ManagerBase implements Deploy private long _hostReservationReleasePeriod = 60L * 60L * 1000L; // one hour by default @Inject protected VMReservationDao _reservationDao; + @Inject + private VMTemplateDao templateDao; private static final long INITIAL_RESERVATION_RELEASE_CHECKER_DELAY = 30L * 1000L; // thirty seconds expressed in milliseconds protected long _nodeId = -1; @@ -346,7 +351,7 @@ public DeployDestination planDeployment(VirtualMachineProfile vmProfile, Deploym storageVolMap.remove(vol); } DeployDestination dest = new DeployDestination(dc, pod, cluster, host, storageVolMap); - s_logger.debug("Returning Deployment Destination: " + dest); + //s_logger.debug("Returning Deployment Destination: " + dest); return dest; } } @@ -484,7 +489,7 @@ public DeployDestination planDeployment(VirtualMachineProfile vmProfile, Deploym } DeployDestination dest = new DeployDestination(dc, pod, cluster, host, storageVolMap); - s_logger.debug("Returning Deployment Destination: " + dest); + //s_logger.debug("Returning Deployment Destination: " + dest); return dest; } } @@ -1100,7 +1105,7 @@ private DeployDestination checkClustersforDestination(List clusterList, Vi storageVolMap.remove(vol); } DeployDestination dest = new DeployDestination(dc, pod, clusterVO, host, storageVolMap); - s_logger.debug("Returning Deployment Destination: " + dest); + //s_logger.debug("Returning Deployment Destination: " + dest); return dest; } } else { @@ -1228,6 +1233,14 @@ private Pair findVMStorageRequirements(VirtualMachineProfile v return new Pair(requiresShared, requiresLocal); } + private boolean isVolumeFromDeployAsIsTemplate(Volume volume) { + if (volume == null) { + return false; + } + VMTemplateVO template = templateDao.findById(volume.getTemplateId()); + return template != null && template.isDeployAsIs(); + } + protected Pair> findPotentialDeploymentResources(List suitableHosts, Map> suitableVolumeStoragePools, ExcludeList avoid, DeploymentPlanner.PlannerResourceUsage resourceUsageRequired, List readyAndReusedVolumes, List preferredHosts) { s_logger.debug("Trying to find a potenial host and associated storage pools from the suitable host/pool lists for this VM"); @@ -1253,6 +1266,7 @@ public int compare(Volume v1, Volume v2) { boolean multipleVolume = volumesOrderBySizeDesc.size() > 1; for (Host potentialHost : suitableHosts) { Map> volumeAllocationMap = new HashMap>(); + StoragePool deployAsIsPool = null; for (Volume vol : volumesOrderBySizeDesc) { haveEnoughSpace = false; s_logger.debug("Checking if host: " + potentialHost.getId() + " can access any suitable storage pool for volume: " + vol.getVolumeType()); @@ -1261,6 +1275,10 @@ public int compare(Volume v1, Volume v2) { hostAffinityCheck = checkAffinity(potentialHost, preferredHosts); for (StoragePool potentialSPool : volumePoolList) { if (hostCanAccessSPool(potentialHost, potentialSPool)) { + if (isVolumeFromDeployAsIsTemplate(vol) && deployAsIsPool != null && !deployAsIsPool.getUuid().equals(potentialSPool.getUuid())) { + //Deploy-as-is volumes must be allocated to the same storage pool + continue; + } hostCanAccessPool = true; if (multipleVolume && !readyAndReusedVolumes.contains(vol)) { List requestVolumes = null; @@ -1270,11 +1288,28 @@ public int compare(Volume v1, Volume v2) { requestVolumes = new ArrayList(); requestVolumes.add(vol); + if (potentialHost.getHypervisorType() == HypervisorType.VMware) { + try { + boolean isStoragePoolStoragepolicyComplaince = _storageMgr.isStoragePoolComplaintWithStoragePolicy(requestVolumes, potentialSPool); + if (!isStoragePoolStoragepolicyComplaince) { + continue; + } + } catch (StorageUnavailableException e) { + s_logger.warn(String.format("Could not verify storage policy complaince against storage pool %s due to exception %s", potentialSPool.getUuid(), e.getMessage())); + continue; + } + } + if (!_storageMgr.storagePoolHasEnoughIops(requestVolumes, potentialSPool) || !_storageMgr.storagePoolHasEnoughSpace(requestVolumes, potentialSPool, potentialHost.getClusterId())) continue; volumeAllocationMap.put(potentialSPool, requestVolumes); } + if (isVolumeFromDeployAsIsTemplate(vol) && deployAsIsPool == null) { + s_logger.info("VM from deploy-as-is template detected, " + + "trying to allocate VM volumes to the same storage pool: " + potentialSPool.getUuid()); + deployAsIsPool = potentialSPool; + } storage.put(vol, potentialSPool); haveEnoughSpace = true; break; diff --git a/server/src/main/java/com/cloud/network/NetworkModelImpl.java b/server/src/main/java/com/cloud/network/NetworkModelImpl.java index aabcf2b10bfc..b6eab90a98cf 100644 --- a/server/src/main/java/com/cloud/network/NetworkModelImpl.java +++ b/server/src/main/java/com/cloud/network/NetworkModelImpl.java @@ -1658,6 +1658,9 @@ public void checkNetworkPermissions(Account owner, Network network) { if (owner.getType() != Account.ACCOUNT_TYPE_PROJECT && networkOwner.getType() == Account.ACCOUNT_TYPE_PROJECT) { User user = CallContext.current().getCallingUser(); Project project = projectDao.findByProjectAccountId(network.getAccountId()); + if (project == null) { + throw new CloudRuntimeException("Unable to find project to which the network belongs to"); + } ProjectAccount projectAccountUser = _projectAccountDao.findByProjectIdUserId(project.getId(), user.getAccountId(), user.getId()); if (projectAccountUser != null) { if (!_projectAccountDao.canUserAccessProjectAccount(user.getAccountId(), user.getId(), network.getAccountId())) { diff --git a/server/src/main/java/com/cloud/network/element/CloudZonesNetworkElement.java b/server/src/main/java/com/cloud/network/element/CloudZonesNetworkElement.java index 3ed5fcc7d432..848340b1a24c 100644 --- a/server/src/main/java/com/cloud/network/element/CloudZonesNetworkElement.java +++ b/server/src/main/java/com/cloud/network/element/CloudZonesNetworkElement.java @@ -218,7 +218,7 @@ public boolean addPasswordAndUserdata(Network network, NicProfile nic, VirtualMa } String serviceOffering = _serviceOfferingDao.findByIdIncludingRemoved(uservm.getServiceOfferingId()).getDisplayText(); String zoneName = _dcDao.findById(network.getDataCenterId()).getName(); - String destHostname = VirtualMachineManager.getHypervisorHostname(dest.getHost().getName()); + String destHostname = VirtualMachineManager.getHypervisorHostname(dest.getHost() != null ? dest.getHost().getName() : ""); cmds.addCommand( "vmdata", generateVmDataCommand(nic.getIPv4Address(), userData, serviceOffering, zoneName, nic.getIPv4Address(), uservm.getHostName(), uservm.getInstanceName(), diff --git a/server/src/main/java/com/cloud/network/element/ConfigDriveNetworkElement.java b/server/src/main/java/com/cloud/network/element/ConfigDriveNetworkElement.java index 4daeda6a530c..7482eca27a89 100644 --- a/server/src/main/java/com/cloud/network/element/ConfigDriveNetworkElement.java +++ b/server/src/main/java/com/cloud/network/element/ConfigDriveNetworkElement.java @@ -369,7 +369,7 @@ private void recreateConfigDriveIso(NicProfile nic, Network network, VirtualMach if (userVm != null) { final boolean isWindows = isWindows(userVm.getGuestOSId()); List vmData = _networkModel.generateVmData(userVm.getUserData(), _serviceOfferingDao.findById(userVm.getServiceOfferingId()).getName(), userVm.getDataCenterId(), userVm.getInstanceName(), vm.getHostName(), vm.getId(), - vm.getUuid(), nic.getMacAddress(), userVm.getDetail("SSH.PublicKey"), (String) vm.getParameter(VirtualMachineProfile.Param.VmPassword), isWindows, VirtualMachineManager.getHypervisorHostname(dest.getHost().getName())); + vm.getUuid(), nic.getMacAddress(), userVm.getDetail("SSH.PublicKey"), (String) vm.getParameter(VirtualMachineProfile.Param.VmPassword), isWindows, VirtualMachineManager.getHypervisorHostname(dest.getHost() != null ? dest.getHost().getName() : "")); vm.setVmData(vmData); vm.setConfigDriveLabel(VirtualMachineManager.VmConfigDriveLabel.value()); createConfigDriveIso(vm, dest, diskToUse); diff --git a/server/src/main/java/com/cloud/network/router/CommandSetupHelper.java b/server/src/main/java/com/cloud/network/router/CommandSetupHelper.java index 00da0d3a1025..4adf1406afb2 100644 --- a/server/src/main/java/com/cloud/network/router/CommandSetupHelper.java +++ b/server/src/main/java/com/cloud/network/router/CommandSetupHelper.java @@ -195,7 +195,7 @@ public void createVmDataCommand(final VirtualRouter router, final UserVm vm, fin final IPAddressVO staticNatIp = _ipAddressDao.findByVmIdAndNetworkId(nic.getNetworkId(), vm.getId()); Host host = _hostDao.findById(vm.getHostId()); - String destHostname = VirtualMachineManager.getHypervisorHostname(host.getName()); + String destHostname = VirtualMachineManager.getHypervisorHostname(host != null ? host.getName() : ""); cmds.addCommand( "vmdata", generateVmDataCommand(router, nic.getIPv4Address(), vm.getUserData(), serviceOffering, zoneName, diff --git a/server/src/main/java/com/cloud/projects/ProjectManagerImpl.java b/server/src/main/java/com/cloud/projects/ProjectManagerImpl.java index 90a27fcafd01..88ad0c2ffc9d 100644 --- a/server/src/main/java/com/cloud/projects/ProjectManagerImpl.java +++ b/server/src/main/java/com/cloud/projects/ProjectManagerImpl.java @@ -239,6 +239,9 @@ public Project createProject(final String name, final String displayText, String } User user = validateUser(userId, accountId, domainId); + if (user != null) { + owner = _accountDao.findById(user.getAccountId()); + } //do resource limit check _resourceLimitMgr.checkResourceLimit(owner, ResourceType.project); @@ -559,9 +562,11 @@ public boolean canAccessProjectAccount(Account caller, long accountId) { } User user = CallContext.current().getCallingUser(); ProjectVO project = _projectDao.findByProjectAccountId(accountId); - ProjectAccount userProjectAccount = _projectAccountDao.findByProjectIdUserId(project.getId(), user.getAccountId(), user.getId()); - if (userProjectAccount != null) { - return _projectAccountDao.canUserAccessProjectAccount(user.getAccountId(), user.getId(), accountId); + if (project != null) { + ProjectAccount userProjectAccount = _projectAccountDao.findByProjectIdUserId(project.getId(), user.getAccountId(), user.getId()); + if (userProjectAccount != null) { + return _projectAccountDao.canUserAccessProjectAccount(user.getAccountId(), user.getId(), accountId); + } } return _projectAccountDao.canAccessProjectAccount(caller.getId(), accountId); } diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index 32fd5b44b551..844f75997d51 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -27,16 +27,20 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.TimeZone; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.function.Predicate; +import java.util.stream.Collectors; import javax.crypto.Mac; import javax.crypto.spec.SecretKeySpec; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.storage.Storage; import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.affinity.AffinityGroupProcessor; import org.apache.cloudstack.affinity.dao.AffinityGroupVMMapDao; @@ -195,8 +199,10 @@ import org.apache.cloudstack.api.command.admin.storage.ListStoragePoolsCmd; import org.apache.cloudstack.api.command.admin.storage.ListStorageProvidersCmd; import org.apache.cloudstack.api.command.admin.storage.ListStorageTagsCmd; +import org.apache.cloudstack.api.command.admin.storage.MigrateSecondaryStorageDataCmd; import org.apache.cloudstack.api.command.admin.storage.PreparePrimaryStorageForMaintenanceCmd; import org.apache.cloudstack.api.command.admin.storage.UpdateCloudToUseObjectStoreCmd; +import org.apache.cloudstack.api.command.admin.storage.UpdateImageStoreCmd; import org.apache.cloudstack.api.command.admin.storage.UpdateStoragePoolCmd; import org.apache.cloudstack.api.command.admin.swift.AddSwiftCmd; import org.apache.cloudstack.api.command.admin.swift.ListSwiftsCmd; @@ -451,7 +457,6 @@ import org.apache.cloudstack.api.command.user.template.DeleteTemplateCmd; import org.apache.cloudstack.api.command.user.template.ExtractTemplateCmd; import org.apache.cloudstack.api.command.user.template.GetUploadParamsForTemplateCmd; -import org.apache.cloudstack.api.command.user.template.ListTemplateOVFProperties; import org.apache.cloudstack.api.command.user.template.ListTemplatePermissionsCmd; import org.apache.cloudstack.api.command.user.template.ListTemplatesCmd; import org.apache.cloudstack.api.command.user.template.RegisterTemplateCmd; @@ -833,6 +838,8 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe private KeystoreManager _ksMgr; @Inject private DpdkHelper dpdkHelper; + @Inject + private PrimaryDataStoreDao _primaryDataStoreDao; private LockMasterListener _lockMasterListener; private final ScheduledExecutorService _eventExecutor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("EventChecker")); @@ -1480,10 +1487,31 @@ public Pair, List> listStorag } else { suitablePools = allPools; } - + List avoidPools = new ArrayList<>(); + if (srcVolumePool.getParent() != 0L) { + StoragePool datastoreCluster = _poolDao.findById(srcVolumePool.getParent()); + avoidPools.add(datastoreCluster); + } + abstractDataStoreClustersList((List) allPools, new ArrayList()); + abstractDataStoreClustersList((List) suitablePools, avoidPools); return new Pair, List>(allPools, suitablePools); } + private void abstractDataStoreClustersList(List storagePools, List avoidPools) { + Predicate childDatastorePredicate = pool -> (pool.getParent() != 0); + List childDatastores = storagePools.stream().filter(childDatastorePredicate).collect(Collectors.toList()); + storagePools.removeAll(avoidPools); + if (!childDatastores.isEmpty()) { + storagePools.removeAll(childDatastores); + Set parentStoragePoolIds = childDatastores.stream().map(mo -> mo.getParent()).collect(Collectors.toSet()); + for (Long parentStoragePoolId : parentStoragePoolIds) { + StoragePool parentPool = _poolDao.findById(parentStoragePoolId); + if (!storagePools.contains(parentPool) && !avoidPools.contains(parentPool)) + storagePools.add(parentPool); + } + } + } + /** * This method looks for all storage pools that are compatible with the given volume. *
    @@ -2765,6 +2793,7 @@ public List> getCommands() { cmdList.add(FindStoragePoolsForMigrationCmd.class); cmdList.add(PreparePrimaryStorageForMaintenanceCmd.class); cmdList.add(UpdateStoragePoolCmd.class); + cmdList.add(UpdateImageStoreCmd.class); cmdList.add(DestroySystemVmCmd.class); cmdList.add(ListSystemVMsCmd.class); cmdList.add(MigrateSystemVMCmd.class); @@ -3155,9 +3184,9 @@ public List> getCommands() { cmdList.add(RevokeTemplateDirectDownloadCertificateCmd.class); cmdList.add(ListMgmtsCmd.class); cmdList.add(GetUploadParamsForIsoCmd.class); - cmdList.add(ListTemplateOVFProperties.class); cmdList.add(GetRouterHealthCheckResultsCmd.class); cmdList.add(StartRollingMaintenanceCmd.class); + cmdList.add(MigrateSecondaryStorageDataCmd.class); // Out-of-band management APIs for admins cmdList.add(EnableOutOfBandManagementForHostCmd.class); @@ -3300,9 +3329,16 @@ public Pair, Integer> searchForSystemVm(final Lis sb.and("nulltype", sb.entity().getType(), SearchCriteria.Op.IN); if (storageId != null) { - final SearchBuilder volumeSearch = _volumeDao.createSearchBuilder(); - volumeSearch.and("poolId", volumeSearch.entity().getPoolId(), SearchCriteria.Op.EQ); - sb.join("volumeSearch", volumeSearch, sb.entity().getId(), volumeSearch.entity().getInstanceId(), JoinBuilder.JoinType.INNER); + StoragePoolVO storagePool = _primaryDataStoreDao.findById(storageId); + if (storagePool.getPoolType() == Storage.StoragePoolType.DatastoreCluster) { + final SearchBuilder volumeSearch = _volumeDao.createSearchBuilder(); + volumeSearch.and("poolId", volumeSearch.entity().getPoolId(), SearchCriteria.Op.IN); + sb.join("volumeSearch", volumeSearch, sb.entity().getId(), volumeSearch.entity().getInstanceId(), JoinBuilder.JoinType.INNER); + } else { + final SearchBuilder volumeSearch = _volumeDao.createSearchBuilder(); + volumeSearch.and("poolId", volumeSearch.entity().getPoolId(), SearchCriteria.Op.EQ); + sb.join("volumeSearch", volumeSearch, sb.entity().getId(), volumeSearch.entity().getInstanceId(), JoinBuilder.JoinType.INNER); + } } final SearchCriteria sc = sb.create(); @@ -3342,7 +3378,14 @@ public Pair, Integer> searchForSystemVm(final Lis } if (storageId != null) { - sc.setJoinParameters("volumeSearch", "poolId", storageId); + StoragePoolVO storagePool = _primaryDataStoreDao.findById(storageId); + if (storagePool.getPoolType() == Storage.StoragePoolType.DatastoreCluster) { + List childDataStores = _primaryDataStoreDao.listChildStoragePoolsInDatastoreCluster(storageId); + List childDatastoreIds = childDataStores.stream().map(mo -> mo.getId()).collect(Collectors.toList()); + sc.setJoinParameters("volumeSearch", "poolId", childDatastoreIds.toArray()); + } else { + sc.setJoinParameters("volumeSearch", "poolId", storageId); + } } final Pair, Integer> result = _vmInstanceDao.searchAndCount(sc, searchFilter); diff --git a/server/src/main/java/com/cloud/storage/ImageStoreServiceImpl.java b/server/src/main/java/com/cloud/storage/ImageStoreServiceImpl.java new file mode 100644 index 000000000000..43f9cd455be2 --- /dev/null +++ b/server/src/main/java/com/cloud/storage/ImageStoreServiceImpl.java @@ -0,0 +1,163 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.storage; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import javax.inject.Inject; +import javax.naming.ConfigurationException; + +import org.apache.cloudstack.api.command.admin.storage.MigrateSecondaryStorageDataCmd; +import org.apache.cloudstack.api.response.MigrationResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.engine.orchestration.service.StorageOrchestrationService; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProvider; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.jobs.AsyncJobManager; +import org.apache.cloudstack.storage.ImageStoreService; +import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; +import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; +import org.apache.commons.lang3.EnumUtils; +import org.apache.log4j.Logger; + +import com.cloud.event.ActionEvent; +import com.cloud.event.EventTypes; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.exception.CloudRuntimeException; + +public class ImageStoreServiceImpl extends ManagerBase implements ImageStoreService { + + private static final Logger s_logger = Logger.getLogger(ImageStoreServiceImpl.class); + @Inject + ImageStoreDao imageStoreDao; + @Inject + private AsyncJobManager jobMgr; + @Inject + private StorageOrchestrationService stgService; + + ConfigKey ImageStoreImbalanceThreshold = new ConfigKey<>("Advanced", Double.class, + "image.store.imbalance.threshold", + "0.3", + "The storage imbalance threshold that is compared with the standard deviation percentage for a storage utilization metric. " + + "The value is a percentage in decimal format.", + true, ConfigKey.Scope.Global); + + + public Integer numConcurrentCopyTasksPerSSVM = null; + + + + @Override + public boolean configure(String name, Map params) throws ConfigurationException { + return true; + } + + @Override + @ActionEvent(eventType = EventTypes.EVENT_IMAGE_STORE_DATA_MIGRATE, eventDescription = "migrating Image store data", async = true) + public MigrationResponse migrateData(MigrateSecondaryStorageDataCmd cmd) { + Long srcImgStoreId = cmd.getId(); + ImageStoreVO srcImageVO = imageStoreDao.findById(srcImgStoreId); + List destImgStoreIds = cmd.getMigrateTo(); + List imagestores = new ArrayList(); + String migrationType = cmd.getMigrationType(); + + // default policy is complete + MigrationPolicy policy = MigrationPolicy.COMPLETE; + + if (migrationType != null) { + if (!EnumUtils.isValidEnum(MigrationPolicy.class, migrationType.toUpperCase())) { + throw new CloudRuntimeException("Not a valid migration policy"); + } + policy = MigrationPolicy.valueOf(migrationType.toUpperCase()); + } + + String message = null; + + if (srcImageVO == null) { + throw new CloudRuntimeException("Cannot find secondary storage with id: " + srcImgStoreId); + } + + Long srcStoreDcId = srcImageVO.getDataCenterId(); + imagestores.add(srcImageVO.getName()); + if (srcImageVO.getRole() != DataStoreRole.Image) { + throw new CloudRuntimeException("Secondary storage is not of Image Role"); + } + + if (!srcImageVO.getProviderName().equals(DataStoreProvider.NFS_IMAGE)) { + throw new InvalidParameterValueException("Migration of datastore objects is supported only for NFS based image stores"); + } + + if (destImgStoreIds.contains(srcImgStoreId)) { + s_logger.debug("One of the destination stores is the same as the source image store ... Ignoring it..."); + destImgStoreIds.remove(srcImgStoreId); + } + + // Validate all the Ids correspond to valid Image stores + List destDatastores = new ArrayList<>(); + for (Long id : destImgStoreIds) { + ImageStoreVO store = imageStoreDao.findById(id); + if (store == null) { + s_logger.warn("Secondary storage with id: " + id + "is not found. Skipping it..."); + continue; + } + if (store.isReadonly()) { + s_logger.warn("Secondary storage: "+ id + " cannot be considered for migration as has read-only permission, Skipping it... "); + continue; + } + + if (!store.getProviderName().equals(DataStoreProvider.NFS_IMAGE)) { + s_logger.warn("Destination image store : " + store.getName() + " not NFS based. Store not suitable for migration!"); + continue; + } + + if (srcStoreDcId != null && store.getDataCenterId() != null && !srcStoreDcId.equals(store.getDataCenterId())) { + s_logger.warn("Source and destination stores are not in the same zone. Skipping destination store: " + store.getName()); + continue; + } + + destDatastores.add(id); + imagestores.add(store.getName()); + } + + if (destDatastores.size() < 1) { + throw new CloudRuntimeException("No destination valid store(s) available to migrate. Could" + + "be due to invalid store ID(s) or store(s) are read-only. Terminating Migration of data"); + } + + if (isMigrateJobRunning()){ + message = "A migrate job is in progress, please try again later..."; + return new MigrationResponse(message, policy.toString(), false); + } + + CallContext.current().setEventDetails("Migrating files/data objects " + "from : " + imagestores.get(0) + " to: " + imagestores.subList(1, imagestores.size())); + return stgService.migrateData(srcImgStoreId, destDatastores, policy); + } + + + // Ensures that only one migrate job may occur at a time, in order to reduce load + private boolean isMigrateJobRunning() { + long count = jobMgr.countPendingJobs(null, MigrateSecondaryStorageDataCmd.class.getName()); + if (count > 1) { + return true; + } + return false; + } +} diff --git a/server/src/main/java/com/cloud/storage/ImageStoreUploadMonitorImpl.java b/server/src/main/java/com/cloud/storage/ImageStoreUploadMonitorImpl.java index a2f97e841278..1ce5b362eb90 100755 --- a/server/src/main/java/com/cloud/storage/ImageStoreUploadMonitorImpl.java +++ b/server/src/main/java/com/cloud/storage/ImageStoreUploadMonitorImpl.java @@ -408,11 +408,11 @@ public void doInTransactionWithoutResult(TransactionStatus status) { VMTemplateVO templateUpdate = _templateDao.createForUpdate(); templateUpdate.setSize(answer.getVirtualSize()); _templateDao.update(tmpTemplate.getId(), templateUpdate); - // For multi-disk OVA, check and create data disk templates + // For multi-disk OVA, check and create data disk templates or root disks as details if (tmpTemplate.getFormat().equals(Storage.ImageFormat.OVA)) { final DataStore store = dataStoreManager.getDataStore(templateDataStore.getDataStoreId(), templateDataStore.getDataStoreRole()); final TemplateInfo templateInfo = templateFactory.getTemplate(tmpTemplate.getId(), store); - if (!templateService.createOvaDataDiskTemplates(templateInfo)) { + if (!templateService.createOvaDataDiskTemplates(templateInfo, template.isDeployAsIs())) { tmpTemplateDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.ABANDONED); tmpTemplateDataStore.setState(State.Failed); stateMachine.transitTo(tmpTemplate, VirtualMachineTemplate.Event.OperationFailed, null, _templateDao); diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index f327b6aebf06..212f6938e651 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -39,6 +39,12 @@ import javax.inject.Inject; +import com.cloud.agent.api.to.StorageFilerTO; +import com.cloud.dc.VsphereStoragePolicyVO; +import com.cloud.dc.dao.VsphereStoragePolicyDao; +import com.cloud.service.dao.ServiceOfferingDetailsDao; +import com.cloud.utils.StringUtils; +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.command.admin.storage.CancelPrimaryStorageMaintenanceCmd; import org.apache.cloudstack.api.command.admin.storage.CreateSecondaryStagingStoreCmd; import org.apache.cloudstack.api.command.admin.storage.CreateStoragePoolCmd; @@ -81,6 +87,8 @@ import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.managed.context.ManagedContextRunnable; import org.apache.cloudstack.management.ManagementServerHost; +import org.apache.cloudstack.resourcedetail.dao.DiskOfferingDetailsDao; +import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplainceCommand; import org.apache.cloudstack.storage.command.DettachCommand; import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; import org.apache.cloudstack.storage.datastore.db.ImageStoreDetailsDao; @@ -175,7 +183,6 @@ import com.cloud.utils.DateUtil; import com.cloud.utils.NumbersUtil; import com.cloud.utils.Pair; -import com.cloud.utils.StringUtils; import com.cloud.utils.UriUtils; import com.cloud.utils.component.ComponentContext; import com.cloud.utils.component.ManagerBase; @@ -298,6 +305,12 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C SnapshotService _snapshotService; @Inject StoragePoolTagsDao _storagePoolTagsDao; + @Inject + DiskOfferingDetailsDao _diskOfferingDetailsDao; + @Inject + ServiceOfferingDetailsDao _serviceOfferingDetailsDao; + @Inject + VsphereStoragePolicyDao _vsphereStoragePolicyDao; protected List _discoverers; @@ -604,6 +617,7 @@ public DataStore createLocalStorage(Host host, StoragePoolInfo pInfo) throws Con params.put("zoneId", host.getDataCenterId()); params.put("clusterId", host.getClusterId()); params.put("podId", host.getPodId()); + params.put("hypervisorType", host.getHypervisorType()); params.put("url", pInfo.getPoolType().toString() + "://" + pInfo.getHost() + "/" + pInfo.getHostPath()); params.put("name", name); params.put("localStorage", true); @@ -689,6 +703,9 @@ public PrimaryDataStoreInfo createPool(CreateStoragePoolCmd cmd) throws Resource && hypervisorType != HypervisorType.Any) { throw new InvalidParameterValueException("zone wide storage pool is not supported for hypervisor type " + hypervisor); } + } else { + ClusterVO clusterVO = _clusterDao.findById(clusterId); + hypervisorType = clusterVO.getHypervisorType(); } Map details = extractApiParamAsMap(cmd.getDetails()); @@ -706,6 +723,7 @@ public PrimaryDataStoreInfo createPool(CreateStoragePoolCmd cmd) throws Resource params.put("zoneId", zone.getId()); params.put("clusterId", clusterId); params.put("podId", podId); + params.put("hypervisorType", hypervisorType.toString()); params.put("url", cmd.getUrl()); params.put("tags", cmd.getTags()); params.put("name", cmd.getStoragePoolName()); @@ -807,6 +825,12 @@ public PrimaryDataStoreInfo updateStoragePool(UpdateStoragePoolCmd cmd) throws I if (s_logger.isDebugEnabled()) { s_logger.debug("Updating Storage Pool Tags to :" + storagePoolTags); } + if (pool.getPoolType() == StoragePoolType.DatastoreCluster) { + List childStoragePools = _storagePoolDao.listChildStoragePoolsInDatastoreCluster(pool.getId()); + for (StoragePoolVO childPool : childStoragePools) { + _storagePoolTagsDao.persist(childPool.getId(), storagePoolTags); + } + } _storagePoolTagsDao.persist(pool.getId(), storagePoolTags); } @@ -903,10 +927,54 @@ public boolean deletePool(DeletePoolCmd cmd) { s_logger.warn("Unable to delete storage id: " + id + " due to it is not in Maintenance state"); throw new InvalidParameterValueException("Unable to delete storage due to it is not in Maintenance state, id: " + id); } - Pair vlms = _volsDao.getCountAndTotalByPool(id); + + if (sPool.getPoolType() == StoragePoolType.DatastoreCluster) { + // FR41 yet to handle on failure of deletion of any of the child storage pool + if (checkIfDataStoreClusterCanbeDeleted(sPool, forced)) { + Transaction.execute(new TransactionCallbackNoReturn() { + @Override + public void doInTransactionWithoutResult(TransactionStatus status) { + List childStoragePools = _storagePoolDao.listChildStoragePoolsInDatastoreCluster(sPool.getId()); + for (StoragePoolVO childPool : childStoragePools) { + deleteDataStoreInternal(childPool, forced); + } + } + }); + } else { + throw new CloudRuntimeException("Cannot delete pool " + sPool.getName() + " as there are associated " + "non-destroyed vols for this pool"); + } + } + return deleteDataStoreInternal(sPool, forced); + } + + private boolean checkIfDataStoreClusterCanbeDeleted(StoragePoolVO sPool, boolean forced) { + List childStoragePools = _storagePoolDao.listChildStoragePoolsInDatastoreCluster(sPool.getId()); + boolean canDelete = true; + for (StoragePoolVO childPool : childStoragePools) { + Pair vlms = _volsDao.getCountAndTotalByPool(childPool.getId()); + if (forced) { + if (vlms.first() > 0) { + Pair nonDstrdVlms = _volsDao.getNonDestroyedCountAndTotalByPool(childPool.getId()); + if (nonDstrdVlms.first() > 0) { + canDelete = false; + break; + } + } + } else { + if (vlms.first() > 0) { + canDelete = false; + break; + } + } + } + return canDelete; + } + + private boolean deleteDataStoreInternal(StoragePoolVO sPool, boolean forced) { + Pair vlms = _volsDao.getCountAndTotalByPool(sPool.getId()); if (forced) { if (vlms.first() > 0) { - Pair nonDstrdVlms = _volsDao.getNonDestroyedCountAndTotalByPool(id); + Pair nonDstrdVlms = _volsDao.getNonDestroyedCountAndTotalByPool(sPool.getId()); if (nonDstrdVlms.first() > 0) { throw new CloudRuntimeException("Cannot delete pool " + sPool.getName() + " as there are associated " + "non-destroyed vols for this pool"); } @@ -942,7 +1010,7 @@ public boolean deletePool(DeletePoolCmd cmd) { } _storagePoolDao.releaseFromLockTable(lock.getId()); - s_logger.trace("Released lock for storage pool " + id); + s_logger.trace("Released lock for storage pool " + sPool.getId()); DataStoreProvider storeProvider = _dataStoreProviderMgr.getDataStoreProvider(sPool.getStorageProviderName()); DataStoreLifeCycle lifeCycle = storeProvider.getDataStoreLifeCycle(); @@ -1368,7 +1436,7 @@ public void cleanupSecondaryStorage(boolean recurring) { // so here we don't need to issue DeleteCommand to resource anymore, only need to remove db entry. try { // Cleanup templates in template_store_ref - List imageStores = _dataStoreMgr.getImageStoresByScope(new ZoneScope(null)); + List imageStores = _dataStoreMgr.getImageStoresByScopeExcludingReadOnly(new ZoneScope(null)); for (DataStore store : imageStores) { try { long storeId = store.getId(); @@ -1471,11 +1539,55 @@ public PrimaryDataStoreInfo preparePrimaryStorageForMaintenance(Long primaryStor DataStoreProvider provider = _dataStoreProviderMgr.getDataStoreProvider(primaryStorage.getStorageProviderName()); DataStoreLifeCycle lifeCycle = provider.getDataStoreLifeCycle(); DataStore store = _dataStoreMgr.getDataStore(primaryStorage.getId(), DataStoreRole.Primary); + + if (primaryStorage.getPoolType() == StoragePoolType.DatastoreCluster) { + if (primaryStorage.getStatus() == StoragePoolStatus.PrepareForMaintenance) { + throw new CloudRuntimeException(String.format("There is already a job running for preparation for maintenance of the storage pool %s", primaryStorage.getUuid())); + } + handlePrepareDatastoreCluserMaintenance(lifeCycle, primaryStorageId); + } lifeCycle.maintain(store); return (PrimaryDataStoreInfo)_dataStoreMgr.getDataStore(primaryStorage.getId(), DataStoreRole.Primary); } + private void handlePrepareDatastoreCluserMaintenance(DataStoreLifeCycle lifeCycle, Long primaryStorageId) { + StoragePoolVO datastoreCluster = _storagePoolDao.findById(primaryStorageId); + datastoreCluster.setStatus(StoragePoolStatus.PrepareForMaintenance); + _storagePoolDao.update(datastoreCluster.getId(), datastoreCluster); + + // Before preparing the datastorecluster to maintenance mode, the storagepools in the datastore cluster needs to put in maintenance + List childDatastores = _storagePoolDao.listChildStoragePoolsInDatastoreCluster(primaryStorageId); + Transaction.execute(new TransactionCallbackNoReturn() { + @Override + public void doInTransactionWithoutResult(TransactionStatus status) { + for (StoragePoolVO childDatastore : childDatastores) { + // set the pool state to prepare for maintenance, so that VMs will not migrate to the storagepools in the same cluster + childDatastore.setStatus(StoragePoolStatus.PrepareForMaintenance); + _storagePoolDao.update(childDatastore.getId(), childDatastore); + } + } + }); + for (Iterator iteratorChildDatastore = childDatastores.listIterator(); iteratorChildDatastore.hasNext(); ) { + DataStore childStore = _dataStoreMgr.getDataStore(iteratorChildDatastore.next().getId(), DataStoreRole.Primary); + try { + lifeCycle.maintain(childStore); + } catch (Exception e) { + if (s_logger.isDebugEnabled()) { + s_logger.debug(String.format("Exception on maintenance preparation of one of the child datastores in datastore cluster %d with error %s", primaryStorageId, e)); + } + // Set to ErrorInMaintenance state of all child storage pools and datastore cluster + for (StoragePoolVO childDatastore : childDatastores) { + childDatastore.setStatus(StoragePoolStatus.ErrorInMaintenance); + _storagePoolDao.update(childDatastore.getId(), childDatastore); + } + datastoreCluster.setStatus(StoragePoolStatus.ErrorInMaintenance); + _storagePoolDao.update(datastoreCluster.getId(), datastoreCluster); + throw new CloudRuntimeException(String.format("Failed to prepare maintenance mode for datastore cluster %d with error %s %s", primaryStorageId, e.getMessage(), e)); + } + } + } + @Override @DB public PrimaryDataStoreInfo cancelPrimaryStorageForMaintenance(CancelPrimaryStorageMaintenanceCmd cmd) throws ResourceUnavailableException { @@ -1498,6 +1610,16 @@ public PrimaryDataStoreInfo cancelPrimaryStorageForMaintenance(CancelPrimaryStor DataStoreProvider provider = _dataStoreProviderMgr.getDataStoreProvider(primaryStorage.getStorageProviderName()); DataStoreLifeCycle lifeCycle = provider.getDataStoreLifeCycle(); DataStore store = _dataStoreMgr.getDataStore(primaryStorage.getId(), DataStoreRole.Primary); + if (primaryStorage.getPoolType() == StoragePoolType.DatastoreCluster) { + primaryStorage.setStatus(StoragePoolStatus.Up); + _storagePoolDao.update(primaryStorage.getId(), primaryStorage); + //FR41 need to handle when one of the primary stores is unable to cancel the maintenance mode + List childDatastores = _storagePoolDao.listChildStoragePoolsInDatastoreCluster(primaryStorageId); + for (StoragePoolVO childDatastore : childDatastores) { + DataStore childStore = _dataStoreMgr.getDataStore(childDatastore.getId(), DataStoreRole.Primary); + lifeCycle.cancelMaintain(childStore); + } + } lifeCycle.cancelMaintain(store); return (PrimaryDataStoreInfo)_dataStoreMgr.getDataStore(primaryStorage.getId(), DataStoreRole.Primary); @@ -1872,6 +1994,57 @@ public boolean storagePoolHasEnoughSpaceForResize(StoragePool pool, long current } } + @Override + public boolean isStoragePoolComplaintWithStoragePolicy(List volumes, StoragePool pool) throws StorageUnavailableException { + if (volumes == null || volumes.isEmpty()) { + return false; + } + List> answers = new ArrayList>(); + + for (Volume volume : volumes) { + String storagePolicyId = null; + if (volume.getVolumeType() == Type.ROOT) { + Long vmId = volume.getInstanceId(); + if (vmId != null) { + VMInstanceVO vm = _vmInstanceDao.findByIdIncludingRemoved(vmId); + storagePolicyId = _serviceOfferingDetailsDao.getDetail(vm.getServiceOfferingId(), ApiConstants.STORAGE_POLICY); + } + } else { + storagePolicyId = _diskOfferingDetailsDao.getDetail(volume.getDiskOfferingId(), ApiConstants.STORAGE_POLICY); + } + if (org.apache.commons.lang.StringUtils.isNotEmpty(storagePolicyId)) { + VsphereStoragePolicyVO storagePolicyVO = _vsphereStoragePolicyDao.findById(Long.parseLong(storagePolicyId)); + List hostIds = getUpHostsInPool(pool.getId()); + Collections.shuffle(hostIds); + + if (hostIds == null || hostIds.isEmpty()) { + throw new StorageUnavailableException("Unable to send command to the pool " + pool.getName() + " due to there is no enabled hosts up in this cluster", pool.getId()); + } + try { + StorageFilerTO storageFilerTO = new StorageFilerTO(pool); + CheckDataStoreStoragePolicyComplainceCommand cmd = new CheckDataStoreStoragePolicyComplainceCommand(storagePolicyVO.getPolicyId(), storageFilerTO); + long targetHostId = _hvGuruMgr.getGuruProcessedCommandTargetHost(hostIds.get(0), cmd); + Answer answer = _agentMgr.send(targetHostId, cmd); + answers.add(new Pair<>(volume, answer)); + } catch (AgentUnavailableException e) { + s_logger.debug("Unable to send storage pool command to " + pool + " via " + hostIds.get(0), e); + throw new StorageUnavailableException("Unable to send command to the pool ", pool.getId()); + } catch (OperationTimedoutException e) { + s_logger.debug("Failed to process storage pool command to " + pool + " via " + hostIds.get(0), e); + throw new StorageUnavailableException("Failed to process storage command to the pool ", pool.getId()); + } + } + } + // check cummilative result for all volumes + for (Pair answer : answers) { + if (!answer.second().getResult()) { + s_logger.debug(String.format("Storage pool %s is not complaince with storage policy for volume %s", pool.getName(), answer.first().getName())); + return false; + } + } + return true; + } + private boolean checkPoolforSpace(StoragePool pool, long allocatedSizeWithTemplate, long totalAskingSize) { // allocated space includes templates StoragePoolVO poolVO = _storagePoolDao.findById(pool.getId()); @@ -2157,6 +2330,18 @@ public ImageStore migrateToObjectStore(String name, String url, String providerN return discoverImageStore(name, url, providerName, null, details); } + @Override + public ImageStore updateImageStoreStatus(Long id, Boolean readonly) { + // Input validation + ImageStoreVO imageStoreVO = _imageStoreDao.findById(id); + if (imageStoreVO == null) { + throw new IllegalArgumentException("Unable to find image store with ID: " + id); + } + imageStoreVO.setReadonly(readonly); + _imageStoreDao.update(id, imageStoreVO); + return imageStoreVO; + } + private void duplicateCacheStoreRecordsToRegionStore(long storeId) { _templateStoreDao.duplicateCacheRecordsOnRegionStore(storeId); _snapshotStoreDao.duplicateCacheRecordsOnRegionStore(storeId); @@ -2511,7 +2696,9 @@ public ConfigKey[] getConfigKeys() { KvmStorageOnlineMigrationWait, KvmAutoConvergence, MaxNumberOfManagedClusteredFileSystems, - PRIMARY_STORAGE_DOWNLOAD_WAIT + PRIMARY_STORAGE_DOWNLOAD_WAIT, + SecStorageMaxMigrateSessions, + MaxDataMigrationWaitTime }; } @@ -2542,6 +2729,15 @@ public DiskTO getDiskWithThrottling(final DataTO volTO, final Volume.Type volume return disk; } + @Override + public boolean isStoragePoolDatastoreClusterParent(StoragePool pool) { + List childStoragePools = _storagePoolDao.listChildStoragePoolsInDatastoreCluster(pool.getId()); + if (childStoragePools != null && !childStoragePools.isEmpty()) { + return true; + } + return false; + } + private void setVolumeObjectTOThrottling(VolumeObjectTO volumeTO, final ServiceOffering offering, final DiskOffering diskOffering) { volumeTO.setBytesReadRate(getDiskBytesReadRate(offering, diskOffering)); volumeTO.setBytesWriteRate(getDiskBytesWriteRate(offering, diskOffering)); diff --git a/server/src/main/java/com/cloud/storage/StoragePoolAutomationImpl.java b/server/src/main/java/com/cloud/storage/StoragePoolAutomationImpl.java index 4ffd7d80b0df..3e8822e67eb3 100644 --- a/server/src/main/java/com/cloud/storage/StoragePoolAutomationImpl.java +++ b/server/src/main/java/com/cloud/storage/StoragePoolAutomationImpl.java @@ -117,9 +117,11 @@ public boolean maintain(DataStore store) { spes = primaryDataStoreDao.listBy(pool.getDataCenterId(), pool.getPodId(), pool.getClusterId(), ScopeType.CLUSTER); } for (StoragePoolVO sp : spes) { - if (sp.getStatus() == StoragePoolStatus.PrepareForMaintenance) { - throw new CloudRuntimeException("Only one storage pool in a cluster can be in PrepareForMaintenance mode, " + sp.getId() + - " is already in PrepareForMaintenance mode "); + if (sp.getParent() != pool.getParent() && sp.getId() != pool.getParent()) { // If Datastore cluster is tried to prepare for maintenance then child storage pools are also kept in PrepareForMaintenance mode + if (sp.getStatus() == StoragePoolStatus.PrepareForMaintenance) { + throw new CloudRuntimeException("Only one storage pool in a cluster can be in PrepareForMaintenance mode, " + sp.getId() + + " is already in PrepareForMaintenance mode "); + } } } StoragePool storagePool = (StoragePool)store; @@ -319,7 +321,7 @@ public boolean cancelMaintain(DataStore store) { } } else { if (s_logger.isDebugEnabled()) { - s_logger.debug("ModifyStoragePool add secceeded"); + s_logger.debug("ModifyStoragePool add succeeded"); } } } diff --git a/server/src/main/java/com/cloud/storage/TemplateProfile.java b/server/src/main/java/com/cloud/storage/TemplateProfile.java index 304b652a589f..b90409480bca 100644 --- a/server/src/main/java/com/cloud/storage/TemplateProfile.java +++ b/server/src/main/java/com/cloud/storage/TemplateProfile.java @@ -52,6 +52,7 @@ public class TemplateProfile { Boolean isDynamicallyScalable; TemplateType templateType; Boolean directDownload; + Boolean deployAsIs; Long size; public TemplateProfile(Long templateId, Long userId, String name, String displayText, Integer bits, Boolean passwordEnabled, Boolean requiresHvm, String url, @@ -95,7 +96,7 @@ public TemplateProfile(Long templateId, Long userId, String name, String display Boolean isPublic, Boolean featured, Boolean isExtractable, ImageFormat format, Long guestOsId, List zoneId, HypervisorType hypervisorType, String accountName, Long domainId, Long accountId, String chksum, Boolean bootable, String templateTag, Map details, - Boolean sshKeyEnabled, Long imageStoreId, Boolean isDynamicallyScalable, TemplateType templateType, Boolean directDownload) { + Boolean sshKeyEnabled, Long imageStoreId, Boolean isDynamicallyScalable, TemplateType templateType, Boolean directDownload, Boolean deployAsIs) { this(templateId, userId, name, @@ -122,6 +123,7 @@ public TemplateProfile(Long templateId, Long userId, String name, String display this.isDynamicallyScalable = isDynamicallyScalable; this.templateType = templateType; this.directDownload = directDownload; + this.deployAsIs = deployAsIs; } public Long getTemplateId() { @@ -331,4 +333,8 @@ public Long getSize() { public void setSize(Long size) { this.size = size; } + + public boolean isDeployAsIs() { + return this.deployAsIs; + } } diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index dc90c37d2a73..4127b9f5fcc2 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -30,6 +30,7 @@ import javax.inject.Inject; +import com.cloud.dc.Pod; import org.apache.cloudstack.api.command.user.volume.AttachVolumeCmd; import org.apache.cloudstack.api.command.user.volume.CreateVolumeCmd; import org.apache.cloudstack.api.command.user.volume.DetachVolumeCmd; @@ -1776,7 +1777,13 @@ public Volume updateVolume(long volumeId, String path, String state, Long storag if (pool.getDataCenterId() != volume.getDataCenterId()) { throw new InvalidParameterValueException("Invalid storageId specified; refers to the pool outside of the volume's zone"); } - volume.setPoolId(pool.getId()); + if (pool.getPoolType() == Storage.StoragePoolType.DatastoreCluster) { + List childDatastores = _storagePoolDao.listChildStoragePoolsInDatastoreCluster(storageId); + Collections.shuffle(childDatastores); + volume.setPoolId(childDatastores.get(0).getId()); + } else { + volume.setPoolId(pool.getId()); + } } if (customId != null) { @@ -2192,6 +2199,13 @@ public Volume migrateVolume(MigrateVolumeCmd cmd) { throw new InvalidParameterValueException("Cannot migrate volume " + vol + "to the destination storage pool " + destPool.getName() + " as the storage pool is in maintenance mode."); } + if (destPool.getPoolType() == Storage.StoragePoolType.DatastoreCluster) { + DataCenter dc = _entityMgr.findById(DataCenter.class, vol.getDataCenterId()); + Pod destPoolPod = _entityMgr.findById(Pod.class, destPool.getPodId()); + + destPool = _volumeMgr.findChildDataStoreInDataStoreCluster(dc, destPoolPod, destPool.getClusterId(), null, null, destPool.getId()); + } + if (!storageMgr.storagePoolHasEnoughSpace(Collections.singletonList(vol), destPool)) { throw new CloudRuntimeException("Storage pool " + destPool.getName() + " does not have enough space to migrate volume " + vol.getName()); } @@ -2788,24 +2802,26 @@ private String orchestrateExtractVolume(long volumeId, long zoneId) { // Copy volume from primary to secondary storage VolumeInfo srcVol = volFactory.getVolume(volumeId); - AsyncCallFuture cvAnswer = volService.copyVolume(srcVol, secStore); - // Check if you got a valid answer. + VolumeInfo destVol = volFactory.getVolume(volumeId, DataStoreRole.Image); VolumeApiResult cvResult = null; - try { - cvResult = cvAnswer.get(); - } catch (InterruptedException e1) { - s_logger.debug("failed copy volume", e1); - throw new CloudRuntimeException("Failed to copy volume", e1); - } catch (ExecutionException e1) { - s_logger.debug("failed copy volume", e1); - throw new CloudRuntimeException("Failed to copy volume", e1); - } - if (cvResult == null || cvResult.isFailed()) { - String errorString = "Failed to copy the volume from the source primary storage pool to secondary storage."; - throw new CloudRuntimeException(errorString); + if (destVol == null) { + AsyncCallFuture cvAnswer = volService.copyVolume(srcVol, secStore); + // Check if you got a valid answer. + try { + cvResult = cvAnswer.get(); + } catch (InterruptedException e1) { + s_logger.debug("failed copy volume", e1); + throw new CloudRuntimeException("Failed to copy volume", e1); + } catch (ExecutionException e1) { + s_logger.debug("failed copy volume", e1); + throw new CloudRuntimeException("Failed to copy volume", e1); + } + if (cvResult == null || cvResult.isFailed()) { + String errorString = "Failed to copy the volume from the source primary storage pool to secondary storage."; + throw new CloudRuntimeException(errorString); + } } - - VolumeInfo vol = cvResult.getVolume(); + VolumeInfo vol = cvResult != null ? cvResult.getVolume() : destVol; String extractUrl = secStore.createEntityExtractUrl(vol.getPath(), vol.getFormat(), vol); VolumeDataStoreVO volumeStoreRef = _volumeStoreDao.findByVolume(volumeId); diff --git a/server/src/main/java/com/cloud/storage/download/DownloadListener.java b/server/src/main/java/com/cloud/storage/download/DownloadListener.java index 51f9d42980cc..25dffb3e8afa 100644 --- a/server/src/main/java/com/cloud/storage/download/DownloadListener.java +++ b/server/src/main/java/com/cloud/storage/download/DownloadListener.java @@ -297,7 +297,7 @@ else if ( cmd instanceof StartupStorageCommand) { }*/ else if (cmd instanceof StartupSecondaryStorageCommand) { try{ - List imageStores = _storeMgr.getImageStoresByScope(new ZoneScope(agent.getDataCenterId())); + List imageStores = _storeMgr.getImageStoresByScopeExcludingReadOnly(new ZoneScope(agent.getDataCenterId())); for (DataStore store : imageStores) { _volumeSrv.handleVolumeSync(store); _imageSrv.handleTemplateSync(store); diff --git a/server/src/main/java/com/cloud/storage/secondary/SecondaryStorageVmManager.java b/server/src/main/java/com/cloud/storage/secondary/SecondaryStorageVmManager.java index 5c50d46f4dda..b57b44303209 100644 --- a/server/src/main/java/com/cloud/storage/secondary/SecondaryStorageVmManager.java +++ b/server/src/main/java/com/cloud/storage/secondary/SecondaryStorageVmManager.java @@ -23,6 +23,7 @@ import com.cloud.host.HostVO; import com.cloud.utils.Pair; import com.cloud.utils.component.Manager; +import com.cloud.vm.SecondaryStorageVm; import com.cloud.vm.SecondaryStorageVmVO; public interface SecondaryStorageVmManager extends Manager { @@ -31,6 +32,7 @@ public interface SecondaryStorageVmManager extends Manager { public static final int DEFAULT_SS_VM_CPUMHZ = 500; // 500 MHz public static final int DEFAULT_SS_VM_MTUSIZE = 1500; public static final int DEFAULT_SS_VM_CAPACITY = 50; // max command execution session per SSVM + public static final int DEFAULT_MIGRATE_SS_VM_CAPACITY = 2; // number of concurrent migrate operations to happen per SSVM public static final int DEFAULT_STANDBY_CAPACITY = 10; // standy capacity to reserve per zone public static final String ALERT_SUBJECT = "secondarystoragevm-alert"; @@ -56,4 +58,6 @@ public interface SecondaryStorageVmManager extends Manager { public List listUpAndConnectingSecondaryStorageVmHost(Long dcId); public HostVO pickSsvmHost(HostVO ssHost); + + void allocCapacity(long dataCenterId, SecondaryStorageVm.Role role); } diff --git a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java index 35ec665b97d3..1eb72e2a24c7 100755 --- a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java @@ -311,7 +311,8 @@ public Snapshot revertSnapshot(Long snapshotId) { if (snapshotStrategy == null) { s_logger.error("Unable to find snaphot strategy to handle snapshot with id '" + snapshotId + "'"); - return null; + String errorMsg = String.format("Revert snapshot command failed for snapshot with id %d, because this command is supported only for KVM hypervisor", snapshotId); + throw new CloudRuntimeException(errorMsg); } boolean result = snapshotStrategy.revertSnapshot(snapshotInfo); diff --git a/server/src/main/java/com/cloud/storage/upload/params/UploadParams.java b/server/src/main/java/com/cloud/storage/upload/params/UploadParams.java index 0d42b760b6d9..be8319c9e570 100644 --- a/server/src/main/java/com/cloud/storage/upload/params/UploadParams.java +++ b/server/src/main/java/com/cloud/storage/upload/params/UploadParams.java @@ -46,4 +46,5 @@ public interface UploadParams { boolean isDynamicallyScalable(); boolean isRoutingType(); boolean isDirectDownload(); + boolean isDeployAsIs(); } diff --git a/server/src/main/java/com/cloud/storage/upload/params/UploadParamsBase.java b/server/src/main/java/com/cloud/storage/upload/params/UploadParamsBase.java index 67b04f7b4800..e5bc1a3c906d 100644 --- a/server/src/main/java/com/cloud/storage/upload/params/UploadParamsBase.java +++ b/server/src/main/java/com/cloud/storage/upload/params/UploadParamsBase.java @@ -214,6 +214,11 @@ public boolean isDirectDownload() { return false; } + @Override + public boolean isDeployAsIs() { + return false; + } + void setIso(boolean iso) { isIso = iso; } diff --git a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java index 85c4a77774e8..18d37d0b8adb 100644 --- a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java +++ b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java @@ -16,10 +16,6 @@ // under the License. package com.cloud.template; -import com.cloud.agent.api.Answer; -import com.cloud.host.HostVO; -import com.cloud.hypervisor.Hypervisor; -import com.cloud.resource.ResourceManager; import java.util.Collections; import java.util.HashSet; import java.util.LinkedList; @@ -30,25 +26,23 @@ import javax.inject.Inject; import com.cloud.configuration.Config; +import com.cloud.deployasis.dao.TemplateDeployAsIsDetailsDao; +import com.cloud.storage.dao.VMTemplateDetailsDao; import com.cloud.utils.db.Transaction; import com.cloud.utils.db.TransactionCallback; import com.cloud.utils.db.TransactionStatus; import org.apache.cloudstack.agent.directdownload.CheckUrlAnswer; import org.apache.cloudstack.agent.directdownload.CheckUrlCommand; -import org.apache.cloudstack.api.command.user.iso.GetUploadParamsForIsoCmd; -import org.apache.cloudstack.api.command.user.template.GetUploadParamsForTemplateCmd; -import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; -import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; -import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand; -import org.apache.cloudstack.utils.security.DigestHelper; -import org.apache.log4j.Logger; -import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.api.command.user.iso.DeleteIsoCmd; +import org.apache.cloudstack.api.command.user.iso.GetUploadParamsForIsoCmd; import org.apache.cloudstack.api.command.user.iso.RegisterIsoCmd; import org.apache.cloudstack.api.command.user.template.DeleteTemplateCmd; +import org.apache.cloudstack.api.command.user.template.GetUploadParamsForTemplateCmd; import org.apache.cloudstack.api.command.user.template.RegisterTemplateCmd; +import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; import org.apache.cloudstack.engine.subsystem.api.storage.Scope; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory; @@ -62,10 +56,15 @@ import org.apache.cloudstack.framework.async.AsyncRpcContext; import org.apache.cloudstack.framework.messagebus.MessageBus; import org.apache.cloudstack.framework.messagebus.PublishScope; +import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; +import org.apache.cloudstack.utils.security.DigestHelper; +import org.apache.log4j.Logger; import com.cloud.agent.AgentManager; +import com.cloud.agent.api.Answer; import com.cloud.alert.AlertManager; import com.cloud.configuration.Resource.ResourceType; import com.cloud.dc.DataCenterVO; @@ -74,11 +73,11 @@ import com.cloud.event.UsageEventUtils; import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.ResourceAllocationException; +import com.cloud.host.HostVO; +import com.cloud.hypervisor.Hypervisor; import com.cloud.org.Grouping; +import com.cloud.resource.ResourceManager; import com.cloud.server.StatsCollector; -import com.cloud.template.VirtualMachineTemplate.State; -import com.cloud.user.Account; -import com.cloud.utils.Pair; import com.cloud.storage.ScopeType; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.Storage.TemplateType; @@ -89,6 +88,9 @@ import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VMTemplateZoneDao; import com.cloud.storage.download.DownloadMonitor; +import com.cloud.template.VirtualMachineTemplate.State; +import com.cloud.user.Account; +import com.cloud.utils.Pair; import com.cloud.utils.UriUtils; import com.cloud.utils.db.DB; import com.cloud.utils.db.EntityManager; @@ -126,6 +128,10 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { ResourceManager resourceManager; @Inject VMTemplateDao templateDao; + @Inject + private VMTemplateDetailsDao templateDetailsDao; + @Inject + private TemplateDeployAsIsDetailsDao templateDeployAsIsDetailsDao; @Override public String getName() { @@ -592,6 +598,14 @@ public boolean delete(TemplateProfile profile) { Pair, Long> tmplt = new Pair, Long>(VirtualMachineTemplate.class, template.getId()); _messageBus.publish(_name, EntityManager.MESSAGE_REMOVE_ENTITY_EVENT, PublishScope.LOCAL, tmplt); + // Remove template details + templateDetailsDao.removeDetails(template.getId()); + + // Remove deploy-as-is details + if (template.isDeployAsIs()) { + templateDeployAsIsDetailsDao.removeDetails(template.getId()); + } + } return success; diff --git a/server/src/main/java/com/cloud/template/TemplateAdapter.java b/server/src/main/java/com/cloud/template/TemplateAdapter.java index c048ceaf1fc2..86dd0d3cad5d 100644 --- a/server/src/main/java/com/cloud/template/TemplateAdapter.java +++ b/server/src/main/java/com/cloud/template/TemplateAdapter.java @@ -72,13 +72,12 @@ public String getName() { boolean delete(TemplateProfile profile); - TemplateProfile prepare(boolean isIso, Long userId, String name, String displayText, Integer bits, Boolean passwordEnabled, Boolean requiresHVM, String url, - Boolean isPublic, Boolean featured, Boolean isExtractable, String format, Long guestOSId, List zoneId, HypervisorType hypervisorType, String accountName, - Long domainId, String chksum, Boolean bootable, Map details, boolean directDownload) throws ResourceAllocationException; - - TemplateProfile prepare(boolean isIso, long userId, String name, String displayText, Integer bits, Boolean passwordEnabled, Boolean requiresHVM, String url, - Boolean isPublic, Boolean featured, Boolean isExtractable, String format, Long guestOSId, List zoneId, HypervisorType hypervisorType, String chksum, - Boolean bootable, String templateTag, Account templateOwner, Map details, Boolean sshKeyEnabled, String imageStoreUuid, Boolean isDynamicallyScalable, - TemplateType templateType, boolean directDownload) throws ResourceAllocationException; + TemplateProfile prepare(boolean isIso, Long userId, String name, String displayText, Integer bits, Boolean passwordEnabled, Boolean requiresHVM, String url, Boolean isPublic, + Boolean featured, Boolean isExtractable, String format, Long guestOSId, List zoneId, HypervisorType hypervisorType, String accountName, Long domainId, String chksum, Boolean bootable, Map details, boolean directDownload, + boolean deployAsIs) throws ResourceAllocationException; + + TemplateProfile prepare(boolean isIso, long userId, String name, String displayText, Integer bits, Boolean passwordEnabled, Boolean requiresHVM, String url, Boolean isPublic, + Boolean featured, Boolean isExtractable, String format, Long guestOSId, List zoneId, HypervisorType hypervisorType, String chksum, Boolean bootable, String templateTag, Account templateOwner, Map details, Boolean sshKeyEnabled, String imageStoreUuid, Boolean isDynamicallyScalable, + TemplateType templateType, boolean directDownload, boolean deployAsIs) throws ResourceAllocationException; } diff --git a/server/src/main/java/com/cloud/template/TemplateAdapterBase.java b/server/src/main/java/com/cloud/template/TemplateAdapterBase.java index 0e88c147f512..d016fed4a2e5 100644 --- a/server/src/main/java/com/cloud/template/TemplateAdapterBase.java +++ b/server/src/main/java/com/cloud/template/TemplateAdapterBase.java @@ -129,16 +129,16 @@ public boolean stop() { @Override public TemplateProfile prepare(boolean isIso, Long userId, String name, String displayText, Integer bits, Boolean passwordEnabled, Boolean requiresHVM, String url, Boolean isPublic, Boolean featured, Boolean isExtractable, String format, Long guestOSId, List zoneId, HypervisorType hypervisorType, String accountName, - Long domainId, String chksum, Boolean bootable, Map details, boolean directDownload) throws ResourceAllocationException { + Long domainId, String chksum, Boolean bootable, Map details, boolean directDownload, boolean deployAsIs) throws ResourceAllocationException { return prepare(isIso, userId, name, displayText, bits, passwordEnabled, requiresHVM, url, isPublic, featured, isExtractable, format, guestOSId, zoneId, - hypervisorType, chksum, bootable, null, null, details, false, null, false, TemplateType.USER, directDownload); + hypervisorType, chksum, bootable, null, null, details, false, null, false, TemplateType.USER, directDownload, deployAsIs); } @Override public TemplateProfile prepare(boolean isIso, long userId, String name, String displayText, Integer bits, Boolean passwordEnabled, Boolean requiresHVM, String url, Boolean isPublic, Boolean featured, Boolean isExtractable, String format, Long guestOSId, List zoneIdList, HypervisorType hypervisorType, String chksum, Boolean bootable, String templateTag, Account templateOwner, Map details, Boolean sshkeyEnabled, String imageStoreUuid, Boolean isDynamicallyScalable, - TemplateType templateType, boolean directDownload) throws ResourceAllocationException { + TemplateType templateType, boolean directDownload, boolean deployAsIs) throws ResourceAllocationException { //Long accountId = null; // parameters verification @@ -257,7 +257,7 @@ public TemplateProfile prepare(boolean isIso, long userId, String name, String d CallContext.current().setEventDetails("Id: " + id + " name: " + name); return new TemplateProfile(id, userId, name, displayText, bits, passwordEnabled, requiresHVM, url, isPublic, featured, isExtractable, imgfmt, guestOSId, zoneIdList, hypervisorType, templateOwner.getAccountName(), templateOwner.getDomainId(), templateOwner.getAccountId(), chksum, bootable, templateTag, details, - sshkeyEnabled, null, isDynamicallyScalable, templateType, directDownload); + sshkeyEnabled, null, isDynamicallyScalable, templateType, directDownload, deployAsIs); } @@ -285,7 +285,8 @@ public TemplateProfile prepare(RegisterTemplateCmd cmd) throws ResourceAllocatio return prepare(false, CallContext.current().getCallingUserId(), cmd.getTemplateName(), cmd.getDisplayText(), cmd.getBits(), cmd.isPasswordEnabled(), cmd.getRequiresHvm(), cmd.getUrl(), cmd.isPublic(), cmd.isFeatured(), cmd.isExtractable(), cmd.getFormat(), cmd.getOsTypeId(), zoneId, hypervisorType, cmd.getChecksum(), true, - cmd.getTemplateTag(), owner, cmd.getDetails(), cmd.isSshKeyEnabled(), null, cmd.isDynamicallyScalable(), isRouting ? TemplateType.ROUTING : TemplateType.USER, cmd.isDirectDownload()); + cmd.getTemplateTag(), owner, cmd.getDetails(), cmd.isSshKeyEnabled(), null, cmd.isDynamicallyScalable(), isRouting ? TemplateType.ROUTING : TemplateType.USER, + cmd.isDirectDownload(), cmd.isDeployAsIs()); } @@ -316,7 +317,7 @@ private TemplateProfile prepareUploadParamsInternal(UploadParams params) throws params.isExtractable(), params.getFormat(), params.getGuestOSId(), zoneList, params.getHypervisorType(), params.getChecksum(), params.isBootable(), params.getTemplateTag(), owner, params.getDetails(), params.isSshKeyEnabled(), params.getImageStoreUuid(), - params.isDynamicallyScalable(), params.isRoutingType() ? TemplateType.ROUTING : TemplateType.USER, params.isDirectDownload()); + params.isDynamicallyScalable(), params.isRoutingType() ? TemplateType.ROUTING : TemplateType.USER, params.isDirectDownload(), false); } @Override @@ -358,7 +359,7 @@ public TemplateProfile prepare(RegisterIsoCmd cmd) throws ResourceAllocationExce return prepare(true, CallContext.current().getCallingUserId(), cmd.getIsoName(), cmd.getDisplayText(), 64, cmd.isPasswordEnabled(), true, cmd.getUrl(), cmd.isPublic(), cmd.isFeatured(), cmd.isExtractable(), ImageFormat.ISO.toString(), cmd.getOsTypeId(), zoneList, HypervisorType.None, cmd.getChecksum(), cmd.isBootable(), null, - owner, null, false, cmd.getImageStoreUuid(), cmd.isDynamicallyScalable(), TemplateType.USER, cmd.isDirectDownload()); + owner, null, false, cmd.getImageStoreUuid(), cmd.isDynamicallyScalable(), TemplateType.USER, cmd.isDirectDownload(), false); } protected VMTemplateVO persistTemplate(TemplateProfile profile, VirtualMachineTemplate.State initialState) { @@ -367,7 +368,7 @@ protected VMTemplateVO persistTemplate(TemplateProfile profile, VirtualMachineTe new VMTemplateVO(profile.getTemplateId(), profile.getName(), profile.getFormat(), profile.isPublic(), profile.isFeatured(), profile.isExtractable(), profile.getTemplateType(), profile.getUrl(), profile.isRequiresHVM(), profile.getBits(), profile.getAccountId(), profile.getCheckSum(), profile.getDisplayText(), profile.isPasswordEnabled(), profile.getGuestOsId(), profile.isBootable(), profile.getHypervisorType(), - profile.getTemplateTag(), profile.getDetails(), profile.isSshKeyEnabled(), profile.IsDynamicallyScalable(), profile.isDirectDownload()); + profile.getTemplateTag(), profile.getDetails(), profile.isSshKeyEnabled(), profile.IsDynamicallyScalable(), profile.isDirectDownload(), profile.isDeployAsIs()); template.setState(initialState); if (profile.isDirectDownload()) { diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index 749f272bf361..85417f76d3b4 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -32,6 +32,7 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.agent.api.to.DatadiskTO; import org.apache.cloudstack.acl.SecurityChecker.AccessType; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseListTemplateOrIsoPermissionsCmd; @@ -616,6 +617,18 @@ public void prepareIsoForVmProfile(VirtualMachineProfile profile, DeployDestinat private void prepareTemplateInOneStoragePool(final VMTemplateVO template, final StoragePoolVO pool) { s_logger.info("Schedule to preload template " + template.getId() + " into primary storage " + pool.getId()); + if (pool.getPoolType() == Storage.StoragePoolType.DatastoreCluster) { + List childDataStores = _poolDao.listChildStoragePoolsInDatastoreCluster(pool.getId()); + s_logger.debug("Schedule to preload template " + template.getId() + " into child datastores of DataStore cluster: " + pool.getId()); + for (StoragePoolVO childDataStore : childDataStores) { + prepareTemplateInOneStoragePoolInternal(template, childDataStore); + } + } else { + prepareTemplateInOneStoragePoolInternal(template, pool); + } + } + + private void prepareTemplateInOneStoragePoolInternal(final VMTemplateVO template, final StoragePoolVO pool) { _preloadExecutor.execute(new ManagedContextRunnable() { @Override protected void runInContext() { @@ -657,7 +670,7 @@ public VMTemplateStoragePoolVO prepareTemplateForCreate(VMTemplateVO templ, Stor VMTemplateStoragePoolVO templateStoragePoolRef = null; TemplateDataStoreVO templateStoreRef = null; - templateStoragePoolRef = _tmpltPoolDao.findByPoolTemplate(poolId, templateId); + templateStoragePoolRef = _tmpltPoolDao.findByPoolTemplate(poolId, templateId, null); if (templateStoragePoolRef != null) { templateStoragePoolRef.setMarkedForGC(false); _tmpltPoolDao.update(templateStoragePoolRef.getId(), templateStoragePoolRef); @@ -697,7 +710,7 @@ public VMTemplateStoragePoolVO prepareTemplateForCreate(VMTemplateVO templ, Stor return null; } - return _tmpltPoolDao.findByPoolTemplate(poolId, templateId); + return _tmpltPoolDao.findByPoolTemplate(poolId, templateId, null); } catch (Exception ex) { s_logger.debug("failed to copy template from image store:" + srcSecStore.getName() + " to primary storage"); } @@ -754,7 +767,7 @@ public boolean copy(long userId, VMTemplateVO template, DataStore srcSecStore, D long tmpltId = template.getId(); long dstZoneId = dstZone.getId(); // find all eligible image stores for the destination zone - List dstSecStores = _dataStoreMgr.getImageStoresByScope(new ZoneScope(dstZoneId)); + List dstSecStores = _dataStoreMgr.getImageStoresByScopeExcludingReadOnly(new ZoneScope(dstZoneId)); if (dstSecStores == null || dstSecStores.isEmpty()) { throw new StorageUnavailableException("Destination zone is not ready, no image store associated", DataCenter.class, dstZone.getId()); } @@ -1009,7 +1022,7 @@ public void evictTemplateFromStoragePool(VMTemplateStoragePoolVO templatePoolVO) } PrimaryDataStore pool = (PrimaryDataStore)_dataStoreMgr.getPrimaryDataStore(templatePoolVO.getPoolId()); - TemplateInfo template = _tmplFactory.getTemplate(templatePoolRef.getTemplateId(), pool); + TemplateInfo template = _tmplFactory.getTemplateOnPrimaryStorage(templatePoolRef.getTemplateId(), pool, templatePoolRef.getDeploymentOption()); try { if (s_logger.isDebugEnabled()) { @@ -1885,7 +1898,7 @@ public VMTemplateVO createPrivateTemplateRecord(CreateTemplateCmd cmd, Account t } privateTemplate = new VMTemplateVO(nextTemplateId, name, ImageFormat.RAW, isPublic, featured, isExtractable, TemplateType.USER, null, requiresHvmValue, bitsValue, templateOwner.getId(), null, description, - passwordEnabledValue, guestOS.getId(), true, hyperType, templateTag, cmd.getDetails(), sshKeyEnabledValue, isDynamicScalingEnabled, false); + passwordEnabledValue, guestOS.getId(), true, hyperType, templateTag, cmd.getDetails(), sshKeyEnabledValue, isDynamicScalingEnabled, false, false); if (sourceTemplateId != null) { if (s_logger.isDebugEnabled()) { @@ -2211,4 +2224,16 @@ public List getTemplateAdapters() { public void setTemplateAdapters(List adapters) { _adapters = adapters; } + + @Override + public List getTemplateDisksOnImageStore(Long templateId, DataStoreRole role, String configurationId) { + TemplateInfo templateObject = _tmplFactory.getTemplate(templateId, role); + if (templateObject == null) { + String msg = String.format("Could not find template %s downloaded on store with role %s", templateId, role.toString()); + s_logger.error(msg); + throw new CloudRuntimeException(msg); + } + return _tmpltSvr.getTemplateDatadisksOnImageStore(templateObject, configurationId); + } + } diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index f99a0663e298..cc2ec6105bd0 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -47,8 +47,12 @@ import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; +import com.cloud.agent.api.to.deployasis.OVFPropertyTO; +import com.cloud.deployasis.UserVmDeployAsIsDetailVO; +import com.cloud.deployasis.dao.UserVmDeployAsIsDetailsDao; import com.cloud.exception.UnsupportedServiceException; import com.cloud.hypervisor.Hypervisor; +import com.cloud.deployasis.dao.TemplateDeployAsIsDetailsDao; import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.acl.SecurityChecker.AccessType; import org.apache.cloudstack.affinity.AffinityGroupService; @@ -79,6 +83,7 @@ import org.apache.cloudstack.api.command.user.vmgroup.CreateVMGroupCmd; import org.apache.cloudstack.api.command.user.vmgroup.DeleteVMGroupCmd; import org.apache.cloudstack.api.command.user.volume.ResizeVolumeCmd; +import com.cloud.agent.api.to.deployasis.OVFNetworkTO; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.cloud.entity.api.VirtualMachineEntity; import org.apache.cloudstack.engine.cloud.entity.api.db.dao.VMNetworkMapDao; @@ -105,6 +110,7 @@ import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import org.apache.commons.codec.binary.Base64; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; @@ -257,7 +263,6 @@ import com.cloud.storage.Storage.TemplateType; import com.cloud.storage.StoragePool; import com.cloud.storage.StoragePoolStatus; -import com.cloud.storage.TemplateOVFPropertyVO; import com.cloud.storage.VMTemplateStorageResourceAssoc; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.VMTemplateZoneVO; @@ -268,7 +273,6 @@ import com.cloud.storage.dao.GuestOSCategoryDao; import com.cloud.storage.dao.GuestOSDao; import com.cloud.storage.dao.SnapshotDao; -import com.cloud.storage.dao.TemplateOVFPropertiesDao; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VMTemplateZoneDao; import com.cloud.storage.dao.VolumeDao; @@ -503,7 +507,9 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir @Inject private ResourceTagDao resourceTagDao; @Inject - private TemplateOVFPropertiesDao templateOVFPropertiesDao; + private TemplateDeployAsIsDetailsDao templateDeployAsIsDetailsDao; + @Inject + private UserVmDeployAsIsDetailsDao userVmDeployAsIsDetailsDao; private ScheduledExecutorService _executor = null; private ScheduledExecutorService _vmIpFetchExecutor = null; @@ -2481,15 +2487,6 @@ public UserVm updateVirtualMachine(UpdateVMCmd cmd) throws ResourceUnavailableEx } } } - for (String detailName : details.keySet()) { - if (detailName.startsWith(ApiConstants.OVF_PROPERTIES)) { - String ovfPropKey = detailName.replace(ApiConstants.OVF_PROPERTIES + "-", ""); - TemplateOVFPropertyVO ovfPropertyVO = templateOVFPropertiesDao.findByTemplateAndKey(vmInstance.getTemplateId(), ovfPropKey); - if (ovfPropertyVO != null && ovfPropertyVO.isPassword()) { - details.put(detailName, DBEncryptionUtil.encrypt(details.get(detailName))); - } - } - } vmInstance.setDetails(details); _vmDao.saveDetails(vmInstance); } @@ -3289,56 +3286,10 @@ public UserVm createAdvancedVirtualMachine(DataCenter zone, ServiceOffering serv List vpcSupportedHTypes = _vpcMgr.getSupportedVpcHypervisors(); if (networkIdList == null || networkIdList.isEmpty()) { - NetworkVO defaultNetwork = null; - - // if no network is passed in - // Check if default virtual network offering has - // Availability=Required. If it's true, search for corresponding - // network - // * if network is found, use it. If more than 1 virtual network is - // found, throw an error - // * if network is not found, create a new one and use it - - List requiredOfferings = _networkOfferingDao.listByAvailability(Availability.Required, false); - if (requiredOfferings.size() < 1) { - throw new InvalidParameterValueException("Unable to find network offering with availability=" + Availability.Required - + " to automatically create the network as a part of vm creation"); - } - - if (requiredOfferings.get(0).getState() == NetworkOffering.State.Enabled) { - // get Virtual networks - List virtualNetworks = _networkModel.listNetworksForAccount(owner.getId(), zone.getId(), Network.GuestType.Isolated); - if (virtualNetworks == null) { - throw new InvalidParameterValueException("No (virtual) networks are found for account " + owner); - } - if (virtualNetworks.isEmpty()) { - long physicalNetworkId = _networkModel.findPhysicalNetworkId(zone.getId(), requiredOfferings.get(0).getTags(), requiredOfferings.get(0).getTrafficType()); - // Validate physical network - PhysicalNetwork physicalNetwork = _physicalNetworkDao.findById(physicalNetworkId); - if (physicalNetwork == null) { - throw new InvalidParameterValueException("Unable to find physical network with id: " + physicalNetworkId + " and tag: " - + requiredOfferings.get(0).getTags()); - } - s_logger.debug("Creating network for account " + owner + " from the network offering id=" + requiredOfferings.get(0).getId() + " as a part of deployVM process"); - Network newNetwork = _networkMgr.createGuestNetwork(requiredOfferings.get(0).getId(), owner.getAccountName() + "-network", owner.getAccountName() + "-network", - null, null, null, false, null, owner, null, physicalNetwork, zone.getId(), ACLType.Account, null, null, null, null, true, null, null, - null); - if (newNetwork != null) { - defaultNetwork = _networkDao.findById(newNetwork.getId()); - } - } else if (virtualNetworks.size() > 1) { - throw new InvalidParameterValueException("More than 1 default Isolated networks are found for account " + owner + "; please specify networkIds"); - } else { - defaultNetwork = _networkDao.findById(virtualNetworks.get(0).getId()); - } - } else { - throw new InvalidParameterValueException("Required network offering id=" + requiredOfferings.get(0).getId() + " is not in " + NetworkOffering.State.Enabled); - } - + NetworkVO defaultNetwork = getDefaultNetwork(zone, owner, false); if (defaultNetwork != null) { networkList.add(defaultNetwork); } - } else { for (Long networkId : networkIdList) { NetworkVO network = _networkDao.findById(networkId); @@ -3375,6 +3326,91 @@ public UserVm createAdvancedVirtualMachine(DataCenter zone, ServiceOffering serv dataDiskTemplateToDiskOfferingMap, userVmOVFPropertiesMap); } + private NetworkVO getNetworkToAddToNetworkList(VirtualMachineTemplate template, Account owner, HypervisorType hypervisor, + List vpcSupportedHTypes, Long networkId) { + NetworkVO network = _networkDao.findById(networkId); + if (network == null) { + throw new InvalidParameterValueException("Unable to find network by id " + networkId); + } + if (network.getVpcId() != null) { + // Only ISOs, XenServer, KVM, and VmWare template types are + // supported for vpc networks + if (template.getFormat() != ImageFormat.ISO && !vpcSupportedHTypes.contains(template.getHypervisorType())) { + throw new InvalidParameterValueException("Can't create vm from template with hypervisor " + template.getHypervisorType() + " in vpc network " + network); + } else if (template.getFormat() == ImageFormat.ISO && !vpcSupportedHTypes.contains(hypervisor)) { + // Only XenServer, KVM, and VMware hypervisors are supported + // for vpc networks + throw new InvalidParameterValueException("Can't create vm of hypervisor type " + hypervisor + " in vpc network"); + } + } + + _networkModel.checkNetworkPermissions(owner, network); + + // don't allow to use system networks + NetworkOffering networkOffering = _entityMgr.findById(NetworkOffering.class, network.getNetworkOfferingId()); + if (networkOffering.isSystemOnly()) { + throw new InvalidParameterValueException("Network id=" + networkId + " is system only and can't be used for vm deployment"); + } + return network; + } + + private NetworkVO getDefaultNetwork(DataCenter zone, Account owner, boolean selectAny) throws InsufficientCapacityException, ResourceAllocationException { + NetworkVO defaultNetwork = null; + + // if no network is passed in + // Check if default virtual network offering has + // Availability=Required. If it's true, search for corresponding + // network + // * if network is found, use it. If more than 1 virtual network is + // found, throw an error + // * if network is not found, create a new one and use it + + List requiredOfferings = _networkOfferingDao.listByAvailability(Availability.Required, false); + if (requiredOfferings.size() < 1) { + throw new InvalidParameterValueException("Unable to find network offering with availability=" + Availability.Required + + " to automatically create the network as a part of vm creation"); + } + + if (requiredOfferings.get(0).getState() == NetworkOffering.State.Enabled) { + // get Virtual networks + List virtualNetworks = _networkModel.listNetworksForAccount(owner.getId(), zone.getId(), Network.GuestType.Isolated); + if (virtualNetworks == null) { + throw new InvalidParameterValueException("No (virtual) networks are found for account " + owner); + } + if (virtualNetworks.isEmpty()) { + defaultNetwork = createDefaultNetworkForAccount(zone, owner, requiredOfferings); + } else if (virtualNetworks.size() > 1 && !selectAny) { + throw new InvalidParameterValueException("More than 1 default Isolated networks are found for account " + owner + "; please specify networkIds"); + } else { + defaultNetwork = _networkDao.findById(virtualNetworks.get(0).getId()); + } + } else { + throw new InvalidParameterValueException("Required network offering id=" + requiredOfferings.get(0).getId() + " is not in " + NetworkOffering.State.Enabled); + } + + return defaultNetwork; + } + + private NetworkVO createDefaultNetworkForAccount(DataCenter zone, Account owner, List requiredOfferings) + throws InsufficientCapacityException, ResourceAllocationException { + NetworkVO defaultNetwork = null; + long physicalNetworkId = _networkModel.findPhysicalNetworkId(zone.getId(), requiredOfferings.get(0).getTags(), requiredOfferings.get(0).getTrafficType()); + // Validate physical network + PhysicalNetwork physicalNetwork = _physicalNetworkDao.findById(physicalNetworkId); + if (physicalNetwork == null) { + throw new InvalidParameterValueException("Unable to find physical network with id: " + physicalNetworkId + " and tag: " + + requiredOfferings.get(0).getTags()); + } + s_logger.debug("Creating network for account " + owner + " from the network offering id=" + requiredOfferings.get(0).getId() + " as a part of deployVM process"); + Network newNetwork = _networkMgr.createGuestNetwork(requiredOfferings.get(0).getId(), owner.getAccountName() + "-network", owner.getAccountName() + "-network", + null, null, null, false, null, owner, null, physicalNetwork, zone.getId(), ACLType.Account, null, null, null, null, true, null, null, + null); + if (newNetwork != null) { + defaultNetwork = _networkDao.findById(newNetwork.getId()); + } + return defaultNetwork; + } + private void verifyExtraDhcpOptionsNetwork(Map> dhcpOptionsMap, List networkList) throws InvalidParameterValueException { if (dhcpOptionsMap != null) { for (String networkUuid : dhcpOptionsMap.keySet()) { @@ -3626,12 +3662,11 @@ private UserVm createVirtualMachine(DataCenter zone, ServiceOffering serviceOffe sshPublicKey = pair.getPublicKey(); } - List> networks = new ArrayList>(); - - LinkedHashMap networkNicMap = new LinkedHashMap(); + LinkedHashMap> networkNicMap = new LinkedHashMap<>(); short defaultNetworkNumber = 0; boolean securityGroupEnabled = false; + int networkIndex = 0; for (NetworkVO network : networkList) { if ((network.getDataCenterId() != zone.getId())) { if (!network.isStrechedL2Network()) { @@ -3669,7 +3704,7 @@ private UserVm createVirtualMachine(DataCenter zone, ServiceOffering serviceOffe } NicProfile profile = new NicProfile(requestedIpPair.getIp4Address(), requestedIpPair.getIp6Address(), requestedIpPair.getMacAddress()); - + profile.setOrderIndex(networkIndex); if (defaultNetworkNumber == 0) { defaultNetworkNumber++; // if user requested specific ip for default network, add it @@ -3696,13 +3731,16 @@ private UserVm createVirtualMachine(DataCenter zone, ServiceOffering serviceOffe } } - networks.add(new Pair(network, profile)); - if (_networkModel.isSecurityGroupSupportedInNetwork(network)) { securityGroupEnabled = true; } - - networkNicMap.put(network.getUuid(), profile); + List profiles = networkNicMap.get(network.getUuid()); + if (CollectionUtils.isEmpty(profiles)) { + profiles = new ArrayList<>(); + } + profiles.add(profile); + networkNicMap.put(network.getUuid(), profiles); + networkIndex++; } if (securityGroupIdList != null && !securityGroupIdList.isEmpty() && !securityGroupEnabled) { @@ -3833,7 +3871,7 @@ private String generateHostName(String uuidName) { private UserVmVO commitUserVm(final boolean isImport, final DataCenter zone, final Host host, final Host lastHost, final VirtualMachineTemplate template, final String hostName, final String displayName, final Account owner, final Long diskOfferingId, final Long diskSize, final String userData, final Account caller, final Boolean isDisplayVm, final String keyboard, - final long accountId, final long userId, final ServiceOffering offering, final boolean isIso, final String sshPublicKey, final LinkedHashMap networkNicMap, + final long accountId, final long userId, final ServiceOffering offering, final boolean isIso, final String sshPublicKey, final LinkedHashMap> networkNicMap, final long id, final String instanceName, final String uuidName, final HypervisorType hypervisorType, final Map customParameters, final Map> extraDhcpOptionMap, final Map dataDiskTemplateToDiskOfferingMap, final Map userVmOVFPropertiesMap, final VirtualMachine.PowerState powerState) throws InsufficientCapacityException { @@ -3940,27 +3978,7 @@ public UserVmVO doInTransaction(TransactionStatus status) throws InsufficientCap } vm.setDetail(VmDetailConstants.DEPLOY_VM, "true"); - if (MapUtils.isNotEmpty(userVmOVFPropertiesMap)) { - for (String key : userVmOVFPropertiesMap.keySet()) { - String detailKey = ApiConstants.OVF_PROPERTIES + "-" + key; - String value = userVmOVFPropertiesMap.get(key); - - // Sanitize boolean values to expected format and encrypt passwords - if (StringUtils.isNotBlank(value)) { - if (value.equalsIgnoreCase("True")) { - value = "True"; - } else if (value.equalsIgnoreCase("False")) { - value = "False"; - } else { - TemplateOVFPropertyVO ovfPropertyVO = templateOVFPropertiesDao.findByTemplateAndKey(vm.getTemplateId(), key); - if (ovfPropertyVO.isPassword()) { - value = DBEncryptionUtil.encrypt(value); - } - } - } - vm.setDetail(detailKey, value); - } - } + persistVMDeployAsIsProperties(vm, userVmOVFPropertiesMap); _vmDao.saveDetails(vm); if (!isImport) { @@ -4005,9 +4023,42 @@ public UserVmVO doInTransaction(TransactionStatus status) throws InsufficientCap }); } + /** + * take the properties and set them on the vm. + * consider should we be complete, and make sure all default values are copied as well if known? + * I.E. iterate over the template details as well to copy any that are not defined yet. + */ + private void persistVMDeployAsIsProperties(UserVmVO vm, Map userVmOVFPropertiesMap) { + if (MapUtils.isNotEmpty(userVmOVFPropertiesMap)) { + for (String key : userVmOVFPropertiesMap.keySet()) { + String detailKey = key; + String value = userVmOVFPropertiesMap.get(key); + + // Sanitize boolean values to expected format and encrypt passwords + if (StringUtils.isNotBlank(value)) { + if (value.equalsIgnoreCase("True")) { + value = "True"; + } else if (value.equalsIgnoreCase("False")) { + value = "False"; + } else { + OVFPropertyTO propertyTO = templateDeployAsIsDetailsDao.findPropertyByTemplateAndKey(vm.getTemplateId(), key); + if (propertyTO != null && propertyTO.isPassword()) { + value = DBEncryptionUtil.encrypt(value); + } + } + } + if (s_logger.isTraceEnabled()) { + s_logger.trace(String.format("setting property '%s' as '%s' with value '%s'", key, detailKey, value)); + } + UserVmDeployAsIsDetailVO detail = new UserVmDeployAsIsDetailVO(vm.getId(), detailKey, value); + userVmDeployAsIsDetailsDao.persist(detail); + } + } + } + private UserVmVO commitUserVm(final DataCenter zone, final VirtualMachineTemplate template, final String hostName, final String displayName, final Account owner, final Long diskOfferingId, final Long diskSize, final String userData, final Account caller, final Boolean isDisplayVm, final String keyboard, - final long accountId, final long userId, final ServiceOfferingVO offering, final boolean isIso, final String sshPublicKey, final LinkedHashMap networkNicMap, + final long accountId, final long userId, final ServiceOfferingVO offering, final boolean isIso, final String sshPublicKey, final LinkedHashMap> networkNicMap, final long id, final String instanceName, final String uuidName, final HypervisorType hypervisorType, final Map customParameters, final Map> extraDhcpOptionMap, final Map dataDiskTemplateToDiskOfferingMap, Map userVmOVFPropertiesMap) throws InsufficientCapacityException { @@ -4336,7 +4387,7 @@ public boolean finalizeVirtualMachineProfile(VirtualMachineProfile profile, Depl if (_networkModel.isSharedNetworkWithoutServices(network.getId())) { final String serviceOffering = _serviceOfferingDao.findByIdIncludingRemoved(vm.getId(), vm.getServiceOfferingId()).getDisplayText(); boolean isWindows = _guestOSCategoryDao.findById(_guestOSDao.findById(vm.getGuestOSId()).getCategoryId()).getName().equalsIgnoreCase("Windows"); - String destHostname = VirtualMachineManager.getHypervisorHostname(dest.getHost().getName()); + String destHostname = VirtualMachineManager.getHypervisorHostname(dest.getHost() != null ? dest.getHost().getName() : ""); List vmData = _networkModel.generateVmData(vm.getUserData(), serviceOffering, vm.getDataCenterId(), vm.getInstanceName(), vm.getHostName(), vm.getId(), vm.getUuid(), defaultNic.getIPv4Address(), vm.getDetail(VmDetailConstants.SSH_PUBLIC_KEY), (String) profile.getParameter(VirtualMachineProfile.Param.VmPassword), isWindows, destHostname); String vmName = vm.getInstanceName(); @@ -5123,6 +5174,12 @@ public UserVm createVirtualMachine(DeployVMCmd cmd) throws InsufficientCapacityE } } + List networkIds = cmd.getNetworkIds(); + LinkedHashMap userVmNetworkMap = getVmOvfNetworkMapping(zone, owner, template, cmd.getVmNetworkMap()); + if (MapUtils.isNotEmpty(userVmNetworkMap)) { + networkIds = new ArrayList<>(userVmNetworkMap.values()); + } + String ipAddress = cmd.getIpAddress(); String ip6Address = cmd.getIp6Address(); String macAddress = cmd.getMacAddress(); @@ -5137,9 +5194,9 @@ public UserVm createVirtualMachine(DeployVMCmd cmd) throws InsufficientCapacityE Boolean displayVm = cmd.isDisplayVm(); String keyboard = cmd.getKeyboard(); Map dataDiskTemplateToDiskOfferingMap = cmd.getDataDiskTemplateToDiskOfferingMap(); - Map userVmOVFProperties = cmd.getVmOVFProperties(); + Map userVmOVFProperties = cmd.getVmProperties(); if (zone.getNetworkType() == NetworkType.Basic) { - if (cmd.getNetworkIds() != null) { + if (networkIds != null) { throw new InvalidParameterValueException("Can't specify network Ids in Basic zone"); } else { vm = createBasicSecurityGroupVirtualMachine(zone, serviceOffering, template, getSecurityGroupIdList(cmd), owner, name, displayName, diskOfferingId, @@ -5149,7 +5206,7 @@ public UserVm createVirtualMachine(DeployVMCmd cmd) throws InsufficientCapacityE } } else { if (zone.isSecurityGroupEnabled()) { - vm = createAdvancedSecurityGroupVirtualMachine(zone, serviceOffering, template, cmd.getNetworkIds(), getSecurityGroupIdList(cmd), owner, name, + vm = createAdvancedSecurityGroupVirtualMachine(zone, serviceOffering, template, networkIds, getSecurityGroupIdList(cmd), owner, name, displayName, diskOfferingId, size, group, cmd.getHypervisor(), cmd.getHttpMethod(), userData, sshKeyPairName, cmd.getIpToNetworkMap(), addrs, displayVm, keyboard, cmd.getAffinityGroupIdList(), cmd.getDetails(), cmd.getCustomId(), cmd.getDhcpOptionsMap(), dataDiskTemplateToDiskOfferingMap, userVmOVFProperties); @@ -5158,7 +5215,7 @@ public UserVm createVirtualMachine(DeployVMCmd cmd) throws InsufficientCapacityE if (cmd.getSecurityGroupIdList() != null && !cmd.getSecurityGroupIdList().isEmpty()) { throw new InvalidParameterValueException("Can't create vm with security groups; security group feature is not enabled per zone"); } - vm = createAdvancedVirtualMachine(zone, serviceOffering, template, cmd.getNetworkIds(), owner, name, displayName, diskOfferingId, size, group, + vm = createAdvancedVirtualMachine(zone, serviceOffering, template, networkIds, owner, name, displayName, diskOfferingId, size, group, cmd.getHypervisor(), cmd.getHttpMethod(), userData, sshKeyPairName, cmd.getIpToNetworkMap(), addrs, displayVm, keyboard, cmd.getAffinityGroupIdList(), cmd.getDetails(), cmd.getCustomId(), cmd.getDhcpOptionsMap(), dataDiskTemplateToDiskOfferingMap, userVmOVFProperties); } @@ -5536,6 +5593,12 @@ public VirtualMachine vmStorageMigration(Long vmId, StoragePool destPool) { } checkDestinationHypervisorType(destPool, vm); + if (destPool.getPoolType() == Storage.StoragePoolType.DatastoreCluster) { + DataCenter dc = _entityMgr.findById(DataCenter.class, vm.getDataCenterId()); + Pod destPoolPod = _entityMgr.findById(Pod.class, destPool.getPodId()); + + destPool = volumeMgr.findChildDataStoreInDataStoreCluster(dc, destPoolPod, destPool.getClusterId(), null, null, destPool.getId()); + } _itMgr.storageMigration(vm.getUuid(), destPool); return _vmDao.findById(vm.getId()); @@ -7261,4 +7324,42 @@ private void checkUnmanagingVMVolumes(UserVmVO vm, List volumes) { } } } -} \ No newline at end of file + + private LinkedHashMap getVmOvfNetworkMapping(DataCenter zone, Account owner, VirtualMachineTemplate template, Map vmNetworkMapping) throws InsufficientCapacityException, ResourceAllocationException { + LinkedHashMap mapping = new LinkedHashMap<>(); + if (ImageFormat.OVA.equals(template.getFormat())) { + List OVFNetworkTOList = + templateDeployAsIsDetailsDao.listNetworkRequirementsByTemplateId(template.getId()); + if (CollectionUtils.isNotEmpty(OVFNetworkTOList)) { + Network lastMappedNetwork = null; + for (OVFNetworkTO OVFNetworkTO : OVFNetworkTOList) { + Long networkId = vmNetworkMapping.get(OVFNetworkTO.getInstanceID()); + if (networkId == null && lastMappedNetwork == null) { + lastMappedNetwork = getNetworkForOvfNetworkMapping(zone, owner); + } + if (networkId == null) { + networkId = lastMappedNetwork.getId(); + } + mapping.put(OVFNetworkTO.getInstanceID(), networkId); + } + } + } + return mapping; + } + + private Network getNetworkForOvfNetworkMapping(DataCenter zone, Account owner) throws InsufficientCapacityException, ResourceAllocationException { + Network network = null; + if (zone.isSecurityGroupEnabled()) { + network = _networkModel.getNetworkWithSGWithFreeIPs(zone.getId()); + if (network == null) { + throw new InvalidParameterValueException("No network with security enabled is found in zone ID: " + zone.getUuid()); + } + } else { + network = getDefaultNetwork(zone, owner, true); + if (network == null) { + throw new InvalidParameterValueException(String.format("Default network not found for zone ID: %s and account ID: %s", zone.getUuid(), owner.getUuid())); + } + } + return network; + } +} diff --git a/server/src/main/java/org/apache/cloudstack/diagnostics/DiagnosticsServiceImpl.java b/server/src/main/java/org/apache/cloudstack/diagnostics/DiagnosticsServiceImpl.java index 49ad2159698f..0184a44ff390 100644 --- a/server/src/main/java/org/apache/cloudstack/diagnostics/DiagnosticsServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/diagnostics/DiagnosticsServiceImpl.java @@ -372,7 +372,7 @@ private VMInstanceVO getSecondaryStorageVmInZone(Long zoneId) { * @return a valid secondary storage with less than DiskQuotaPercentageThreshold set by global config */ private DataStore getImageStore(Long zoneId) { - List stores = storeMgr.getImageStoresByScope(new ZoneScope(zoneId)); + List stores = storeMgr.getImageStoresByScopeExcludingReadOnly(new ZoneScope(zoneId)); if (CollectionUtils.isEmpty(stores)) { throw new CloudRuntimeException("No Secondary storage found in Zone with Id: " + zoneId); } diff --git a/server/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java b/server/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java index a05c4b9e4aa5..dcbc9656448c 100644 --- a/server/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java @@ -263,13 +263,13 @@ public void downloadTemplate(long templateId, long poolId, long hostId) { Answer answer = sendDirectDownloadCommand(cmd, template, poolId, host); - VMTemplateStoragePoolVO sPoolRef = vmTemplatePoolDao.findByPoolTemplate(poolId, templateId); + VMTemplateStoragePoolVO sPoolRef = vmTemplatePoolDao.findByPoolTemplate(poolId, templateId, null); if (sPoolRef == null) { if (s_logger.isDebugEnabled()) { s_logger.debug("Not found (templateId:" + templateId + " poolId: " + poolId + ") in template_spool_ref, persisting it"); } DirectDownloadAnswer ans = (DirectDownloadAnswer) answer; - sPoolRef = new VMTemplateStoragePoolVO(poolId, templateId); + sPoolRef = new VMTemplateStoragePoolVO(poolId, templateId, null); sPoolRef.setDownloadPercent(100); sPoolRef.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOADED); sPoolRef.setState(ObjectInDataStoreStateMachine.State.Ready); diff --git a/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java b/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java index 26f8675f2261..ecf1145dfb0e 100644 --- a/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java @@ -103,7 +103,6 @@ import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.storage.GuestOS; import com.cloud.storage.GuestOSHypervisor; -import com.cloud.storage.Storage; import com.cloud.storage.StoragePool; import com.cloud.storage.VMTemplateStoragePoolVO; import com.cloud.storage.VMTemplateVO; @@ -491,23 +490,20 @@ private StoragePool getStoragePool(final UnmanagedInstanceTO.Disk disk, final Da final String dsPath = disk.getDatastorePath(); final String dsType = disk.getDatastoreType(); final String dsName = disk.getDatastoreName(); - if (dsType.equals("VMFS")) { - List pools = primaryDataStoreDao.listPoolsByCluster(cluster.getId()); - pools.addAll(primaryDataStoreDao.listByDataCenterId(zone.getId())); + if (dsType != null) { + List pools = primaryDataStoreDao.listPoolByHostPath(dsHost, dsPath); for (StoragePool pool : pools) { - if (pool.getPoolType() != Storage.StoragePoolType.VMFS) { - continue; - } - if (pool.getPath().endsWith(dsName)) { + if (pool.getDataCenterId() == zone.getId() && + (pool.getClusterId() == null || pool.getClusterId().equals(cluster.getId()))) { storagePool = pool; break; } } } else { - List pools = primaryDataStoreDao.listPoolByHostPath(dsHost, dsPath); + List pools = primaryDataStoreDao.listPoolsByCluster(cluster.getId()); + pools.addAll(primaryDataStoreDao.listByDataCenterId(zone.getId())); for (StoragePool pool : pools) { - if (pool.getDataCenterId() == zone.getId() && - (pool.getClusterId() == null || pool.getClusterId().equals(cluster.getId()))) { + if (pool.getPath().endsWith(dsName)) { storagePool = pool; break; } diff --git a/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml b/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml index 91522f3a5009..672f7c0d4144 100644 --- a/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml +++ b/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml @@ -262,6 +262,8 @@ + + ()); Mockito.when(_volumeDao.findByDc(anyLong())).thenReturn(new ArrayList()); Mockito.when(_physicalNetworkDao.listByZone(anyLong())).thenReturn(new ArrayList()); - Mockito.when(_imageStoreDao.findByScope(any(ZoneScope.class))).thenReturn(new ArrayList()); + Mockito.when(_imageStoreDao.findByZone(any(ZoneScope.class), anyBoolean())).thenReturn(new ArrayList()); configurationMgr.checkIfZoneIsDeletable(new Random().nextLong()); } @@ -712,7 +712,7 @@ public void checkIfZoneIsDeletableFailureOnHostTest() { Mockito.when(_vmInstanceDao.listByZoneId(anyLong())).thenReturn(new ArrayList()); Mockito.when(_volumeDao.findByDc(anyLong())).thenReturn(new ArrayList()); Mockito.when(_physicalNetworkDao.listByZone(anyLong())).thenReturn(new ArrayList()); - Mockito.when(_imageStoreDao.findByScope(any(ZoneScope.class))).thenReturn(new ArrayList()); + Mockito.when(_imageStoreDao.findByZone(any(ZoneScope.class), anyBoolean())).thenReturn(new ArrayList()); configurationMgr.checkIfZoneIsDeletable(new Random().nextLong()); } @@ -730,7 +730,7 @@ public void checkIfZoneIsDeletableFailureOnPodTest() { Mockito.when(_vmInstanceDao.listByZoneId(anyLong())).thenReturn(new ArrayList()); Mockito.when(_volumeDao.findByDc(anyLong())).thenReturn(new ArrayList()); Mockito.when(_physicalNetworkDao.listByZone(anyLong())).thenReturn(new ArrayList()); - Mockito.when(_imageStoreDao.findByScope(any(ZoneScope.class))).thenReturn(new ArrayList()); + Mockito.when(_imageStoreDao.findByZone(any(ZoneScope.class), anyBoolean())).thenReturn(new ArrayList()); configurationMgr.checkIfZoneIsDeletable(new Random().nextLong()); } @@ -744,7 +744,7 @@ public void checkIfZoneIsDeletableFailureOnPrivateIpAddressTest() { Mockito.when(_vmInstanceDao.listByZoneId(anyLong())).thenReturn(new ArrayList()); Mockito.when(_volumeDao.findByDc(anyLong())).thenReturn(new ArrayList()); Mockito.when(_physicalNetworkDao.listByZone(anyLong())).thenReturn(new ArrayList()); - Mockito.when(_imageStoreDao.findByScope(any(ZoneScope.class))).thenReturn(new ArrayList()); + Mockito.when(_imageStoreDao.findByZone(any(ZoneScope.class), anyBoolean())).thenReturn(new ArrayList()); configurationMgr.checkIfZoneIsDeletable(new Random().nextLong()); } @@ -758,7 +758,7 @@ public void checkIfZoneIsDeletableFailureOnPublicIpAddressTest() { Mockito.when(_vmInstanceDao.listByZoneId(anyLong())).thenReturn(new ArrayList()); Mockito.when(_volumeDao.findByDc(anyLong())).thenReturn(new ArrayList()); Mockito.when(_physicalNetworkDao.listByZone(anyLong())).thenReturn(new ArrayList()); - Mockito.when(_imageStoreDao.findByScope(any(ZoneScope.class))).thenReturn(new ArrayList()); + Mockito.when(_imageStoreDao.findByZone(any(ZoneScope.class), anyBoolean())).thenReturn(new ArrayList()); configurationMgr.checkIfZoneIsDeletable(new Random().nextLong()); } @@ -776,7 +776,7 @@ public void checkIfZoneIsDeletableFailureOnVmInstanceTest() { Mockito.when(_vmInstanceDao.listByZoneId(anyLong())).thenReturn(arrayList); Mockito.when(_volumeDao.findByDc(anyLong())).thenReturn(new ArrayList()); Mockito.when(_physicalNetworkDao.listByZone(anyLong())).thenReturn(new ArrayList()); - Mockito.when(_imageStoreDao.findByScope(any(ZoneScope.class))).thenReturn(new ArrayList()); + Mockito.when(_imageStoreDao.findByZone(any(ZoneScope.class), anyBoolean())).thenReturn(new ArrayList()); configurationMgr.checkIfZoneIsDeletable(new Random().nextLong()); } @@ -794,7 +794,7 @@ public void checkIfZoneIsDeletableFailureOnVolumeTest() { Mockito.when(_vmInstanceDao.listByZoneId(anyLong())).thenReturn(new ArrayList()); Mockito.when(_volumeDao.findByDc(anyLong())).thenReturn(arrayList); Mockito.when(_physicalNetworkDao.listByZone(anyLong())).thenReturn(new ArrayList()); - Mockito.when(_imageStoreDao.findByScope(any(ZoneScope.class))).thenReturn(new ArrayList()); + Mockito.when(_imageStoreDao.findByZone(any(ZoneScope.class), anyBoolean())).thenReturn(new ArrayList()); configurationMgr.checkIfZoneIsDeletable(new Random().nextLong()); } @@ -812,7 +812,7 @@ public void checkIfZoneIsDeletableFailureOnPhysicalNetworkTest() { Mockito.when(_vmInstanceDao.listByZoneId(anyLong())).thenReturn(new ArrayList()); Mockito.when(_volumeDao.findByDc(anyLong())).thenReturn(new ArrayList()); Mockito.when(_physicalNetworkDao.listByZone(anyLong())).thenReturn(arrayList); - Mockito.when(_imageStoreDao.findByScope(any(ZoneScope.class))).thenReturn(new ArrayList()); + Mockito.when(_imageStoreDao.findByZone(any(ZoneScope.class), anyBoolean())).thenReturn(new ArrayList()); configurationMgr.checkIfZoneIsDeletable(new Random().nextLong()); } diff --git a/server/src/test/java/com/cloud/template/TemplateManagerImplTest.java b/server/src/test/java/com/cloud/template/TemplateManagerImplTest.java index bd21643b4a1a..47180cda0a1e 100755 --- a/server/src/test/java/com/cloud/template/TemplateManagerImplTest.java +++ b/server/src/test/java/com/cloud/template/TemplateManagerImplTest.java @@ -119,6 +119,7 @@ import java.util.concurrent.atomic.AtomicInteger; import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.nullable; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyBoolean; import static org.mockito.Matchers.anyLong; @@ -287,7 +288,7 @@ public void testPrepareTemplateIsSeeded() { when(dataStoreManager.getPrimaryDataStore(anyLong())).thenReturn(mockPrimaryDataStore); when(vmTemplateDao.findById(anyLong(), anyBoolean())).thenReturn(mockTemplate); - when(vmTemplatePoolDao.findByPoolTemplate(anyLong(), anyLong())).thenReturn(mockTemplateStore); + when(vmTemplatePoolDao.findByPoolTemplate(anyLong(), anyLong(), nullable(String.class))).thenReturn(mockTemplateStore); doNothing().when(mockTemplateStore).setMarkedForGC(anyBoolean()); @@ -309,7 +310,7 @@ public void testPrepareTemplateNotDownloaded() { when(dataStoreManager.getPrimaryDataStore(anyLong())).thenReturn(mockPrimaryDataStore); when(vmTemplateDao.findById(anyLong(), anyBoolean())).thenReturn(mockTemplate); - when(vmTemplatePoolDao.findByPoolTemplate(anyLong(), anyLong())).thenReturn(null); + when(vmTemplatePoolDao.findByPoolTemplate(anyLong(), anyLong(), nullable(String.class))).thenReturn(null); when(templateDataStoreDao.findByTemplateZoneDownloadStatus(202l, 1l, VMTemplateStorageResourceAssoc.Status.DOWNLOADED)).thenReturn(null); VMTemplateStoragePoolVO returnObject = templateManager.prepareTemplateForCreate(mockTemplate, (StoragePool) mockPrimaryDataStore); @@ -332,7 +333,7 @@ public void testPrepareTemplateNoHostConnectedToPool() { when(dataStoreManager.getPrimaryDataStore(anyLong())).thenReturn(mockPrimaryDataStore); when(vmTemplateDao.findById(anyLong(), anyBoolean())).thenReturn(mockTemplate); - when(vmTemplatePoolDao.findByPoolTemplate(anyLong(), anyLong())).thenReturn(null); + when(vmTemplatePoolDao.findByPoolTemplate(anyLong(), anyLong(), nullable(String.class))).thenReturn(null); when(templateDataStoreDao.findByTemplateZoneDownloadStatus(202l, 1l, VMTemplateStorageResourceAssoc.Status.DOWNLOADED)).thenReturn(mockTemplateDataStore); when(storagePoolHostDao.listByHostStatus(2l, Status.Up)).thenReturn(null); @@ -361,7 +362,7 @@ public void testTemplateScheduledForDownloadInOnePool() { when(vmTemplateDao.findById(anyLong())).thenReturn(mockTemplate); when(dataStoreManager.getPrimaryDataStore(anyLong())).thenReturn(mockPrimaryDataStore); when(vmTemplateDao.findById(anyLong(), anyBoolean())).thenReturn(mockTemplate); - when(vmTemplatePoolDao.findByPoolTemplate(anyLong(), anyLong())).thenReturn(mockTemplateStore); + when(vmTemplatePoolDao.findByPoolTemplate(anyLong(), anyLong(), nullable(String.class))).thenReturn(mockTemplateStore); when(primaryDataStoreDao.findById(anyLong())).thenReturn(mockPool); doNothing().when(mockTemplateStore).setMarkedForGC(anyBoolean()); @@ -390,7 +391,7 @@ public void testTemplateScheduledForDownloadInDisabledPool() { when(vmTemplateDao.findById(anyLong())).thenReturn(mockTemplate); when(dataStoreManager.getPrimaryDataStore(anyLong())).thenReturn(mockPrimaryDataStore); when(vmTemplateDao.findById(anyLong(), anyBoolean())).thenReturn(mockTemplate); - when(vmTemplatePoolDao.findByPoolTemplate(anyLong(), anyLong())).thenReturn(mockTemplateStore); + when(vmTemplatePoolDao.findByPoolTemplate(anyLong(), anyLong(), nullable(String.class))).thenReturn(mockTemplateStore); when(primaryDataStoreDao.findById(anyLong())).thenReturn(mockPool); doNothing().when(mockTemplateStore).setMarkedForGC(anyBoolean()); @@ -432,7 +433,7 @@ public void testTemplateScheduledForDownloadInMultiplePool() { when(vmTemplateDao.findById(anyLong())).thenReturn(mockTemplate); when(dataStoreManager.getPrimaryDataStore(anyLong())).thenReturn(mockPrimaryDataStore); when(vmTemplateDao.findById(anyLong(), anyBoolean())).thenReturn(mockTemplate); - when(vmTemplatePoolDao.findByPoolTemplate(anyLong(), anyLong())).thenReturn(mockTemplateStore); + when(vmTemplatePoolDao.findByPoolTemplate(anyLong(), anyLong(), nullable(String.class))).thenReturn(mockTemplateStore); when(primaryDataStoreDao.findById(2l)).thenReturn(mockPool1); when(primaryDataStoreDao.findById(3l)).thenReturn(mockPool2); when(primaryDataStoreDao.findById(4l)).thenReturn(mockPool3); diff --git a/server/src/test/java/com/cloud/vm/DeploymentPlanningManagerImplTest.java b/server/src/test/java/com/cloud/vm/DeploymentPlanningManagerImplTest.java index e73c0c6bd7dc..d356570b6332 100644 --- a/server/src/test/java/com/cloud/vm/DeploymentPlanningManagerImplTest.java +++ b/server/src/test/java/com/cloud/vm/DeploymentPlanningManagerImplTest.java @@ -29,6 +29,8 @@ import javax.naming.ConfigurationException; import com.cloud.host.Host; +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.dao.VMTemplateDao; import org.apache.cloudstack.affinity.dao.AffinityGroupDomainMapDao; import org.junit.Before; import org.junit.BeforeClass; @@ -141,6 +143,9 @@ public class DeploymentPlanningManagerImplTest { @Inject UserVmDetailsDao vmDetailsDao; + @Inject + VMTemplateDao templateDao; + @Mock Host host; @@ -162,6 +167,10 @@ public void testSetUp() { Mockito.when(_plannerHostReserveDao.findById(Matchers.anyLong())).thenReturn(reservationVO); Mockito.when(_affinityGroupVMMapDao.countAffinityGroupsForVm(Matchers.anyLong())).thenReturn(0L); + VMTemplateVO template = Mockito.mock(VMTemplateVO.class); + Mockito.when(template.isDeployAsIs()).thenReturn(false); + Mockito.when(templateDao.findById(Mockito.anyLong())).thenReturn(template); + VMInstanceVO vm = new VMInstanceVO(); Mockito.when(vmProfile.getVirtualMachine()).thenReturn(vm); @@ -456,6 +465,11 @@ public HostGpuGroupsDao hostGpuGroupsDap() { return Mockito.mock(HostGpuGroupsDao.class); } + @Bean + public VMTemplateDao vmTemplateDao() { + return Mockito.mock(VMTemplateDao.class); + } + public static class Library implements TypeFilter { @Override diff --git a/server/src/test/resources/createNetworkOffering.xml b/server/src/test/resources/createNetworkOffering.xml index 8dee0e8a54ee..55343ef835d4 100644 --- a/server/src/test/resources/createNetworkOffering.xml +++ b/server/src/test/resources/createNetworkOffering.xml @@ -60,4 +60,5 @@ + diff --git a/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/PremiumSecondaryStorageManagerImpl.java b/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/PremiumSecondaryStorageManagerImpl.java index ecfc67eaff6d..d21ec614f405 100644 --- a/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/PremiumSecondaryStorageManagerImpl.java +++ b/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/PremiumSecondaryStorageManagerImpl.java @@ -16,6 +16,8 @@ // under the License. package org.apache.cloudstack.secondarystorage; +import java.util.ArrayList; +import java.util.Collections; import java.util.Date; import java.util.List; import java.util.Map; @@ -27,16 +29,19 @@ import com.cloud.agent.api.Command; import com.cloud.configuration.Config; +import com.cloud.host.Host; import com.cloud.host.HostVO; import com.cloud.host.Status; import com.cloud.host.dao.HostDao; import com.cloud.resource.ResourceManager; import com.cloud.secstorage.CommandExecLogDao; import com.cloud.secstorage.CommandExecLogVO; +import com.cloud.storage.StorageManager; import com.cloud.storage.secondary.SecondaryStorageVmManager; import com.cloud.utils.DateUtil; import com.cloud.utils.NumbersUtil; import com.cloud.utils.Pair; +import com.cloud.utils.db.Filter; import com.cloud.utils.db.JoinBuilder.JoinType; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; @@ -51,8 +56,13 @@ public class PremiumSecondaryStorageManagerImpl extends SecondaryStorageManagerI private static final Logger s_logger = Logger.getLogger(PremiumSecondaryStorageManagerImpl.class); private int _capacityPerSSVM = SecondaryStorageVmManager.DEFAULT_SS_VM_CAPACITY; + private int migrateCapPerSSVM = DEFAULT_MIGRATE_SS_VM_CAPACITY; private int _standbyCapacity = SecondaryStorageVmManager.DEFAULT_STANDBY_CAPACITY; private int _maxExecutionTimeMs = 1800000; + private int maxDataMigrationWaitTime = 900000; + long currentTime = DateUtil.currentGMTTime().getTime(); + long nextSpawnTime = currentTime + maxDataMigrationWaitTime; + private List migrationSSVMS = new ArrayList<>(); @Inject SecondaryStorageVmDao _secStorageVmDao; @@ -63,6 +73,7 @@ public class PremiumSecondaryStorageManagerImpl extends SecondaryStorageManagerI @Inject ResourceManager _resourceMgr; protected SearchBuilder activeCommandSearch; + protected SearchBuilder activeCopyCommandSearch; protected SearchBuilder hostSearch; @Override @@ -75,16 +86,27 @@ public boolean configure(String name, Map params) throws Configu int nMaxExecutionMinutes = NumbersUtil.parseInt(_configDao.getValue(Config.SecStorageCmdExecutionTimeMax.key()), 30); _maxExecutionTimeMs = nMaxExecutionMinutes * 60 * 1000; + migrateCapPerSSVM = StorageManager.SecStorageMaxMigrateSessions.value(); + int nMaxDataMigrationWaitTime = StorageManager.MaxDataMigrationWaitTime.value(); + maxDataMigrationWaitTime = nMaxDataMigrationWaitTime * 60 * 1000; + nextSpawnTime = currentTime + maxDataMigrationWaitTime; + hostSearch = _hostDao.createSearchBuilder(); hostSearch.and("dc", hostSearch.entity().getDataCenterId(), Op.EQ); hostSearch.and("status", hostSearch.entity().getStatus(), Op.EQ); activeCommandSearch = _cmdExecLogDao.createSearchBuilder(); activeCommandSearch.and("created", activeCommandSearch.entity().getCreated(), Op.GTEQ); - activeCommandSearch.join("hostSearch", hostSearch, activeCommandSearch.entity().getInstanceId(), hostSearch.entity().getId(), JoinType.INNER); + activeCommandSearch.join("hostSearch", hostSearch, activeCommandSearch.entity().getHostId(), hostSearch.entity().getId(), JoinType.INNER); + + activeCopyCommandSearch = _cmdExecLogDao.createSearchBuilder(); + activeCopyCommandSearch.and("created", activeCopyCommandSearch.entity().getCreated(), Op.GTEQ); + activeCopyCommandSearch.and("command_name", activeCopyCommandSearch.entity().getCommandName(), Op.EQ); + activeCopyCommandSearch.join("hostSearch", hostSearch, activeCopyCommandSearch.entity().getHostId(), hostSearch.entity().getId(), JoinType.INNER); hostSearch.done(); activeCommandSearch.done(); + activeCopyCommandSearch.done(); return true; } @@ -96,7 +118,6 @@ public Pair scanPool(Long pool) { } Date cutTime = new Date(DateUtil.currentGMTTime().getTime() - _maxExecutionTimeMs); - _cmdExecLogDao.expungeExpiredRecords(cutTime); boolean suspendAutoLoading = !reserveStandbyCapacity(); @@ -134,19 +155,54 @@ public Pair scanPool(Long pool) { return new Pair(AfterScanAction.nop, null); } - alreadyRunning = _secStorageVmDao.getSecStorageVmListInStates(null, dataCenterId, State.Running, State.Migrating, State.Starting); + alreadyRunning = _secStorageVmDao.getSecStorageVmListInStates(null, dataCenterId, State.Running, State.Migrating, State.Starting); List activeCmds = findActiveCommands(dataCenterId, cutTime); - if (alreadyRunning.size() * _capacityPerSSVM - activeCmds.size() < _standbyCapacity) { - s_logger.info("secondary storage command execution standby capactiy low (running VMs: " + alreadyRunning.size() + ", active cmds: " + activeCmds.size() + - "), starting a new one"); - return new Pair(AfterScanAction.expand, SecondaryStorageVm.Role.commandExecutor); - } + List copyCmdsInPipeline = findAllActiveCopyCommands(dataCenterId, cutTime); + return scaleSSVMOnLoad(alreadyRunning, activeCmds, copyCmdsInPipeline, dataCenterId); + } + return new Pair(AfterScanAction.nop, null); + } + + private Pair scaleSSVMOnLoad(List alreadyRunning, List activeCmds, + List copyCmdsInPipeline, long dataCenterId) { + Integer hostsCount = _hostDao.countAllByTypeInZone(dataCenterId, Host.Type.Routing); + Integer maxSsvms = (hostsCount < MaxNumberOfSsvmsForMigration.value()) ? hostsCount : MaxNumberOfSsvmsForMigration.value(); + int halfLimit = Math.round((float) (alreadyRunning.size() * migrateCapPerSSVM) / 2); + currentTime = DateUtil.currentGMTTime().getTime(); + if (alreadyRunning.size() * _capacityPerSSVM - activeCmds.size() < _standbyCapacity) { + s_logger.info("secondary storage command execution standby capactiy low (running VMs: " + alreadyRunning.size() + ", active cmds: " + activeCmds.size() + + "), starting a new one"); + return new Pair(AfterScanAction.expand, SecondaryStorageVm.Role.commandExecutor); + } + else if (!copyCmdsInPipeline.isEmpty() && copyCmdsInPipeline.size() >= halfLimit && + ((Math.abs(currentTime - copyCmdsInPipeline.get(halfLimit - 1).getCreated().getTime()) > maxDataMigrationWaitTime )) && + (currentTime > nextSpawnTime) && alreadyRunning.size() <= maxSsvms) { + nextSpawnTime = currentTime + maxDataMigrationWaitTime; + s_logger.debug("scaling SSVM to handle migration tasks"); + return new Pair(AfterScanAction.expand, SecondaryStorageVm.Role.commandExecutor); + } + scaleDownSSVMOnLoad(alreadyRunning, activeCmds, copyCmdsInPipeline); return new Pair(AfterScanAction.nop, null); } + private void scaleDownSSVMOnLoad(List alreadyRunning, List activeCmds, + List copyCmdsInPipeline) { + int halfLimit = Math.round((float) (alreadyRunning.size() * migrateCapPerSSVM) / 2); + if ((copyCmdsInPipeline.size() < halfLimit && alreadyRunning.size() * _capacityPerSSVM - activeCmds.size() > (_standbyCapacity + 5)) && alreadyRunning.size() > 1) { + Collections.reverse(alreadyRunning); + for(SecondaryStorageVmVO vm : alreadyRunning) { + long count = activeCmds.stream().filter(cmd -> cmd.getInstanceId() == vm.getId()).count(); + if (count == 0 && copyCmdsInPipeline.size() == 0 && vm.getRole() != SecondaryStorageVm.Role.templateProcessor) { + destroySecStorageVm(vm.getId()); + break; + } + } + } + } + @Override public Pair assignSecStorageVm(long zoneId, Command cmd) { @@ -159,26 +215,33 @@ public Pair assignSecStorageVm(long zoneId, Comman if (host != null && host.getStatus() == Status.Up) return new Pair(host, secStorageVm); } - return null; } private List findActiveCommands(long dcId, Date cutTime) { SearchCriteria sc = activeCommandSearch.create(); - sc.setParameters("created", cutTime); sc.setJoinParameters("hostSearch", "dc", dcId); sc.setJoinParameters("hostSearch", "status", Status.Up); - + List result = _cmdExecLogDao.search(sc, null); return _cmdExecLogDao.search(sc, null); } + private List findAllActiveCopyCommands(long dcId, Date cutTime) { + SearchCriteria sc = activeCopyCommandSearch.create(); + sc.setParameters("created", cutTime); + sc.setParameters("command_name", "DataMigrationCommand"); + sc.setJoinParameters("hostSearch", "dc", dcId); + sc.setJoinParameters("hostSearch", "status", Status.Up); + Filter filter = new Filter(CommandExecLogVO.class, "created", true, null, null); + return _cmdExecLogDao.search(sc, filter); + } + private boolean reserveStandbyCapacity() { String value = _configDao.getValue(Config.SystemVMAutoReserveCapacity.key()); if (value != null && value.equalsIgnoreCase("true")) { return true; } - return false; } } diff --git a/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java b/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java index efa43cb022af..5ee60eb7e039 100644 --- a/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java +++ b/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java @@ -266,6 +266,9 @@ public class SecondaryStorageManagerImpl extends ManagerBase implements Secondar static final ConfigKey NTPServerConfig = new ConfigKey(String.class, "ntp.server.list", "Advanced", null, "Comma separated list of NTP servers to configure in Secondary storage VM", false, ConfigKey.Scope.Global, null); + static final ConfigKey MaxNumberOfSsvmsForMigration = new ConfigKey("Advanced", Integer.class, "max.ssvm.count", "5", + "Number of additional SSVMs to handle migration of data objects concurrently", true, ConfigKey.Scope.Global); + public SecondaryStorageManagerImpl() { } @@ -720,7 +723,7 @@ public SecondaryStorageVmVO assignSecStorageVmFromStoppedPool(long dataCenterId, return null; } - private void allocCapacity(long dataCenterId, SecondaryStorageVm.Role role) { + public void allocCapacity(long dataCenterId, SecondaryStorageVm.Role role) { if (s_logger.isTraceEnabled()) { s_logger.trace("Allocate secondary storage vm standby capacity for data center : " + dataCenterId); } @@ -822,7 +825,7 @@ public boolean isZoneReady(Map zoneHostInfoMap, long dataCen return false; } - List stores = _dataStoreMgr.getImageStoresByScope(new ZoneScope(dataCenterId)); + List stores = _dataStoreMgr.getImageStoresByScopeExcludingReadOnly(new ZoneScope(dataCenterId)); if (stores.size() < 1) { s_logger.debug("No image store added in zone " + dataCenterId + ", wait until it is ready to launch secondary storage vm"); return false; @@ -1374,7 +1377,7 @@ public Pair scanPool(Long pool) { _secStorageVmDao.getSecStorageVmListInStates(SecondaryStorageVm.Role.templateProcessor, dataCenterId, State.Running, State.Migrating, State.Starting, State.Stopped, State.Stopping); int vmSize = (ssVms == null) ? 0 : ssVms.size(); - List ssStores = _dataStoreMgr.getImageStoresByScope(new ZoneScope(dataCenterId)); + List ssStores = _dataStoreMgr.getImageStoresByScopeExcludingReadOnly(new ZoneScope(dataCenterId)); int storeSize = (ssStores == null) ? 0 : ssStores.size(); if (storeSize > vmSize) { s_logger.info("No secondary storage vms found in datacenter id=" + dataCenterId + ", starting a new one"); @@ -1516,7 +1519,7 @@ public String getConfigComponentName() { @Override public ConfigKey[] getConfigKeys() { - return new ConfigKey[] {NTPServerConfig}; + return new ConfigKey[] {NTPServerConfig, MaxNumberOfSsvmsForMigration}; } } diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java index 7e25296db801..0022e966e03c 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java @@ -54,6 +54,7 @@ import javax.naming.ConfigurationException; import org.apache.cloudstack.framework.security.keystore.KeystoreManager; +import org.apache.cloudstack.storage.NfsMountManagerImpl.PathParser; import org.apache.cloudstack.storage.command.CopyCmdAnswer; import org.apache.cloudstack.storage.command.CopyCommand; import org.apache.cloudstack.storage.command.DeleteCommand; @@ -67,7 +68,6 @@ import org.apache.cloudstack.storage.configdrive.ConfigDriveBuilder; import org.apache.cloudstack.storage.template.DownloadManager; import org.apache.cloudstack.storage.template.DownloadManagerImpl; -import org.apache.cloudstack.storage.NfsMountManagerImpl.PathParser; import org.apache.cloudstack.storage.template.UploadEntity; import org.apache.cloudstack.storage.template.UploadManager; import org.apache.cloudstack.storage.template.UploadManagerImpl; @@ -190,6 +190,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S private static final String TEMPLATE_ROOT_DIR = "template/tmpl"; private static final String VOLUME_ROOT_DIR = "volumes"; private static final String POST_UPLOAD_KEY_LOCATION = "/etc/cloudstack/agent/ms-psk"; + private static final String ORIGINAL_FILE_EXTENSION = ".orig"; private static final Map updatableConfigData = Maps.newHashMap(); static { @@ -391,6 +392,7 @@ protected void copyLocalToNfs(File localFile, File isoFile, DataStoreTO destData public Answer execute(GetDatadisksCommand cmd) { DataTO srcData = cmd.getData(); + String configurationId = cmd.getConfigurationId(); TemplateObjectTO template = (TemplateObjectTO)srcData; DataStoreTO srcStore = srcData.getDataStore(); if (!(srcStore instanceof NfsTO)) { @@ -435,7 +437,7 @@ public Answer execute(GetDatadisksCommand cmd) { Script command = new Script("cp", _timeout, s_logger); command.add(ovfFilePath); - command.add(ovfFilePath + ".orig"); + command.add(ovfFilePath + ORIGINAL_FILE_EXTENSION); String result = command.execute(); if (result != null) { String msg = "Unable to rename original OVF, error msg: " + result; @@ -445,7 +447,7 @@ public Answer execute(GetDatadisksCommand cmd) { s_logger.debug("Reading OVF " + ovfFilePath + " to retrive the number of disks present in OVA"); OVFHelper ovfHelper = new OVFHelper(); - List disks = ovfHelper.getOVFVolumeInfo(ovfFilePath); + List disks = ovfHelper.getOVFVolumeInfoFromFile(ovfFilePath, configurationId); return new GetDatadisksAnswer(disks); } catch (Exception e) { String msg = "Get Datadisk Template Count failed due to " + e.getMessage(); @@ -503,7 +505,7 @@ public Answer execute(CreateDatadiskTemplateCommand cmd) { throw new Exception(msg); } command = new Script("cp", _timeout, s_logger); - command.add(ovfFilePath + ".orig"); + command.add(ovfFilePath + ORIGINAL_FILE_EXTENSION); command.add(newTmplDirAbsolute); result = command.execute(); if (result != null) { @@ -517,7 +519,7 @@ public Answer execute(CreateDatadiskTemplateCommand cmd) { // Create OVF for the disk String newOvfFilePath = newTmplDirAbsolute + File.separator + ovfFilePath.substring(ovfFilePath.lastIndexOf(File.separator) + 1); OVFHelper ovfHelper = new OVFHelper(); - ovfHelper.rewriteOVFFile(ovfFilePath + ".orig", newOvfFilePath, diskName); + ovfHelper.rewriteOVFFileForSingleDisk(ovfFilePath + ORIGINAL_FILE_EXTENSION, newOvfFilePath, diskName); postCreatePrivateTemplate(newTmplDirAbsolute, templateId, templateUniqueName, physicalSize, virtualSize); writeMetaOvaForTemplate(newTmplDirAbsolute, ovfFilePath.substring(ovfFilePath.lastIndexOf(File.separator) + 1), diskName, templateUniqueName, physicalSize); @@ -1050,6 +1052,10 @@ protected Answer execute(CopyCommand cmd) { DataStoreTO srcDataStore = srcData.getDataStore(); DataStoreTO destDataStore = destData.getDataStore(); + if (DataStoreRole.Image == srcDataStore.getRole() && DataStoreRole.Image == destDataStore.getRole()) { + return copyFromNfsToNfs(cmd); + } + if (srcData.getObjectType() == DataObjectType.SNAPSHOT && destData.getObjectType() == DataObjectType.TEMPLATE) { return createTemplateFromSnapshot(cmd); } @@ -1254,7 +1260,6 @@ protected long getVirtualSize(File file, ImageFormat format) { } protected File findFile(String path) { - File srcFile = _storage.getFile(path); if (!srcFile.exists()) { srcFile = _storage.getFile(path + ".qcow2"); @@ -1275,6 +1280,87 @@ protected File findFile(String path) { return srcFile; } + protected Answer copyFromNfsToNfs(CopyCommand cmd) { + final DataTO srcData = cmd.getSrcTO(); + final DataTO destData = cmd.getDestTO(); + DataStoreTO srcDataStore = srcData.getDataStore(); + NfsTO srcStore = (NfsTO)srcDataStore; + DataStoreTO destDataStore = destData.getDataStore(); + final NfsTO destStore = (NfsTO) destDataStore; + try { + File srcFile = new File(getDir(srcStore.getUrl(), _nfsVersion), srcData.getPath()); + File destFile = new File(getDir(destStore.getUrl(), _nfsVersion), destData.getPath()); + ImageFormat format = getTemplateFormat(srcFile.getName()); + + if (srcFile == null) { + return new CopyCmdAnswer("Can't find src file:" + srcFile); + } + if (srcData instanceof TemplateObjectTO || srcData instanceof VolumeObjectTO) { + File srcDir = null; + if (srcFile.isFile() || srcFile.getName().contains(".")) { + srcDir = new File(srcFile.getParent()); + } + File destDir = null; + if (destFile.isFile()) { + destDir = new File(destFile.getParent()); + } + try { + FileUtils.copyDirectory((srcDir == null ? srcFile : srcDir), (destDir == null? destFile : destDir)); + } catch (IOException e) { + String msg = "Failed to copy file to destination"; + s_logger.info(msg); + return new CopyCmdAnswer(msg); + } + } else { + destFile = new File(destFile, srcFile.getName()); + try { + if (srcFile.isFile()) { + FileUtils.copyFile(srcFile, destFile); + } else { + // for vmware + srcFile = new File(srcFile.getParent()); + FileUtils.copyDirectory(srcFile, destFile); + } + } catch (IOException e) { + String msg = "Failed to copy file to destination"; + s_logger.info(msg); + return new CopyCmdAnswer(msg); + } + } + + DataTO retObj = null; + if (destData.getObjectType() == DataObjectType.TEMPLATE) { + TemplateObjectTO newTemplate = new TemplateObjectTO(); + newTemplate.setPath(destData.getPath() + File.separator + srcFile.getName()); + newTemplate.setSize(getVirtualSize(srcFile, format)); + newTemplate.setPhysicalSize(srcFile.length()); + newTemplate.setFormat(format); + retObj = newTemplate; + } else if (destData.getObjectType() == DataObjectType.VOLUME) { + VolumeObjectTO newVol = new VolumeObjectTO(); + if (srcFile.isFile()) { + newVol.setPath(destData.getPath() + File.separator + srcFile.getName()); + } else { + newVol.setPath(destData.getPath()); + } + newVol.setSize(getVirtualSize(srcFile, format)); + retObj = newVol; + } else if (destData.getObjectType() == DataObjectType.SNAPSHOT) { + SnapshotObjectTO newSnapshot = new SnapshotObjectTO(); + if (srcFile.isFile()) { + newSnapshot.setPath(destData.getPath() + File.separator + destFile.getName()); + } else { + newSnapshot.setPath(destData.getPath() + File.separator + destFile.getName() + File.separator + destFile.getName()); + } + retObj = newSnapshot; + } + return new CopyCmdAnswer(retObj); + } catch (Exception e) { + s_logger.error("failed to copy file" + srcData.getPath(), e); + return new CopyCmdAnswer("failed to copy file" + srcData.getPath() + e.toString()); + } + } + protected Answer copyFromNfsToS3(CopyCommand cmd) { final DataTO srcData = cmd.getSrcTO(); final DataTO destData = cmd.getDestTO(); @@ -2433,6 +2519,18 @@ protected Answer deleteVolume(final DeleteCommand cmd) { } + private String getDir(String secUrl, String nfsVersion) { + try { + URI uri = new URI(secUrl); + String dir = mountUri(uri, nfsVersion); + return _parent + "/" + dir; + } catch (Exception e) { + String msg = "GetRootDir for " + secUrl + " failed due to " + e.toString(); + s_logger.error(msg, e); + throw new CloudRuntimeException(msg); + } + } + @Override synchronized public String getRootDir(String secUrl, String nfsVersion) { if (!_inSystemVM) { diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java index c31bc9b38c6e..d60d7b8fb202 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java @@ -37,7 +37,10 @@ import javax.naming.ConfigurationException; -import com.cloud.agent.api.storage.OVFPropertyTO; +import com.cloud.agent.api.to.deployasis.OVFEulaSectionTO; +import com.cloud.agent.api.to.deployasis.OVFVirtualHardwareSectionTO; +import com.cloud.agent.api.to.DatadiskTO; +import com.cloud.agent.api.to.deployasis.OVFPropertyTO; import com.cloud.storage.template.Processor; import com.cloud.storage.template.S3TemplateDownloader; import com.cloud.storage.template.TemplateDownloader; @@ -55,6 +58,7 @@ import com.cloud.storage.template.TARProcessor; import com.cloud.storage.template.VhdProcessor; import com.cloud.storage.template.TemplateConstants; +import com.cloud.agent.api.to.deployasis.OVFNetworkTO; import org.apache.cloudstack.storage.command.DownloadCommand; import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType; import org.apache.cloudstack.storage.command.DownloadProgressCommand; @@ -128,6 +132,10 @@ private static class DownloadJob { private final long id; private final ResourceType resourceType; private List ovfProperties; + private List networks; + private List disks; + private OVFVirtualHardwareSectionTO hardwareSection; + private List eulaSections; public DownloadJob(TemplateDownloader td, String jobId, long id, String tmpltName, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum, String installPathPrefix, ResourceType resourceType) { @@ -230,6 +238,38 @@ public List getOvfProperties() { public void setOvfProperties(List ovfProperties) { this.ovfProperties = ovfProperties; } + + public List getNetworks() { + return networks; + } + + public void setNetworks(List networks) { + this.networks = networks; + } + + public List getDisks() { + return disks; + } + + public void setDisks(List disks) { + this.disks = disks; + } + + public void setVirtualHardwareSection(OVFVirtualHardwareSectionTO section) { + this.hardwareSection = section; + } + + public OVFVirtualHardwareSectionTO getVirtualHardwareSection() { + return this.hardwareSection; + } + + public List getEulaSections() { + return eulaSections; + } + + public void setEulaSections(List eulaSections) { + this.eulaSections = eulaSections; + } } public static final Logger LOGGER = Logger.getLogger(DownloadManagerImpl.class); @@ -509,7 +549,7 @@ private String postProcessAfterDownloadComplete(DownloadJob dnld, String resourc while (en.hasNext()) { Processor processor = en.next(); - FormatInfo info = null; + FormatInfo info; try { info = processor.process(resourcePath, null, templateName, this._processTimeout); } catch (InternalErrorException e) { @@ -526,6 +566,16 @@ private String postProcessAfterDownloadComplete(DownloadJob dnld, String resourc if (CollectionUtils.isNotEmpty(info.ovfProperties)) { dnld.setOvfProperties(info.ovfProperties); } + if (CollectionUtils.isNotEmpty(info.networks)) { + dnld.setNetworks(info.networks); + } + if (CollectionUtils.isNotEmpty(info.disks)) { + dnld.setDisks(info.disks); + } + dnld.setVirtualHardwareSection(info.hardwareSection); + if (CollectionUtils.isNotEmpty(info.eulaSections)) { + dnld.setEulaSections(info.eulaSections); + } break; } } @@ -829,6 +879,16 @@ private DownloadAnswer handleDownloadProgressCmd(SecondaryStorageResource resour if (CollectionUtils.isNotEmpty(dj.getOvfProperties())) { answer.setOvfProperties(dj.getOvfProperties()); } + if (CollectionUtils.isNotEmpty(dj.getNetworks())) { + answer.setNetworkRequirements(dj.getNetworks()); + } + if (CollectionUtils.isNotEmpty(dj.getDisks())) { + answer.setDisks(dj.getDisks()); + } + answer.setOvfHardwareSection(dj.getVirtualHardwareSection()); + if (CollectionUtils.isNotEmpty(dj.getEulaSections())) { + answer.setEulaSections(dj.getEulaSections()); + } jobs.remove(jobId); return answer; default: diff --git a/test/integration/smoke/test_secondary_storage.py b/test/integration/smoke/test_secondary_storage.py index b80b3e6813db..baa0f98935ed 100644 --- a/test/integration/smoke/test_secondary_storage.py +++ b/test/integration/smoke/test_secondary_storage.py @@ -24,6 +24,8 @@ from marvin.lib.base import * from marvin.lib.common import * from nose.plugins.attrib import attr +from marvin.cloudstackAPI import (listImageStores) +from marvin.cloudstackAPI import (updateImageStore) #Import System modules import time @@ -224,3 +226,174 @@ def test_02_sys_template_ready(self): True, "Builtin template is not ready %s in zone %s"%(template.status, zid) ) + + @attr(tags = ["advanced", "advancedns", "smoke", "basic", "eip", "sg"], required_hardware="false") + def test_03_check_read_only_flag(self): + """Test the secondary storage read-only flag + """ + + # Validate the following + # It is possible to enable/disable the read-only flag on a secondary storage and filter by it + # 1. Make the first secondary storage as read-only and verify its state has been changed + # 2. Search for the read-only storages and make sure ours is in the list + # 3. Make it again read/write and verify it has been set properly + + first_storage = self.list_secondary_storages(self.apiclient)[0] + first_storage_id = first_storage['id'] + # Step 1 + self.update_secondary_storage(self.apiclient, first_storage_id, True) + updated_storage = self.list_secondary_storages(self.apiclient, first_storage_id)[0] + self.assertEqual( + updated_storage['readonly'], + True, + "Check if the secondary storage status has been set to read-only" + ) + + # Step 2 + readonly_storages = self.list_secondary_storages(self.apiclient, readonly=True) + self.assertEqual( + isinstance(readonly_storages, list), + True, + "Check list response returns a valid list" + ) + result = any(d['id'] == first_storage_id for d in readonly_storages) + self.assertEqual( + result, + True, + "Check if we are able to list storages by their read-only status" + ) + + # Step 3 + self.update_secondary_storage(self.apiclient, first_storage_id, False) + updated_storage = self.list_secondary_storages(self.apiclient, first_storage_id)[0] + self.assertEqual( + updated_storage['readonly'], + False, + "Check if the secondary storage status has been set back to read-write" + ) + + @attr(tags = ["advanced", "advancedns", "smoke", "basic", "eip", "sg"], required_hardware="false") + def test_04_migrate_to_read_only_storage(self): + """Test migrations to a read-only secondary storage + """ + + # Validate the following + # It is not possible to migrate a storage to a read-only one + # NOTE: This test requires more than one secondary storage in the system + # 1. Make the first storage read-only + # 2. Try complete migration from the second to the first storage - it should fail + # 3. Try balanced migration from the second to the first storage - it should fail + # 4. Make the first storage read-write again + + storages = self.list_secondary_storages(self.apiclient) + if (len(storages)) < 2: + self.skipTest( + "This test requires more than one secondary storage") + + first_storage = self.list_secondary_storages(self.apiclient)[0] + first_storage_id = first_storage['id'] + second_storage = self.list_secondary_storages(self.apiclient)[1] + second_storage_id = second_storage['id'] + + # Set the first storage to read-only + self.update_secondary_storage(self.apiclient, first_storage_id, True) + + # Try complete migration from second to the first storage + + + success = False + try: + self.migrate_secondary_storage(self.apiclient, second_storage_id, first_storage_id, "complete") + except Exception as ex: + if re.search("No destination valid store\(s\) available to migrate.", str(ex)): + success = True + else: + self.debug("Secondary storage complete migration to a read-only one\ + did not fail appropriately. Error was actually : " + str(ex)); + + self.assertEqual(success, True, "Check if a complete migration to a read-only storage one fails appropriately") + + # Try balanced migration from second to the first storage + success = False + try: + self.migrate_secondary_storage(self.apiclient, second_storage_id, first_storage_id, "balance") + except Exception as ex: + if re.search("No destination valid store\(s\) available to migrate.", str(ex)): + success = True + else: + self.debug("Secondary storage balanced migration to a read-only one\ + did not fail appropriately. Error was actually : " + str(ex)) + + self.assertEqual(success, True, "Check if a balanced migration to a read-only storage one fails appropriately") + + # Set the first storage back to read-write + self.update_secondary_storage(self.apiclient, first_storage_id, False) + + @attr(tags = ["advanced", "advancedns", "smoke", "basic", "eip", "sg"], required_hardware="false") + def test_05_migrate_to_less_free_space(self): + """Test migrations when the destination storage has less space + """ + + # Validate the following + # Migration to a secondary storage with less space should be refused + # NOTE: This test requires more than one secondary storage in the system + # 1. Try complete migration from a storage with more (or equal) free space - migration should be refused + + storages = self.list_secondary_storages(self.apiclient) + if (len(storages)) < 2: + self.skipTest( + "This test requires more than one secondary storage") + + first_storage = self.list_secondary_storages(self.apiclient)[0] + first_storage_disksizeused = first_storage['disksizeused'] + first_storage_disksizetotal = first_storage['disksizetotal'] + second_storage = self.list_secondary_storages(self.apiclient)[1] + second_storage_disksizeused = second_storage['disksizeused'] + second_storage_disksizetotal = second_storage['disksizetotal'] + + first_storage_freespace = first_storage_disksizetotal - first_storage_disksizeused + second_storage_freespace = second_storage_disksizetotal - second_storage_disksizeused + + if first_storage_freespace == second_storage_freespace: + self.skipTest( + "This test requires two secondary storages with different free space") + + # Setting the storage with more free space as source storage + if first_storage_freespace > second_storage_freespace: + src_storage = first_storage['id'] + dst_storage = second_storage['id'] + else: + src_storage = second_storage['id'] + dst_storage = first_storage['id'] + + response = self.migrate_secondary_storage(self.apiclient, src_storage, dst_storage, "complete") + + success = False + if re.search("has equal or more free space than destination", str(response)): + success = True + else: + self.debug("Secondary storage complete migration to a storage \ + with less space was not refused. Here is the command output : " + str(response)) + + self.assertEqual(success, True, "Secondary storage complete migration to a storage\ + with less space was properly refused.") + + def list_secondary_storages(self, apiclient, id=None, readonly=None): + cmd = listImageStores.listImageStoresCmd() + cmd.id = id + cmd.readonly = readonly + return apiclient.listImageStores(cmd) + + def update_secondary_storage(self, apiclient, id, readonly): + cmd = updateImageStore.updateImageStoreCmd() + cmd.id = id + cmd.readonly = readonly + apiclient.updateImageStore(cmd) + + def migrate_secondary_storage(self, apiclient, first_id, second_id, type): + cmd = migrateSecondaryStorageData.migrateSecondaryStorageDataCmd() + cmd.srcpool = first_id + cmd.destpools = second_id + cmd.migrationtype = type + response = apiclient.migrateSecondaryStorageData(cmd) + return response diff --git a/test/integration/smoke/test_templates.py b/test/integration/smoke/test_templates.py index 9e9dd9fd3d60..ae34f7628f4b 100644 --- a/test/integration/smoke/test_templates.py +++ b/test/integration/smoke/test_templates.py @@ -961,6 +961,40 @@ def test_08_list_system_templates(self): ) return + @attr(tags = ["advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") + def test_09_list_templates_download_details(self): + """Test if list templates returns download details""" + + # Validate the following + # 1. ListTemplates API has been extended to support viewing the download details - progress, download states and datastore + + list_template_response = Template.list( + self.apiclient, + templatefilter='all', + account=self.user.name, + domainid=self.user.domainid + ) + self.assertEqual( + isinstance(list_template_response, list), + True, + "Check list response returns a valid list" + ) + + self.assertNotEqual( + len(list_template_response), + 0, + "Check template available in List Templates" + ) + + for template in list_template_response: + self.assertNotEqual( + len(template.downloaddetails), + 0, + "Not all templates have download details" + ) + + return + class TestCopyAndDeleteTemplatesAcrossZones(cloudstackTestCase): @classmethod diff --git a/test/integration/smoke/test_vm_life_cycle.py b/test/integration/smoke/test_vm_life_cycle.py index 68ec359f1a23..08668e47b727 100644 --- a/test/integration/smoke/test_vm_life_cycle.py +++ b/test/integration/smoke/test_vm_life_cycle.py @@ -27,7 +27,9 @@ migrateVirtualMachine, migrateVirtualMachineWithVolume, unmanageVirtualMachine, - listUnmanagedInstances) + listUnmanagedInstances, + listNics, + listVolumes) from marvin.lib.utils import * from marvin.lib.base import (Account, @@ -45,13 +47,17 @@ from marvin.lib.common import (get_domain, get_zone, get_suitable_test_template, + get_test_ovf_templates, list_hosts, - list_virtual_machines) + list_virtual_machines, + get_vm_vapp_configs) from marvin.codes import FAILED, PASS from nose.plugins.attrib import attr from marvin.lib.decoratorGenerators import skipTestIf # Import System modules import time +import json +from operator import itemgetter _multiprocess_shared_ = True @@ -1679,3 +1685,273 @@ def test_01_unmanage_vm_cycle(self): "PowerOn", "Unmanaged VM is still running" ) + + +class TestVAppsVM(cloudstackTestCase): + + @classmethod + def setUpClass(cls): + testClient = super(TestVAppsVM, cls).getClsTestClient() + cls.apiclient = testClient.getApiClient() + cls.services = testClient.getParsedTestDataConfig() + cls.hypervisor = testClient.getHypervisorInfo() + cls._cleanup = [] + + # Get Zone, Domain and templates + cls.domain = get_domain(cls.apiclient) + cls.zone = get_zone(cls.apiclient, cls.testClient.getZoneForTests()) + cls.services['mode'] = cls.zone.networktype + + cls.hypervisorNotSupported = cls.hypervisor.lower() != "vmware" + + if cls.hypervisorNotSupported == False: + + cls.account = Account.create( + cls.apiclient, + cls.services["account"], + domainid=cls.domain.id + ) + + cls.templates = get_test_ovf_templates( + cls.apiclient, + cls.zone.id, + cls.services['test_ovf_templates'], + cls.hypervisor + ) + if len(cls.templates) == 0: + assert False, "get_test_ovf_templates() failed to return templates" + + cls.custom_offering = ServiceOffering.create( + cls.apiclient, + cls.services["custom_service_offering"] + ) + + cls.isolated_network_offering = NetworkOffering.create( + cls.apiclient, + cls.services["isolated_network_offering"], + ) + cls.isolated_network_offering.update(cls.apiclient, state='Enabled') + + cls.l2_network_offering = NetworkOffering.create( + cls.apiclient, + cls.services["l2-network_offering"], + ) + cls.l2_network_offering.update(cls.apiclient, state='Enabled') + + cls._cleanup = [ + cls.account, + cls.custom_offering, + cls.isolated_network_offering, + cls.l2_network_offering + ] + + # Uncomment when tests are finished, to cleanup the test templates + for template in cls.templates: + cls._cleanup.append(template) + + @classmethod + def tearDownClass(cls): + try: + cleanup_resources(cls.apiclient, cls._cleanup) + except Exception as e: + raise Exception("Warning: Exception during class cleanup : %s" % e) + + def setUp(self): + self.apiclient = self.testClient.getApiClient() + self.cleanup = [] + + def tearDown(self): + try: + cleanup_resources(self.apiclient, self.cleanup) + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + + def get_ova_parsed_information_from_template(self, template): + if not template: + return None + details = template.details.__dict__ + configurations = [] + disks = [] + isos = [] + networks = [] + for propKey in details: + if propKey.startswith('ACS-configuration'): + configurations.append(json.loads(details[propKey])) + elif propKey.startswith('ACS-disk'): + detail = json.loads(details[propKey]) + if detail['isIso'] == True: + isos.append(detail) + else: + disks.append(detail) + elif propKey.startswith('ACS-network'): + networks.append(json.loads(details[propKey])) + + return configurations, disks, isos, networks + + def verify_nics(self, nic_networks, vm_id): + cmd = listNics.listNicsCmd() + cmd.virtualmachineid = vm_id + vm_nics = self.apiclient.listNics(cmd) + self.assertEqual( + isinstance(vm_nics, list), + True, + "Check listNics response returns a valid list" + ) + self.assertEqual( + len(nic_networks), + len(vm_nics), + msg="VM NIC count is different, expected = {}, result = {}".format(len(nic_networks), len(vm_nics)) + ) + nic_networks.sort(key=itemgetter('nic')) # CS will create NIC in order of InstanceID. Check network order + vm_nics.sort(key=itemgetter('deviceid')) + for i in range(len(vm_nics)): + nic = vm_nics[i] + nic_network = nic_networks[i] + self.assertEqual( + nic.networkid, + nic_network["network"], + msg="VM NIC(InstanceID: {}) network mismatch, expected = {}, result = {}".format(nic_network["nic"], nic_network["network"], nic.networkid) + ) + + def verify_disks(self, template_disks, vm_id): + cmd = listVolumes.listVolumesCmd() + cmd.virtualmachineid = vm_id + cmd.listall = True + vm_volumes = self.apiclient.listVolumes(cmd) + self.assertEqual( + isinstance(vm_volumes, list), + True, + "Check listVolumes response returns a valid list" + ) + self.assertEqual( + len(template_disks), + len(vm_volumes), + msg="VM volumes count is different, expected = {}, result = {}".format(len(template_disks), len(vm_volumes)) + ) + template_disks.sort(key=itemgetter('diskNumber')) + vm_volumes.sort(key=itemgetter('deviceid')) + for j in range(len(vm_volumes)): + volume = vm_volumes[j] + disk = template_disks[j] + self.assertEqual( + volume.size, + disk["virtualSize"], + msg="VM Volume(diskNumber: {}) network mismatch, expected = {}, result = {}".format(disk["diskNumber"], disk["virtualSize"], volume.size) + ) + + @attr(tags=["advanced", "advancedns", "smoke", "sg", "dev"], required_hardware="false") + @skipTestIf("hypervisorNotSupported") + def test_01_vapps_vm_cycle(self): + """ + Test the following for all found ovf templates: + 1. Deploy VM + 2. Verify VM has correct properties + 3. Verify VM has correct disks + 4. Verify VM has correct nics + 5. Destroy VM + """ + + for template in self.templates: + configurations, disks, isos, network = self.get_ova_parsed_information_from_template(template) + + if configurations: + conf = configurations[0] + items = conf['hardwareItems'] + cpu_speed = 1000 + cpu_number = 0 + memory = 0 + for item in items: + if item['resourceType'] == 'Memory': + memory = item['virtualQuantity'] + elif item['resourceType'] == 'Processor': + cpu_number = item['virtualQuantity'] + + nicnetworklist = [] + networks = [] + vm_service = self.services["virtual_machine_vapps"][template.name] + network_mappings = vm_service["nicnetworklist"] + for network_mapping in network_mappings: + network_service = self.services["isolated_network"] + network_offering_id = self.isolated_network_offering.id + if network_mapping["network"] == 'l2': + network_service = self.services["l2-network"] + network_offering_id = self.l2_network_offering.id + network = Network.create( + self.apiclient, + network_service, + networkofferingid=network_offering_id, + accountid=self.account.name, + domainid=self.account.domainid, + zoneid=self.zone.id) + networks.append(network) + for interface in network_mapping["nic"]: + nicnetworklist.append({"nic": interface, "network": network.id}) + + vm = VirtualMachine.create( + self.apiclient, + vm_service, + accountid=self.account.name, + domainid=self.account.domainid, + templateid=template.id, + serviceofferingid=self.custom_offering.id, + zoneid=self.zone.id, + customcpunumber=cpu_number, + customcpuspeed=cpu_speed, + custommemory=memory, + properties=vm_service['properties'], + nicnetworklist=nicnetworklist + ) + + list_vm_response = VirtualMachine.list( + self.apiclient, + id=vm.id + ) + self.debug( + "Verify listVirtualMachines response for virtual machine: %s" \ + % vm.id + ) + self.assertEqual( + isinstance(list_vm_response, list), + True, + "Check list response returns a valid list" + ) + self.assertNotEqual( + len(list_vm_response), + 0, + "Check VM available in List Virtual Machines" + ) + vm_response = list_vm_response[0] + self.assertEqual( + vm_response.id, + vm.id, + "Check virtual machine id in listVirtualMachines" + ) + self.assertEqual( + vm_response.name, + vm.name, + "Check virtual machine name in listVirtualMachines" + ) + self.assertEqual( + vm_response.state, + 'Running', + msg="VM is not in Running state" + ) + + # Verify nics + self.verify_nics(nicnetworklist, vm.id) + # Verify disks + self.verify_disks(disks, vm.id) + # Verify properties + original_properties = vm_service['properties'] + vm_properties = get_vm_vapp_configs(self.apiclient, self.config, self.zone, vm.instancename) + for property in original_properties: + if property["key"] in vm_properties: + self.assertEqual( + vm_properties[property["key"]], + property["value"], + "Check VM property %s with original value" % property["key"] + ) + + cmd = destroyVirtualMachine.destroyVirtualMachineCmd() + cmd.id = vm.id + self.apiclient.destroyVirtualMachine(cmd) diff --git a/tools/apidoc/gen_toc.py b/tools/apidoc/gen_toc.py index ef98b1358987..0b0702ac86c0 100644 --- a/tools/apidoc/gen_toc.py +++ b/tools/apidoc/gen_toc.py @@ -155,6 +155,7 @@ 'createSecondaryStagingStore': 'Image Store', 'deleteSecondaryStagingStore': 'Image Store', 'listSecondaryStagingStores': 'Image Store', + 'updateImageStore': 'Image Store', 'InternalLoadBalancer': 'Internal LB', 'DeploymentPlanners': 'Configuration', 'ObjectStore': 'Image Store', @@ -195,8 +196,10 @@ 'KubernetesSupportedVersion': 'Kubernetes Service', 'KubernetesCluster': 'Kubernetes Service', 'UnmanagedInstance': 'Virtual Machine', - 'Rolling': 'Rolling Maintenance' - } + 'Rolling': 'Rolling Maintenance', + 'importVsphereStoragePolicies' : 'vSphere storage policies', + 'listVsphereStoragePolicies' : 'vSphere storage policies' +} categories = {} diff --git a/tools/marvin/marvin/config/test_data.py b/tools/marvin/marvin/config/test_data.py index 929741b7b0d1..a2d4579063d4 100644 --- a/tools/marvin/marvin/config/test_data.py +++ b/tools/marvin/marvin/config/test_data.py @@ -986,7 +986,60 @@ "ispublic": "True" } }, - + "test_ovf_templates": [ + { + "name": "test-ovf", + "displaytext": "test-ovf", + "format": "ova", + "hypervisor": "vmware", + "ostype": "Other Linux (64-bit)", + "url": "http://172.17.0.1/machina-2dd-iso.ova", + "deployasis": "True", + "requireshvm": "True", + "ispublic": "True" + } + ], + "virtual_machine_vapps": { + "test-ovf": { + "name": "testvm-vapps", + "displayname": "Test VM vApps", + "properties": [ + { + "key": "used.by.admin", + "value": "marvin" + }, + { + "key": "use.type", + "value": "test" + }, + { + "key": "usefull.property", + "value": "True" + } + ], + "nicnetworklist": [ + { + "network": "l2", + "nic": [15, 18] + }, + { + "network": "l2", + "nic": [16] + }, + { + "network": "l2", + "nic": [17] + } + ] + } + }, + "custom_service_offering": { + "name": "Custom Service Offering for vApps", + "displaytext": "Custom Service Offering for vApps", + "cpunumber": "", + "cpuspeed": "", + "memory": "" + }, "coreos_volume": { "diskname": "Volume_core", "urlvmware":"http://dl.openvm.eu/cloudstack/coreos/x86_64/coreos_production_cloudstack_image-vmware.ova", diff --git a/tools/marvin/marvin/lib/base.py b/tools/marvin/marvin/lib/base.py index 10ea666b2de9..a53e295c9744 100755 --- a/tools/marvin/marvin/lib/base.py +++ b/tools/marvin/marvin/lib/base.py @@ -502,7 +502,8 @@ def create(cls, apiclient, services, templateid=None, accountid=None, hostid=None, keypair=None, ipaddress=None, mode='default', method='GET', hypervisor=None, customcpunumber=None, customcpuspeed=None, custommemory=None, rootdisksize=None, - rootdiskcontroller=None, macaddress=None, datadisktemplate_diskoffering_list={}): + rootdiskcontroller=None, macaddress=None, datadisktemplate_diskoffering_list={}, + properties=None, nicnetworklist=None): """Create the instance""" cmd = deployVirtualMachine.deployVirtualMachineCmd() @@ -631,6 +632,12 @@ def create(cls, apiclient, services, templateid=None, accountid=None, elif macaddress in services: cmd.macaddress = services["macaddress"] + if properties: + cmd.properties = properties + + if nicnetworklist: + cmd.nicnetworklist = nicnetworklist + virtual_machine = apiclient.deployVirtualMachine(cmd, method=method) if 'password' in virtual_machine.__dict__.keys(): @@ -1414,6 +1421,9 @@ def register(cls, apiclient, services, zoneid=None, if "directdownload" in services: cmd.directdownload = services["directdownload"] + if "deployasis" in services: + cmd.deployasis = services["deployasis"] + # Register Template template = apiclient.registerTemplate(cmd) diff --git a/tools/marvin/marvin/lib/common.py b/tools/marvin/marvin/lib/common.py index 1c873a3d97ae..fc9c72d88d67 100644 --- a/tools/marvin/marvin/lib/common.py +++ b/tools/marvin/marvin/lib/common.py @@ -395,6 +395,74 @@ def get_test_template(apiclient, zone_id=None, hypervisor=None, test_templates=N return FAILED +def get_test_ovf_templates(apiclient, zone_id=None, test_ovf_templates=None, hypervisor=None): + """ + @Name : get_test_ovf_templates + @Desc : Retrieves the list of test ovf templates used to running tests. When the template + is missing it will be download at most one in a zone for a hypervisor. + @Input : returns a list of templates + """ + result = [] + + if test_ovf_templates is None: + test_ovf_templates = test_data["test_ovf_templates"] + if test_ovf_templates is None: + return result + if hypervisor is None: + return result + hypervisor = hypervisor.lower() + if hypervisor != 'vmware': + return result + + for test_template in test_ovf_templates: + + cmd = listTemplates.listTemplatesCmd() + cmd.name = test_template['name'] + cmd.templatefilter = 'all' + if zone_id is not None: + cmd.zoneid = zone_id + if hypervisor is not None: + cmd.hypervisor = hypervisor + templates = apiclient.listTemplates(cmd) + + if validateList(templates)[0] != PASS: + template = Template.register(apiclient, test_template, zoneid=zone_id, hypervisor=hypervisor.lower(), randomize_name=False) + template.download(apiclient) + retries = 3 + while (template.details == None or len(template.details.__dict__) == 0) and retries > 0: + time.sleep(10) + template_list = Template.list(apiclient, id=template.id, zoneid=zone_id, templatefilter='all') + if isinstance(template_list, list): + template = Template(template_list[0].__dict__) + retries = retries - 1 + if template.details == None or len(template.details.__dict__) == 0: + template.delete(apiclient) + else: + result.append(template) + + if templates: + for template in templates: + if template.isready and template.ispublic and template.details != None and len(template.details.__dict__) > 0: + result.append(template.__dict__) + + return result + +def get_vm_vapp_configs(apiclient, config, setup_zone, vm_name): + zoneDetailsInConfig = [zone for zone in config.zones + if zone.name == setup_zone.name][0] + vcenterusername = zoneDetailsInConfig.vmwaredc.username + vcenterpassword = zoneDetailsInConfig.vmwaredc.password + vcenterip = zoneDetailsInConfig.vmwaredc.vcenter + vcenterObj = Vcenter( + vcenterip, + vcenterusername, + vcenterpassword) + + vms = vcenterObj.get_vms(vm_name) + if vms: + return vms[0]['vm']['properties'] + + def get_windows_template( apiclient, zone_id=None, ostype_desc=None, template_filter="featured", template_type='USER', template_id=None, template_name=None, account=None, domain_id=None, project_id=None, diff --git a/tools/marvin/marvin/lib/vcenter.py b/tools/marvin/marvin/lib/vcenter.py index d14f364bd1cc..78ca50e19d87 100644 --- a/tools/marvin/marvin/lib/vcenter.py +++ b/tools/marvin/marvin/lib/vcenter.py @@ -114,6 +114,12 @@ def _parse_vm(obj): for network in obj.network: vm_details['networks'].append({'name': network.name}) vm_details['raw'] = obj + vm_details['properties'] = {} + config = obj.config + if config and config.vAppConfig: + for prop in config.vAppConfig.property: + vm_details['properties'][prop.id] = prop.value + return vm_details def parse_details(self, obj, vimtype): diff --git a/ui/l10n/en.js b/ui/l10n/en.js index 8e25c54b04cd..7e705131bcf7 100644 --- a/ui/l10n/en.js +++ b/ui/l10n/en.js @@ -674,6 +674,7 @@ var dictionary = { "label.deleting.processing":"Deleting....", "label.deny":"Deny", "label.deployment.planner":"Deployment planner", +"label.deploy.as.is":"Deploy As-Is", "label.description":"Description", "label.destination.physical.network.id":"Destination physical network ID", "label.destination.zone":"Destination Zone", diff --git a/ui/scripts/cloudStack.js b/ui/scripts/cloudStack.js index e6550b730746..9e01271fc137 100644 --- a/ui/scripts/cloudStack.js +++ b/ui/scripts/cloudStack.js @@ -280,6 +280,15 @@ var loginCmdText = array1.join(""); + // Logout before login is called to purge any duplicate sessionkey cookies + // to handle edge cases around upgrades and using legacy UI with Primate + $.ajax({ + url: createURL('logout'), + async: false, + success: function() {}, + error: function() {} + }); + $.ajax({ type: "POST", data: "command=login" + loginCmdText + "&response=json", diff --git a/ui/scripts/docs.js b/ui/scripts/docs.js index 7f29f2b3ac2d..19a7ad0441b3 100755 --- a/ui/scripts/docs.js +++ b/ui/scripts/docs.js @@ -711,7 +711,7 @@ cloudStack.docs = { externalLink: '' }, helpPrimaryStorageProtocol: { - desc: 'For XenServer, choose NFS, iSCSI, or PreSetup. For KVM, choose NFS, SharedMountPoint, RDB, CLVM or Gluster. For vSphere, choose VMFS (iSCSI or FiberChannel) or NFS. For Hyper-V, choose SMB/CIFS. For LXC, choose NFS or SharedMountPoint. For OVM, choose NFS or ocfs2.', + desc: 'For XenServer, choose NFS, iSCSI, or PreSetup. For KVM, choose NFS, SharedMountPoint, RDB, CLVM or Gluster. For vSphere, choose PreSetup (VMFS or iSCSI or FiberChannel or vSAN or vVols) or NFS. For Hyper-V, choose SMB/CIFS. For LXC, choose NFS or SharedMountPoint. For OVM, choose NFS or ocfs2.', externalLink: '' }, helpPrimaryStorageServer: { @@ -1243,6 +1243,10 @@ cloudStack.docs = { desc: 'The Management Server will download the file from the specified URL, such as http://my.web.server/filename.vhd.gz', externalLink: '' }, + helpRegisterTemplateDeployAsIs: { + desc: 'Vmware Only: Deploy with specifications from OVF instead of orchestrated specs', + externalLink: '' + }, helpRegisterTemplateDirectDownload: { desc: 'KVM Only: Secondary Storage is bypassed and template/ISO is downloaded to Primary Storage on deployment', externalLink: '' diff --git a/ui/scripts/instanceWizard.js b/ui/scripts/instanceWizard.js index f06cd9046d48..466db99788b8 100644 --- a/ui/scripts/instanceWizard.js +++ b/ui/scripts/instanceWizard.js @@ -965,17 +965,6 @@ } }); - if (selectedTemplateObj) { - $.ajax({ - url: createURL("listTemplateOvfProperties&id=" + selectedTemplateObj.id), - dataType: "json", - async: false, - success: function(json) { - ovfProps = json.listtemplateovfpropertiesresponse.ovfproperty; - } - }); - } - var $step = $('.step.sshkeyPairs:visible'); if (ovfProps == null || ovfProps.length === 0) { $step.addClass('next-skip-ovf-properties'); @@ -984,15 +973,6 @@ } }, - // Step PRE-8: Configure OVF Properties (if available) for the template - function(args) { - args.response.success({ - data: { - ovfProperties: ovfProps - } - }); - }, - // Step 8: Review function(args) { var $step = $('.step.review:visible'); @@ -1054,8 +1034,8 @@ } }); for (var k = 0; k < deployOvfProperties.length; k++) { - deployVmData["ovfproperties[" + k + "].key"] = deployOvfProperties[k].key; - deployVmData["ovfproperties[" + k + "].value"] = deployOvfProperties[k].value; + deployVmData["properties[" + k + "].key"] = deployOvfProperties[k].key; + deployVmData["properties[" + k + "].value"] = deployOvfProperties[k].value; } } diff --git a/ui/scripts/system.js b/ui/scripts/system.js index 418ce53bc4c1..ca0eb7e3aeb7 100755 --- a/ui/scripts/system.js +++ b/ui/scripts/system.js @@ -18735,8 +18735,8 @@ description: "nfs" }); items.push({ - id: "vmfs", - description: "vmfs" + id: "presetup", + description: "presetup" }); items.push({ id: "custom", @@ -18885,7 +18885,7 @@ $form.find('.form-item[rel=rbdsecret]').hide(); $form.find('.form-item[rel=glustervolume]').hide(); - } else if (protocol == "PreSetup") { + } else if (protocol == "PreSetup" && selectedClusterObj.hypervisortype != "VMware") { $form.find('.form-item[rel=server]').hide(); $form.find('.form-item[rel=server]').find(".value").find("input").val("localhost"); @@ -18983,7 +18983,7 @@ $form.find('.form-item[rel=rbdsecret]').hide(); $form.find('.form-item[rel=glustervolume]').hide(); - } else if (protocol == "vmfs") { + } else if (protocol == "presetup" && selectedClusterObj.hypervisortype == "VMware") { $form.find('.form-item[rel=server]').css('display', 'inline-block'); $form.find('.form-item[rel=server]').find(".value").find("input").val(""); @@ -19408,7 +19408,7 @@ array1.push("&details[0].user=" + args.data.smbUsername); array1.push("&details[1].password=" + encodeURIComponent(args.data.smbPassword)); array1.push("&details[2].domain=" + args.data.smbDomain); - } else if (args.data.protocol == "PreSetup") { + } else if (args.data.protocol == "PreSetup" && selectedClusterObj.hypervisortype != "VMware") { var path = args.data.path; if (path.substring(0, 1) != "/") path = "/" + path; @@ -19434,12 +19434,12 @@ var rbdid = args.data.rbdid; var rbdsecret = args.data.rbdsecret; url = rbdURL(rbdmonitor, rbdpool, rbdid, rbdsecret); - } else if (args.data.protocol == "vmfs") { + } else if (args.data.protocol == "presetup" && selectedClusterObj.hypervisortype == "VMware") { var path = args.data.vCenterDataCenter; if (path.substring(0, 1) != "/") path = "/" + path; path += "/" + args.data.vCenterDataStore; - url = vmfsURL("dummy", path); + url = presetupURL("dummy", path); } else if (args.data.protocol == "gluster") { var glustervolume = args.data.glustervolume; diff --git a/ui/scripts/templates.js b/ui/scripts/templates.js index 1516286f01a9..aeae6f04cc0d 100644 --- a/ui/scripts/templates.js +++ b/ui/scripts/templates.js @@ -248,6 +248,7 @@ $form.find('.form-item[rel=rootDiskControllerType]').css('display', 'inline-block'); $form.find('.form-item[rel=nicAdapterType]').css('display', 'inline-block'); $form.find('.form-item[rel=keyboardType]').css('display', 'inline-block'); + $form.find('.form-item[rel=deployAsIs]').css('display', 'inline-block'); $form.find('.form-item[rel=xenserverToolsVersion61plus]').hide(); $form.find('.form-item[rel=rootDiskControllerTypeKVM]').hide(); $form.find('.form-item[rel=directdownload]').hide(); @@ -258,6 +259,7 @@ $form.find('.form-item[rel=keyboardType]').hide(); $form.find('.form-item[rel=rootDiskControllerTypeKVM]').hide(); $form.find('.form-item[rel=directdownload]').hide(); + $form.find('.form-item[rel=deployAsIs]').hide(); $form.find('.form-item[rel=requireshvm]').css('display', 'inline-block'); if (isAdmin()) { @@ -268,6 +270,7 @@ $form.find('.form-item[rel=nicAdapterType]').hide(); $form.find('.form-item[rel=keyboardType]').hide(); $form.find('.form-item[rel=xenserverToolsVersion61plus]').hide(); + $form.find('.form-item[rel=deployAsIs]').hide(); $form.find('.form-item[rel=rootDiskControllerTypeKVM]').css('display', 'inline-block'); $('#label_root_disk_controller').prop('selectedIndex', 2); $form.find('.form-item[rel=requireshvm]').css('display', 'inline-block'); @@ -281,6 +284,7 @@ $form.find('.form-item[rel=xenserverToolsVersion61plus]').hide(); $form.find('.form-item[rel=rootDiskControllerTypeKVM]').hide(); $form.find('.form-item[rel=directdownload]').hide(); + $form.find('.form-item[rel=deployAsIs]').hide(); $form.find('.form-item[rel=requireshvm]').css('display', 'inline-block'); } }); @@ -463,6 +467,13 @@ }); } }, + deployAsIs : { + label: 'label.deploy.as.is', + docID: 'helpRegisterTemplateDeployAsIs', + isBoolean: true, + dependsOn: 'hypervisor', + isHidden: true + }, // fields for hypervisor == "VMware" (ends here) format: { @@ -683,6 +694,11 @@ 'details[0].keyboard': args.data.keyboardType }); } + if (args.$form.find('.form-item[rel=deployAsIs]').css("display") != "none" && args.data.deployAsIs != "") { + $.extend(data, { + deployAsIs: (args.data.deployAsIs == "on") ? "true" : "false" + }); + } // for hypervisor == VMware (ends here) $.ajax({ @@ -1823,18 +1839,7 @@ } }, tabFilter: function (args) { - $.ajax({ - url: createURL("listTemplateOvfProperties&id=" + args.context.templates[0].id), - dataType: "json", - async: false, - success: function(json) { - ovfprops = json.listtemplateovfpropertiesresponse.ovfproperty; - } - }); var hiddenTabs = []; - if (ovfprops == null || ovfprops.length === 0) { - hiddenTabs.push("ovfpropertiestab"); - } return hiddenTabs; }, tabs: { @@ -1919,6 +1924,11 @@ isBoolean: true, converter: cloudStack.converters.toBooleanText }, + deployAsIs: { + label: 'label.deploy.as.is', + isBoolean: true, + converter: cloudStack.converters.toBooleanText + }, isextractable: { label: 'label.extractable.lower', isBoolean: true, @@ -2605,57 +2615,7 @@ } } }) - }, - - /** - * OVF properties tab (only displayed when OVF properties are available) - */ - ovfpropertiestab: { - title: 'label.ovf.properties', - listView: { - id: 'ovfproperties', - fields: { - label: { - label: 'label.label' - }, - description: { - label: 'label.description' - }, - value: { - label: 'label.value' - } - }, - hideSearchBar: true, - dataProvider: function(args) { - $.ajax({ - url: createURL("listTemplateOvfProperties"), - data: { - id: args.context.templates[0].id - }, - success: function(json) { - var ovfprops = json.listtemplateovfpropertiesresponse.ovfproperty; - var listDetails = []; - for (index in ovfprops){ - var prop = ovfprops[index]; - var det = {}; - det['label'] = prop['label']; - det['description'] = prop['description']; - det['value'] = prop['value']; - listDetails.push(det); - } - args.response.success({ - data: listDetails - }); - }, - - error: function(json) { - args.response.error(parseXMLHttpResponse(json)); - } - }); - - } - } - } + } } } } @@ -2851,6 +2811,11 @@ docID: 'helpRegisterISOFeatured', isBoolean: true, isHidden: true + }, + deployAsIs : { + label: 'label.deploy.as.is', + docID: 'helpRegisterTemplateDeployAsIs', + isBoolean: true } } }, @@ -2864,7 +2829,8 @@ zoneid: args.data.zone, isextractable: (args.data.isExtractable == "on"), bootable: (args.data.isBootable == "on"), - directdownload: (args.data.directdownload == "on") + directdownload: (args.data.directdownload == "on"), + deployAsIs: (args.data.deployAsIs == "on") }; if (args.$form.find('.form-item[rel=osTypeId]').css("display") != "none") { @@ -3701,6 +3667,11 @@ isBoolean: true, converter: cloudStack.converters.toBooleanText }, + deployAsIs: { + label: 'label.deploy.as.is', + isBoolean: true, + converter: cloudStack.converters.toBooleanText + }, size: { label: 'label.size', converter: function(args) { diff --git a/ui/scripts/ui-custom/instanceWizard.js b/ui/scripts/ui-custom/instanceWizard.js index 4aefa97b335b..2004257b2c64 100644 --- a/ui/scripts/ui-custom/instanceWizard.js +++ b/ui/scripts/ui-custom/instanceWizard.js @@ -172,10 +172,20 @@ .html(qualifier) propertyField.append(option); }); - } else if (qualifiers.startsWith("MaxLen")) { - var length = qualifiers.replace("MaxLen(","").slice(0,-1); + } else if (qualifiers.includes("MinLen") || qualifiers.includes("MaxLen") ) { + var split = qualifiers.split(","); + var minLen = 0; + var maxLen = 524288; /* 524288 being the default according to w3schools */ + for ( var i = 0; i < split.length; i++ ) { + if (split[i].startsWith("MaxLen")) { + maxLen = split[i].replace("MaxLen(","").replace(")",""); + } else if (split[i].startsWith("MinLen")) { + minLen = split[i].replace("MinLen(","").replace(")",""); + } + } propertyField = $('') - .attr({maxlength : length, type: fieldType}) + .attr({pattern : '.{' + minLen + ',' + maxLen + '}'}) + .attr({type: fieldType}) .addClass('name').val(_s(this[fields.value])) } } else { diff --git a/ui/scripts/zoneWizard.js b/ui/scripts/zoneWizard.js index f4d0e5db5214..b7e5d019ce5d 100755 --- a/ui/scripts/zoneWizard.js +++ b/ui/scripts/zoneWizard.js @@ -1434,8 +1434,8 @@ description: "nfs" }); items.push({ - id: "vmfs", - description: "vmfs" + id: "presetup", + description: "presetup" }); args.response.success({ data: items @@ -1576,7 +1576,7 @@ $form.find('[rel=rbdsecret]').hide(); $form.find('[rel=glustervolume]').hide(); - } else if (protocol == "PreSetup") { + } else if (protocol == "PreSetup" && selectedClusterObj.hypervisortype != "VMware") { $form.find('[rel=server]').hide(); $form.find('[rel=server]').find(".value").find("input").val("localhost"); @@ -1649,7 +1649,7 @@ $form.find('[rel=rbdsecret]').hide(); $form.find('[rel=glustervolume]').hide(); - } else if (protocol == "vmfs") { + } else if (protocol == "presetup" && selectedClusterObj.hypervisortype == "VMware") { $form.find('[rel=server]').css('display', 'block'); $form.find('[rel=server]').find(".value").find("input").val(""); @@ -4529,7 +4529,7 @@ array1.push("&details[0].user=" + args.data.primaryStorage.smbUsername); array1.push("&details[1].password=" + encodeURIComponent(args.data.primaryStorage.smbPassword)); array1.push("&details[2].domain=" + args.data.primaryStorage.smbDomain); - } else if (args.data.primaryStorage.protocol == "PreSetup") { + } else if (args.data.primaryStorage.protocol == "PreSetup" && selectedClusterObj.hypervisortype != "VMware") { var path = args.data.primaryStorage.path; if (path.substring(0, 1) != "/") path = "/" + path; @@ -4555,12 +4555,12 @@ var rbdid = args.data.primaryStorage.rbdid; var rbdsecret = args.data.primaryStorage.rbdsecret; url = rbdURL(rbdmonitor, rbdpool, rbdid, rbdsecret); - } else if (args.data.primaryStorage.protocol == "vmfs") { + } else if (args.data.primaryStorage.protocol == "presetup" && selectedClusterObj.hypervisortype == "VMware") { var path = args.data.primaryStorage.vCenterDataCenter; if (path.substring(0, 1) != "/") path = "/" + path; path += "/" + args.data.primaryStorage.vCenterDataStore; - url = vmfsURL("dummy", path); + url = presetupURL("dummy", path); } else { var iqn = args.data.primaryStorage.iqn; if (iqn.substring(0, 1) != "/") diff --git a/utils/pom.xml b/utils/pom.xml index 8a745aa21a71..75c9cd463aa3 100755 --- a/utils/pom.xml +++ b/utils/pom.xml @@ -178,6 +178,10 @@ jackson-databind ${cs.jackson.version} + + org.apache.commons + commons-compress + diff --git a/utils/src/main/java/com/cloud/utils/compression/CompressionUtil.java b/utils/src/main/java/com/cloud/utils/compression/CompressionUtil.java new file mode 100644 index 000000000000..20f0bc8eb7c7 --- /dev/null +++ b/utils/src/main/java/com/cloud/utils/compression/CompressionUtil.java @@ -0,0 +1,58 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package com.cloud.utils.compression; + +import org.apache.log4j.Logger; + +import java.io.BufferedReader; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; + +public class CompressionUtil { + + private static final Logger s_logger = Logger.getLogger(CompressionUtil.class); + + public byte[] compressString(String inputStr) throws IOException { + ByteArrayOutputStream obj = new ByteArrayOutputStream(); + GZIPOutputStream gzip = new GZIPOutputStream(obj); + gzip.write(inputStr.getBytes(StandardCharsets.UTF_8)); + gzip.close(); + return obj.toByteArray(); + } + + public String decompressByteArary(byte[] compressed) throws IOException { + GZIPInputStream gis = new GZIPInputStream(new ByteArrayInputStream(compressed)); + BufferedReader bf = new BufferedReader(new InputStreamReader(gis, StandardCharsets.UTF_8)); + String line = bf.readLine(); + StringBuilder builder = new StringBuilder(); + while (line != null) { + builder.append(line); + line = bf.readLine(); + if (line != null) { + builder.append("\n"); + } + } + return builder.toString(); + } +} diff --git a/utils/src/test/java/com/cloud/utils/compression/CompressionUtilTest.java b/utils/src/test/java/com/cloud/utils/compression/CompressionUtilTest.java new file mode 100644 index 000000000000..e247d6c9435e --- /dev/null +++ b/utils/src/test/java/com/cloud/utils/compression/CompressionUtilTest.java @@ -0,0 +1,128 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package com.cloud.utils.compression; + +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; + +public class CompressionUtilTest { + + private CompressionUtil util = new CompressionUtil(); + + private String testEula = "END USER LICENSE AGREEMENT\n" + + "\n" + + "IMPORTANT: PLEASE READ THIS END USER LICENSE AGREEMENT CAREFULLY. IT IS VERY IMPORTANT THAT YOU CHECK THAT YOU ARE PURCHASING CISCO SOFTWARE OR EQUIPMENT FROM AN APPROVED SOURCE AND THAT YOU, OR THE ENTITY YOU REPRESENT (COLLECTIVELY, THE \"CUSTOMER\") HAVE BEEN REGISTERED AS THE END USER FOR THE PURPOSES OF THIS CISCO END USER LICENSE AGREEMENT. IF YOU ARE NOT REGISTERED AS THE END USER YOU HAVE NO LICENSE TO USE THE SOFTWARE AND THE LIMITED WARRANTY IN THIS END USER LICENSE AGREEMENT DOES NOT APPLY. ASSUMING YOU HAVE PURCHASED FROM AN APPROVED SOURCE, DOWNLOADING, INSTALLING OR USING CISCO OR CISCO-SUPPLIED SOFTWARE CONSTITUTES ACCEPTANCE OF THIS AGREEMENT.\n" + + "\n" + + "CISCO SYSTEMS, INC. OR ITS AFFILIATE LICENSING THE SOFTWARE (\"CISCO\") IS WILLING TO LICENSE THIS SOFTWARE TO YOU ONLY UPON THE CONDITION THAT YOU PURCHASED THE SOFTWARE FROM AN APPROVED SOURCE AND THAT YOU ACCEPT ALL OF THE TERMS CONTAINED IN THIS END USER LICENSE AGREEMENT PLUS ANY ADDITIONAL LIMITATIONS ON THE LICENSE SET FORTH IN A SUPPLEMENTAL LICENSE AGREEMENT ACCOMPANYING THE PRODUCT, MADE AVAILABLE AT THE TIME OF YOUR ORDER, OR POSTED ON THE CISCO WEBSITE AT www.cisco.com/go/terms (COLLECTIVELY THE \"AGREEMENT\"). TO THE EXTENT OF ANY CONFLICT BETWEEN THE TERMS OF THIS END USER LICENSE AGREEMENT AND ANY SUPPLEMENTAL LICENSE AGREEMENT, THE SUPPLEMENTAL " + + "LICENSE AGREEMENT SHALL APPLY. BY DOWNLOADING, INSTALLING, OR USING THE SOFTWARE, YOU ARE REPRESENTING THAT YOU PURCHASED THE SOFTWARE FROM AN APPROVED SOURCE AND BINDING YOURSELF TO THE AGREEMENT. IF YOU DO NOT AGREE TO ALL OF THE TERMS OF THE AGREEMENT, THEN CISCO IS UNWILLING TO LICENSE THE SOFTWARE TO YOU AND (A) YOU MAY NOT DOWNLOAD, INSTALL OR USE THE SOFTWARE, AND (B) YOU MAY RETURN THE SOFTWARE (INCLUDING ANY UNOPENED CD PACKAGE AND ANY WRITTEN MATERIALS) FOR A FULL REFUND, OR, IF THE SOFTWARE AND WRITTEN MATERIALS ARE SUPPLIED AS PART OF ANOTHER PRODUCT, YOU MAY RETURN THE ENTIRE PRODUCT FOR A FULL REFUND. YOUR RIGHT TO RETURN AND REFUND EXPIRES 30 " + + "DAYS AFTER PURCHASE FROM AN APPROVED SOURCE, AND APPLIES ONLY IF YOU ARE THE ORIGINAL AND REGISTERED END USER PURCHASER. FOR THE PURPOSES OF THIS END USER LICENSE AGREEMENT, AN \"APPROVED SOURCE\" MEANS (A) CISCO; OR (B) A DISTRIBUTOR OR SYSTEMS INTEGRATOR AUTHORIZED BY CISCO TO DISTRIBUTE / SELL CISCO EQUIPMENT, SOFTWARE AND SERVICES WITHIN YOUR TERRITORY TO END USERS; OR (C) A RESELLER AUTHORIZED BY ANY SUCH DISTRIBUTOR OR SYSTEMS INTEGRATOR IN ACCORDANCE WITH THE TERMS OF THE DISTRIBUTOR'S AGREEMENT WITH CISCO TO DISTRIBUTE / SELL THE CISCO EQUIPMENT, SOFTWARE AND SERVICES WITHIN YOUR TERRITORY TO END USERS.\n" + + "\n" + + "THE FOLLOWING TERMS OF THE AGREEMENT GOVERN CUSTOMER'S USE OF THE SOFTWARE (DEFINED BELOW), EXCEPT TO THE EXTENT: (A) THERE IS A SEPARATE SIGNED CONTRACT BETWEEN CUSTOMER AND CISCO GOVERNING CUSTOMER'S USE OF THE SOFTWARE, OR (B) THE SOFTWARE INCLUDES A SEPARATE \"CLICK-ACCEPT\" LICENSE AGREEMENT OR THIRD PARTY LICENSE AGREEMENT AS PART OF THE INSTALLATION OR DOWNLOAD PROCESS GOVERNING CUSTOMER'S USE OF THE SOFTWARE. TO THE EXTENT OF A CONFLICT BETWEEN THE PROVISIONS OF THE FOREGOING DOCUMENTS, THE ORDER OF PRECEDENCE SHALL BE (1)THE SIGNED CONTRACT, (2) THE CLICK-ACCEPT AGREEMENT OR THIRD PARTY LICENSE AGREEMENT, AND (3) THE AGREEMENT. FOR PURPOSES OF THE " + + "AGREEMENT, \"SOFTWARE\" SHALL MEAN COMPUTER PROGRAMS, INCLUDING FIRMWARE AND COMPUTER PROGRAMS EMBEDDED IN CISCO EQUIPMENT, AS PROVIDED TO CUSTOMER BY AN APPROVED SOURCE, AND ANY UPGRADES, UPDATES, BUG FIXES OR MODIFIED VERSIONS THERETO (COLLECTIVELY, \"UPGRADES\"), ANY OF THE SAME WHICH HAS BEEN RELICENSED UNDER THE CISCO SOFTWARE TRANSFER AND RE-LICENSING POLICY (AS MAY BE AMENDED BY CISCO FROM TIME TO TIME) OR BACKUP COPIES OF ANY OF THE FOREGOING.\n" + + "\n" + + "License. Conditioned upon compliance with the terms and conditions of the Agreement, Cisco grants to Customer a nonexclusive and nontransferable license to use for Customer's internal business purposes the Software and the Documentation for which Customer has paid the required license fees to an Approved Source. \"Documentation\" means written information (whether contained in user or technical manuals, training materials, specifications or otherwise) pertaining to the Software and made available by an Approved Source with the Software in any manner (including on CD-Rom, or on-line). In order to use the Software, Customer may be required to input a registration number or product authorization key and register Customer's copy of the Software online at Cisco's website to obtain the necessary license key or license file.\n" + + "\n" + + "Customer's license to use the Software shall be limited to, and Customer shall not use the Software in excess of, a single hardware chassis or card or such other limitations as are set forth in the applicable Supplemental License Agreement or in the applicable purchase order which has been accepted by an Approved Source and for which Customer has paid to an Approved Source the required license fee (the \"Purchase Order\").\n" + + "\n" + + "Unless otherwise expressly provided in the Documentation or any applicable Supplemental License Agreement, Customer shall use the Software solely as embedded in, for execution on, or (where the applicable Documentation permits installation on non-Cisco equipment) for communication with Cisco equipment owned or leased by Customer and used for Customer's internal business purposes. No other licenses are granted by implication, estoppel or otherwise.\n" + + "\n" + + "For evaluation or beta copies for which Cisco does not charge a license fee, the above requirement to pay license fees does not apply.\n" + + "\n" + + "General Limitations. This is a license, not a transfer of title, to the Software and Documentation, and Cisco retains ownership of all copies of the Software and Documentation. Customer acknowledges that the Software and Documentation contain trade secrets of Cisco or its suppliers or licensors, including but not limited to the specific internal design and structure of individual programs and associated interface information. Except as otherwise expressly provided under the Agreement, Customer shall only use the Software in connection with the use of Cisco equipment purchased by the Customer from an Approved Source and Customer shall have no right, and Customer specifically agrees not to:\n" + + "\n" + + "(i) transfer, assign or sublicense its license rights to any other person or entity (other than in compliance with any Cisco relicensing/transfer policy then in force), or use the Software on Cisco equipment not purchased by the Customer from an Approved Source or on secondhand Cisco equipment, and Customer acknowledges that any attempted transfer, assignment, sublicense or use shall be void;\n" + + "\n" + + "(ii) make error corrections to or otherwise modify or adapt the Software or create derivative works based upon the Software, or permit third parties to do the same;\n" + + "\n" + + "(iii) reverse engineer or decompile, decrypt, disassemble or otherwise reduce the Software to human-readable form, except to the extent otherwise expressly permitted under applicable law notwithstanding this restriction or except to the extent that Cisco is legally required to permit such specific activity pursuant to any applicable open source license;\n" + + "\n" + + "(iv) publish any results of benchmark tests run on the Software;\n" + + "\n" + + "(v) use or permit the Software to be used to perform services for third parties, whether on a service bureau or time sharing basis or otherwise, without the express written authorization of Cisco; or\n" + + "\n" + + "(vi) disclose, provide, or otherwise make available trade secrets contained within the Software and Documentation in any form to any third party without the prior written consent of Cisco. Customer shall implement reasonable security measures to protect such trade secrets.\n" + + "\n" + + "To the extent required by applicable law, and at Customer's written request, Cisco shall provide Customer with the interface information needed to achieve interoperability between the Software and another independently created program, on payment of Cisco's applicable fee, if any. Customer shall observe strict obligations of confidentiality with respect to such information and shall use such information in compliance with any applicable terms and conditions upon which Cisco makes such information available.\n" + + "\n" + + "Software, Upgrades and Additional Copies. NOTWITHSTANDING ANY OTHER PROVISION OF THE AGREEMENT: (1) CUSTOMER HAS NO LICENSE OR RIGHT TO MAKE OR USE ANY ADDITIONAL COPIES OR UPGRADES UNLESS CUSTOMER, AT THE TIME OF MAKING OR ACQUIRING SUCH COPY OR UPGRADE, ALREADY HOLDS A VALID LICENSE TO THE ORIGINAL SOFTWARE AND HAS PAID THE APPLICABLE FEE TO AN APPROVED SOURCE FOR THE UPGRADE OR ADDITIONAL COPIES; (2) USE OF UPGRADES IS LIMITED TO CISCO EQUIPMENT SUPPLIED BY AN APPROVED SOURCE FOR WHICH CUSTOMER IS THE ORIGINAL END USER PURCHASER OR LESSEE OR OTHERWISE HOLDS A VALID LICENSE TO USE THE SOFTWARE WHICH IS BEING UPGRADED; AND (3) THE MAKING AND USE OF ADDITIONAL COPIES IS LIMITED TO NECESSARY BACKUP PURPOSES ONLY.\n" + + "\n" + + "Proprietary Notices. Customer agrees to maintain and reproduce all copyright, proprietary, and other notices on all copies, in any form, of the Software in the same form and manner that such copyright and other proprietary notices are included on the Software. Except as expressly authorized in the Agreement, Customer shall not make any copies or duplicates of any Software without the prior written permission of Cisco.\n" + + "\n" + + "Term and Termination. The Agreement and the license granted herein shall remain effective until terminated. Customer may terminate the Agreement and the license at any time by destroying all copies of Software and any Documentation. Customer's rights under the Agreement will terminate immediately without notice from Cisco if Customer fails to comply with any provision of the Agreement. Upon termination, Customer shall destroy all copies of Software and Documentation in its possession or control. All confidentiality obligations of Customer, " + + "all restrictions and limitations imposed on the Customer under the section titled \"General Limitations\" and all limitations of liability and disclaimers and restrictions of warranty shall survive termination of this Agreement. In addition, the provisions of the sections titled \"U.S. Government End User Purchasers\" and \"General Terms Applicable to the Limited Warranty Statement and End User License Agreement\" shall survive termination of the Agreement.\n" + + "\n" + + "Customer Records. Customer grants to Cisco and its independent accountants the right to examine Customer's books, records and accounts during Customer's normal business hours to verify compliance with this Agreement. In the event such audit discloses non-compliance with this Agreement, Customer shall promptly pay to Cisco the appropriate license fees, plus the reasonable cost of conducting the audit.\n" + + "\n" + + "Export, Re-Export, Transfer and Use Controls. The Software, Documentation and technology or direct products thereof (hereafter referred to as Software and Technology), supplied by Cisco under the Agreement are subject to export controls under the laws and regulations of the United States (\"U.S.\") and any other applicable countries' laws and regulations. Customer shall comply with such laws and regulations governing export, re-export, import, transfer and use of Cisco Software and Technology and will obtain all required U.S. and local authorizations, permits, or licenses. Cisco and Customer each agree to provide the other information, support documents, and assistance as may reasonably be required by the other in connection with securing authorizations or licenses. Information regarding compliance with export, re-export, transfer and use may be located at the following URL: www.cisco.com/web/about/doing_business/legal/global_export_trade/general_export/contract_compliance.html\n" + + "\n" + + "U.S. Government End User Purchasers. The Software and Documentation qualify as \"commercial items,\" as that term is defined at Federal Acquisition Regulation (\"FAR\") (48 C.F.R.) 2.101, consisting of \"commercial computer software\" and \"commercial computer software documentation\" as such terms are used in FAR 12.212. Consistent with FAR 12.212 and DoD FAR Supp. 227.7202-1 through 227.7202-4, and notwithstanding any other FAR or other contractual clause to the contrary in any agreement into which the Agreement may be incorporated, Customer may provide to Government end user or, if the Agreement is direct, Government end user will acquire, the Software and Documentation with only those rights set forth in the Agreement. Use of either the Software or Documentation or both constitutes agreement by the Government that the Software and Documentation are \"commercial computer software\" and \"commercial " + + "computer software documentation,\" and constitutes acceptance of the rights and restrictions herein.\n" + + "\n" + + "Identified Components; Additional Terms. The Software may contain or be delivered with one or more components, which may include third-party components, identified by Cisco in the Documentation, readme.txt file, third-party click-accept or elsewhere (e.g. on www.cisco.com) (the \"Identified Component(s)\") as being subject to different license agreement terms, disclaimers of warranties, limited warranties or other terms and conditions (collectively, \"Additional Terms\") than those set forth herein. You agree to the applicable Additional Terms for any such Identified Component(s).\n" + + "\n" + + "Limited Warranty\n" + + "\n" + + "Subject to the limitations and conditions set forth herein, Cisco warrants that commencing from the date of shipment to Customer (but in case of resale by an Approved Source other than Cisco, commencing not more than ninety (90) days after original shipment by Cisco), and continuing for a period of the longer of (a) ninety (90) days or (b) the warranty period (if any) expressly set forth as applicable specifically to software in the warranty card accompanying the product of which the Software is a part (the \"Product\") (if any): (a) the media on which the Software is furnished will be free of defects in materials and workmanship under normal use; and (b) the Software substantially conforms to the Documentation. The date of shipment of a Product by Cisco is set forth " + + "on the packaging material in which the Product is shipped. Except for the foregoing, the Software is provided \"AS IS\". This limited warranty extends only to the Software purchased from an Approved Source by a Customer who is the first registered end user. Customer's sole and exclusive remedy and the entire liability of Cisco and its suppliers under this limited warranty will be (i) replacement of defective media and/or (ii) at Cisco's option, repair, replacement, or refund of the purchase price of the Software, in both cases subject to the condition that any error or defect constituting a breach of this limited warranty is reported to the Approved Source supplying the Software to Customer, within the warranty period. Cisco or the Approved Source supplying the Software to Customer may, at its option, require return of the Software and/or Documentation as a condition to the remedy. " + + "In no event does Cisco warrant that the Software is error free or that Customer will be able to operate the Software without problems or interruptions. In addition, due to the continual development of new techniques for intruding upon and attacking networks, Cisco does not warrant that the Software or any equipment, system or network on which the Software is used will be free of vulnerability to intrusion or attack.\n" + + "\n" + + "Restrictions. This warranty does not apply if the Software, Product or any other equipment upon which the Software is authorized to be used (a) has been altered, except by Cisco or its authorized representative, (b) has not been installed, operated, repaired, or maintained in accordance with instructions supplied by Cisco, (c) has been subjected to abnormal physical or electrical stress, abnormal environmental conditions, misuse, negligence, or accident; or (d) is licensed for beta, evaluation, testing or demonstration purposes. The Software warranty also does not apply to (e) any temporary Software modules; (f) any Software not posted on Cisco's Software Center; (g) any Software that Cisco expressly provides on an \"AS IS\" basis on Cisco's Software Center; (h) any Software for which an Approved Source does not receive a license fee; and (i) Software supplied by any third party which is not an Approved Source.\n" + + "\n" + + "DISCLAIMER OF WARRANTY\n" + + "\n" + + "EXCEPT AS SPECIFIED IN THIS WARRANTY SECTION, ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS, AND WARRANTIES INCLUDING, WITHOUT LIMITATION, ANY IMPLIED WARRANTY OR CONDITION OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, SATISFACTORY QUALITY, NON-INTERFERENCE, ACCURACY OF INFORMATIONAL CONTENT, OR ARISING FROM A COURSE OF DEALING, LAW, USAGE, OR TRADE PRACTICE, ARE HEREBY EXCLUDED TO THE EXTENT ALLOWED BY APPLICABLE LAW AND ARE EXPRESSLY DISCLAIMED BY CISCO, ITS SUPPLIERS AND LICENSORS. TO THE EXTENT THAT ANY OF THE SAME CANNOT BE EXCLUDED, SUCH IMPLIED CONDITION, REPRESENTATION AND/OR WARRANTY IS LIMITED IN DURATION TO THE EXPRESS WARRANTY PERIOD REFERRED TO IN THE \"LIMITED WARRANTY\" SECTION ABOVE. BECAUSE SOME STATES OR JURISDICTIONS " + + "DO NOT ALLOW LIMITATIONS ON HOW LONG AN IMPLIED WARRANTY LASTS, THE ABOVE LIMITATION MAY NOT APPLY IN SUCH STATES. THIS WARRANTY GIVES CUSTOMER SPECIFIC LEGAL RIGHTS, AND CUSTOMER MAY ALSO HAVE OTHER RIGHTS WHICH VARY FROM JURISDICTION TO JURISDICTION. This disclaimer and exclusion shall apply even if the express warranty set forth above fails of its essential purpose.\n" + + "\n" + + "Disclaimer of Liabilities-Limitation of Liability. IF YOU ACQUIRED THE SOFTWARE IN THE UNITED STATES, LATIN AMERICA, CANADA, JAPAN OR THE CARIBBEAN, NOTWITHSTANDING ANYTHING ELSE IN THE AGREEMENT TO THE CONTRARY, ALL LIABILITY OF CISCO, ITS AFFILIATES, OFFICERS, DIRECTORS, EMPLOYEES, AGENTS, SUPPLIERS AND LICENSORS COLLECTIVELY, TO CUSTOMER, WHETHER IN CONTRACT, TORT (INCLUDING NEGLIGENCE), BREACH OF WARRANTY OR OTHERWISE, SHALL NOT EXCEED THE PRICE PAID BY CUSTOMER TO ANY APPROVED SOURCE FOR THE SOFTWARE THAT GAVE RISE TO THE CLAIM OR IF THE SOFTWARE IS PART OF ANOTHER PRODUCT, THE PRICE PAID FOR SUCH OTHER PRODUCT. THIS LIMITATION OF LIABILITY FOR SOFTWARE IS CUMULATIVE AND NOT PER INCIDENT (I.E. THE EXISTENCE OF TWO OR MORE CLAIMS WILL NOT ENLARGE THIS LIMIT).\n" + + "\n" + + "IF YOU ACQUIRED THE SOFTWARE IN EUROPE, THE MIDDLE EAST, AFRICA, ASIA OR OCEANIA, NOTWITHSTANDING ANYTHING ELSE IN THE AGREEMENT TO THE CONTRARY, ALL LIABILITY OF CISCO, ITS AFFILIATES, OFFICERS, DIRECTORS, EMPLOYEES, AGENTS, SUPPLIERS AND LICENSORS COLLECTIVELY, TO CUSTOMER, WHETHER IN CONTRACT, TORT (INCLUDING NEGLIGENCE), BREACH OF WARRANTY OR OTHERWISE, SHALL NOT EXCEED THE PRICE PAID BY CUSTOMER TO CISCO FOR THE SOFTWARE THAT GAVE RISE TO THE CLAIM OR IF THE SOFTWARE IS PART OF ANOTHER PRODUCT, THE PRICE PAID FOR SUCH OTHER PRODUCT. THIS LIMITATION OF LIABILITY FOR SOFTWARE IS CUMULATIVE AND NOT PER INCIDENT (I.E. THE EXISTENCE OF TWO OR MORE CLAIMS WILL NOT ENLARGE THIS LIMIT). NOTHING IN THE AGREEMENT SHALL LIMIT (I) THE LIABILITY OF CISCO, ITS AFFILIATES, OFFICERS, DIRECTORS, EMPLOYEES, AGENTS, SUPPLIERS AND LICENSORS TO CUSTOMER FOR PERSONAL " + + "INJURY OR DEATH CAUSED BY THEIR NEGLIGENCE, (II) CISCO'S LIABILITY FOR FRAUDULENT MISREPRESENTATION, OR (III) ANY LIABILITY OF CISCO WHICH CANNOT BE EXCLUDED UNDER APPLICABLE LAW.\n" + + "\n" + + "Disclaimer of Liabilities-Waiver of Consequential Damages and Other Losses. IF YOU ACQUIRED THE SOFTWARE IN THE UNITED STATES, LATIN AMERICA, THE CARIBBEAN OR CANADA, REGARDLESS OF WHETHER ANY REMEDY SET FORTH HEREIN FAILS OF ITS ESSENTIAL PURPOSE OR OTHERWISE, IN NO EVENT WILL CISCO OR ITS SUPPLIERS BE LIABLE FOR ANY LOST REVENUE, PROFIT, OR LOST OR DAMAGED DATA, BUSINESS INTERRUPTION, LOSS OF CAPITAL, OR FOR SPECIAL, INDIRECT, CONSEQUENTIAL, INCIDENTAL, OR PUNITIVE DAMAGES HOWEVER CAUSED AND REGARDLESS OF THE THEORY OF LIABILITY OR WHETHER ARISING OUT OF THE USE OF OR INABILITY TO USE SOFTWARE OR OTHERWISE AND EVEN IF CISCO OR ITS SUPPLIERS OR LICENSORS HAVE BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. BECAUSE SOME STATES OR JURISDICTIONS DO NOT ALLOW LIMITATION OR EXCLUSION OF CONSEQUENTIAL OR INCIDENTAL DAMAGES, THE ABOVE LIMITATION MAY NOT APPLY TO YOU.\n" + + "\n" + + "IF YOU ACQUIRED THE SOFTWARE IN JAPAN, EXCEPT FOR LIABILITY ARISING OUT OF OR IN CONNECTION WITH DEATH OR PERSONAL INJURY, FRAUDULENT MISREPRESENTATION, AND REGARDLESS OF WHETHER ANY REMEDY SET FORTH HEREIN FAILS OF ITS ESSENTIAL PURPOSE OR OTHERWISE, IN NO EVENT WILL CISCO, ITS AFFILIATES, OFFICERS, DIRECTORS, EMPLOYEES, AGENTS, SUPPLIERS AND LICENSORS BE LIABLE FOR ANY LOST REVENUE, PROFIT, OR LOST OR DAMAGED DATA, BUSINESS INTERRUPTION, LOSS OF CAPITAL, OR FOR SPECIAL, INDIRECT, CONSEQUENTIAL, INCIDENTAL, OR PUNITIVE DAMAGES HOWEVER CAUSED AND REGARDLESS OF THE THEORY OF LIABILITY OR WHETHER ARISING OUT OF THE USE OF OR INABILITY TO USE SOFTWARE OR OTHERWISE AND EVEN IF CISCO OR ANY APPROVED SOURCE OR THEIR SUPPLIERS OR LICENSORS HAVE BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.\n" + + "\n" + + "IF YOU ACQUIRED THE SOFTWARE IN EUROPE, THE MIDDLE EAST, AFRICA, ASIA OR OCEANIA, IN NO EVENT WILL CISCO, ITS AFFILIATES, OFFICERS, DIRECTORS, EMPLOYEES, AGENTS, SUPPLIERS AND LICENSORS, BE LIABLE FOR ANY LOST REVENUE, LOST PROFIT, OR LOST OR DAMAGED DATA, BUSINESS INTERRUPTION, LOSS OF CAPITAL, OR FOR SPECIAL, INDIRECT, CONSEQUENTIAL, INCIDENTAL, OR PUNITIVE DAMAGES, HOWSOEVER ARISING, INCLUDING, WITHOUT LIMITATION, IN CONTRACT, TORT (INCLUDING NEGLIGENCE) OR WHETHER ARISING OUT OF THE USE OF OR INABILITY TO USE THE SOFTWARE, EVEN IF, IN EACH CASE, CISCO, ITS AFFILIATES, OFFICERS, DIRECTORS, EMPLOYEES, AGENTS, SUPPLIERS AND LICENSORS, HAVE BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. BECAUSE SOME STATES OR JURISDICTIONS DO NOT ALLOW LIMITATION OR EXCLUSION OF CONSEQUENTIAL OR INCIDENTAL" + + " DAMAGES, THE ABOVE LIMITATION MAY NOT FULLY APPLY TO YOU. THE FOREGOING EXCLUSION SHALL NOT APPLY TO ANY LIABILITY ARISING OUT OF OR IN CONNECTION WITH: (I) DEATH OR PERSONAL INJURY, (II) FRAUDULENT MISREPRESENTATION, OR (III) CISCO'S LIABILITY IN CONNECTION WITH ANY TERMS THAT CANNOT BE EXCLUDED UNDER APPLICABLE LAW.\n" + + "\n" + + "Customer acknowledges and agrees that Cisco has set its prices and entered into the Agreement in reliance upon the disclaimers of warranty and the limitations of liability set forth herein, that the same reflect an allocation of risk between the parties (including the risk that a contract remedy may fail of its essential purpose and cause consequential loss), and that the same form an essential basis of the bargain between the parties.\n" + + "\n" + + "Controlling Law, Jurisdiction. If you acquired, by reference to the address on the purchase order accepted by the Approved Source, the Software in the United States, Latin America, or the Caribbean, the Agreement and warranties (\"Warranties\") are controlled by and construed under the laws of the State of California, United States of America, notwithstanding any conflicts of law provisions; and the state and federal courts of California shall have exclusive jurisdiction over any claim arising under the Agreement or Warranties. If you acquired the Software in Canada, unless expressly prohibited by local law, the Agreement and Warranties are controlled by and construed under the laws of the Province of Ontario, Canada, notwithstanding any conflicts of law provisions; and the courts of the " + + "Province of Ontario shall have exclusive jurisdiction over any claim arising under the Agreement or Warranties. If you acquired the Software in Europe, the Middle East, Africa, Asia or Oceania (excluding Australia), unless expressly prohibited by local law, the Agreement and Warranties are controlled by and construed under the laws of England, notwithstanding any conflicts of law provisions; and the English courts shall have exclusive jurisdiction over any " + + "claim arising under the Agreement or Warranties. In addition, if the Agreement is controlled by the laws of England, no person who is not a party to the Agreement shall be entitled to enforce or take the benefit of any of its terms under the Contracts (Rights of Third Parties) Act 1999. If you acquired the Software in Japan, unless expressly prohibited by local law, the Agreement and Warranties are controlled by and construed under the laws of Japan, notwithstanding any conflicts of law provisions; and the Tokyo District Court of Japan shall have exclusive jurisdiction over any claim arising under the Agreement or Warranties. If you acquired the Software in Australia, unless expressly prohibited by local law, the Agreement and Warranties are controlled by and construed under the laws of the " + + "State of New South Wales, Australia, notwithstanding any conflicts of law provisions; and the State and federal courts of New South Wales shall have exclusive jurisdiction over any claim arising under the Agreement or Warranties. If you acquired the Software in any other country, unless expressly prohibited by local law, the Agreement and Warranties are controlled by and construed under the laws of the State of California, United States of America, " + + "notwithstanding any conflicts of law provisions; and the state and federal courts of California shall have exclusive jurisdiction over any claim arising under the Agreement or Warranties.\n" + + "\n" + + "For all countries referred to above, the parties specifically disclaim the application of the UN Convention on Contracts for the International Sale of Goods. Notwithstanding the foregoing, either party may seek interim injunctive relief in any court of appropriate jurisdiction with respect to any alleged breach of such party's intellectual property or proprietary rights. If any portion hereof is found to be void or unenforceable, the remaining provisions of the Agreement and Warranties shall remain in full force and effect. Except as expressly provided herein, the Agreement constitutes the entire agreement between the parties with respect to the license of the Software and Documentation and supersedes any conflicting or additional terms contained in any Purchase Order or elsewhere, all of which terms are excluded. The Agreement has been written in the English language, and the parties agree that the English version will govern.\n" + + "\n" + + "Product warranty terms and other information applicable to Cisco products are available at the following URL: www.cisco.com/go/warranty\n" + + "\n" + + "Cisco and the Cisco logo are trademarks or registered trademarks of Cisco and/or its affiliates in the U.S. and other countries. To view a list of Cisco trademarks, go to this URL: www.cisco.com/go/trademarks. Third-party trademarks mentioned are the property of their respective owners. The use of the word partner does not imply a partnership relationship between Cisco and any other company. (1110R)\n" + + "\n" + + "© 1998, 2001, 2003, 2008-2014 Cisco Systems, Inc. All rights reserved."; + + @Test + public void testCompressDecompressString() throws IOException { + byte[] compressed = util.compressString(testEula); + String decompressed = util.decompressByteArary(compressed); + Assert.assertEquals(decompressed, testEula); + } +} diff --git a/vmware-base/pom.xml b/vmware-base/pom.xml index c84580f90b4b..ef03d9ada22c 100644 --- a/vmware-base/pom.xml +++ b/vmware-base/pom.xml @@ -75,5 +75,11 @@ ${cs.jaxws.version} pom + + com.cloud.com.vmware + vmware-pbm + ${cs.vmware.api.version} + compile + diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/ClusterMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/ClusterMO.java index b8afdc84cfde..c47b8f8817fd 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/ClusterMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/ClusterMO.java @@ -385,7 +385,7 @@ public boolean createVm(VirtualMachineConfigSpec vmSpec) throws Exception { } @Override - public void importVmFromOVF(String ovfFilePath, String vmName, DatastoreMO dsMo, String diskOption) throws Exception { + public void importVmFromOVF(String ovfFilePath, String vmName, DatastoreMO dsMo, String diskOption, String configurationId) throws Exception { if (s_logger.isTraceEnabled()) s_logger.trace("vCenter API trace - importVmFromOVF(). target MOR: " + _mor.getValue() + ", ovfFilePath: " + ovfFilePath + ", vmName: " + vmName + ", datastore: " + dsMo.getMor().getValue() + ", diskOption: " + diskOption); @@ -396,7 +396,7 @@ public void importVmFromOVF(String ovfFilePath, String vmName, DatastoreMO dsMo, if (s_logger.isTraceEnabled()) s_logger.trace("vCenter API trace - importVmFromOVF(). resource pool: " + morRp.getValue()); - HypervisorHostHelper.importVmFromOVF(this, ovfFilePath, vmName, dsMo, diskOption, morRp, null); + HypervisorHostHelper.importVmFromOVF(this, ovfFilePath, vmName, dsMo, diskOption, morRp, null, configurationId); if (s_logger.isTraceEnabled()) s_logger.trace("vCenter API trace - importVmFromOVF() done"); @@ -422,7 +422,7 @@ public boolean createBlankVm(String vmName, String vmInternalCSName, int cpuCoun } @Override - public ManagedObjectReference mountDatastore(boolean vmfsDatastore, String poolHostAddress, int poolHostPort, String poolPath, String poolUuid) throws Exception { + public ManagedObjectReference mountDatastore(boolean vmfsDatastore, String poolHostAddress, int poolHostPort, String poolPath, String poolUuid, boolean createBaseFolder) throws Exception { if (s_logger.isTraceEnabled()) s_logger.trace("vCenter API trace - mountDatastore(). target MOR: " + _mor.getValue() + ", vmfs: " + vmfsDatastore + ", poolHost: " + poolHostAddress + @@ -434,7 +434,7 @@ public ManagedObjectReference mountDatastore(boolean vmfsDatastore, String poolH if (hosts != null && hosts.size() > 0) { for (ManagedObjectReference morHost : hosts) { HostMO hostMo = new HostMO(_context, morHost); - morDs = hostMo.mountDatastore(vmfsDatastore, poolHostAddress, poolHostPort, poolPath, poolUuid); + morDs = hostMo.mountDatastore(vmfsDatastore, poolHostAddress, poolHostPort, poolPath, poolUuid, true); if (morDsFirst == null) morDsFirst = morDs; diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatacenterMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatacenterMO.java index b0b91fb7d5b2..af89757d521f 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatacenterMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatacenterMO.java @@ -186,6 +186,20 @@ public ManagedObjectReference findDatastore(String name) throws Exception { return null; } + public ManagedObjectReference listDatastore(String name) throws Exception { + assert (name != null); + + List ocs = getDatastorePropertiesOnDatacenter(new String[] {"name"}); + if (ocs != null) { + for (ObjectContent oc : ocs) { + if (oc.getPropSet().get(0).getVal().toString().equals(name)) { + return oc.getObj(); + } + } + } + return null; + } + public ManagedObjectReference findHost(String name) throws Exception { List ocs = getHostPropertiesOnDatacenterHostFolder(new String[] {"name"}); diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatastoreMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatastoreMO.java index 1e065a15daa6..804af6286d10 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatastoreMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatastoreMO.java @@ -16,11 +16,10 @@ // under the License. package com.cloud.hypervisor.vmware.mo; -import java.util.ArrayList; -import java.util.List; - -import org.apache.log4j.Logger; - +import com.cloud.exception.CloudException; +import com.cloud.hypervisor.vmware.util.VmwareContext; +import com.cloud.utils.Pair; +import com.vmware.pbm.PbmProfile; import com.vmware.vim25.DatastoreHostMount; import com.vmware.vim25.DatastoreSummary; import com.vmware.vim25.FileInfo; @@ -35,10 +34,10 @@ import com.vmware.vim25.PropertySpec; import com.vmware.vim25.SelectionSpec; import com.vmware.vim25.TraversalSpec; +import org.apache.log4j.Logger; -import com.cloud.exception.CloudException; -import com.cloud.hypervisor.vmware.util.VmwareContext; -import com.cloud.utils.Pair; +import java.util.ArrayList; +import java.util.List; import static com.cloud.utils.NumbersUtil.toHumanReadableSize; @@ -64,7 +63,7 @@ public String getName() throws Exception { return _name; } - public DatastoreSummary getSummary() throws Exception { + public DatastoreSummary getDatastoreSummary() throws Exception { return (DatastoreSummary)_context.getVimClient().getDynamicProperty(_mor, "summary"); } @@ -137,6 +136,16 @@ public void makeDirectory(String path, ManagedObjectReference morDc) throws Exce fullPath = String.format("[%s] %s", datastoreName, path); _context.getService().makeDirectory(morFileManager, fullPath, morDc, true); + + int retry = 2; + for (int i = 0; i < retry; i++) { + DatastoreFile datastoreFile = new DatastoreFile(fullPath); + if (!folderExists(String.format("[%s]", datastoreName), datastoreFile.getFileName())) { + _context.getService().makeDirectory(morFileManager, fullPath, morDc, true); + } else { + return; + } + } } String getDatastoreRootPath() throws Exception { @@ -260,6 +269,17 @@ public boolean moveDatastoreFile(String srcFilePath, ManagedObjectReference morS if (!DatastoreFile.isFullDatastorePath(destFullPath)) destFullPath = String.format("[%s] %s", destDsName, destFilePath); + DatastoreMO srcDsMo = new DatastoreMO(_context, morDestDs); + try { + if (!srcDsMo.fileExists(srcFullPath)) { + s_logger.error(String.format("Cannot move file to destination datastore due to file %s does not exists", srcFullPath)); + return false; + } + } catch (Exception e) { + s_logger.error(String.format("Cannot move file to destination datastore due to file %s due to exeception %s", srcFullPath, e.getMessage())); + return false; + } + ManagedObjectReference morTask = _context.getService().moveDatastoreFileTask(morFileManager, srcFullPath, morSrcDc, destFullPath, morDestDc, forceOverwrite); boolean result = _context.getVimClient().waitForTask(morTask); @@ -267,52 +287,11 @@ public boolean moveDatastoreFile(String srcFilePath, ManagedObjectReference morS _context.waitForTaskProgressDone(morTask); return true; } else { - s_logger.error("VMware moveDatgastoreFile_Task failed due to " + TaskMO.getTaskFailureInfo(_context, morTask)); + s_logger.error("VMware moveDatastoreFile_Task failed due to " + TaskMO.getTaskFailureInfo(_context, morTask)); } return false; } - public String[] getVmdkFileChain(String rootVmdkDatastoreFullPath) throws Exception { - Pair dcPair = getOwnerDatacenter(); - - List files = new ArrayList<>(); - files.add(rootVmdkDatastoreFullPath); - - String currentVmdkFullPath = rootVmdkDatastoreFullPath; - while (true) { - String url = getContext().composeDatastoreBrowseUrl(dcPair.second(), currentVmdkFullPath); - byte[] content = getContext().getResourceContent(url); - if (content == null || content.length == 0) - break; - - VmdkFileDescriptor descriptor = new VmdkFileDescriptor(); - descriptor.parse(content); - - String parentFileName = descriptor.getParentFileName(); - if (parentFileName == null) - break; - - if (parentFileName.startsWith("/")) { - // when parent file is not at the same directory as it is, assume it is at parent directory - // this is only valid in Apache CloudStack primary storage deployment - DatastoreFile dsFile = new DatastoreFile(currentVmdkFullPath); - String dir = dsFile.getDir(); - if (dir != null && dir.lastIndexOf('/') > 0) - dir = dir.substring(0, dir.lastIndexOf('/')); - else - dir = ""; - - currentVmdkFullPath = new DatastoreFile(dsFile.getDatastoreName(), dir, parentFileName.substring(parentFileName.lastIndexOf('/') + 1)).getPath(); - files.add(currentVmdkFullPath); - } else { - currentVmdkFullPath = DatastoreFile.getCompanionDatastorePath(currentVmdkFullPath, parentFileName); - files.add(currentVmdkFullPath); - } - } - - return files.toArray(new String[0]); - } - @Deprecated public String[] listDirContent(String path) throws Exception { String fullPath = path; @@ -427,6 +406,8 @@ public String searchFileInSubFolders(String fileName, boolean caseInsensitive, S s_logger.info("Found file " + fileName + " in datastore at " + absoluteFileName); if (parentFolderPath.endsWith("]")) absoluteFileName += " "; + else if (!parentFolderPath.endsWith("/")) + absoluteFileName +="/"; absoluteFileName += fi.getPath(); if(isValidCloudStackFolderPath(parentFolderPath, searchExcludedFolders)) { return absoluteFileName; @@ -465,4 +446,19 @@ public boolean isAccessibleToHost(String hostValue) throws Exception { } return isAccessible; } + + public boolean isDatastoreStoragePolicyComplaint(String storagePolicyId) throws Exception { + PbmProfileManagerMO profMgrMo = new PbmProfileManagerMO(_context); + PbmProfile profile = profMgrMo.getStorageProfile(storagePolicyId); + + PbmPlacementSolverMO placementSolverMo = new PbmPlacementSolverMO(_context); + boolean isDatastoreCompatible = placementSolverMo.isDatastoreCompatibleWithStorageProfile(_mor, profile); + + return isDatastoreCompatible; + } + + public String getDatastoreType() throws Exception { + DatastoreSummary summary = _context.getVimClient().getDynamicProperty(getMor(), "summary"); + return summary.getType(); + } } diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HostDatastoreSystemMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HostDatastoreSystemMO.java index f38f610e145d..30798e31e194 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HostDatastoreSystemMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HostDatastoreSystemMO.java @@ -16,11 +16,7 @@ // under the License. package com.cloud.hypervisor.vmware.mo; -import java.net.URI; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - +import com.cloud.hypervisor.vmware.util.VmwareContext; import com.vmware.vim25.CustomFieldStringValue; import com.vmware.vim25.DatastoreInfo; import com.vmware.vim25.DynamicProperty; @@ -35,12 +31,18 @@ import com.vmware.vim25.ObjectSpec; import com.vmware.vim25.PropertyFilterSpec; import com.vmware.vim25.PropertySpec; +import com.vmware.vim25.RetrieveOptions; +import com.vmware.vim25.RetrieveResult; +import com.vmware.vim25.SelectionSpec; import com.vmware.vim25.TraversalSpec; import com.vmware.vim25.VmfsDatastoreCreateSpec; import com.vmware.vim25.VmfsDatastoreExpandSpec; import com.vmware.vim25.VmfsDatastoreOption; -import com.cloud.hypervisor.vmware.util.VmwareContext; +import java.net.URI; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; public class HostDatastoreSystemMO extends BaseMO { @@ -53,6 +55,14 @@ public HostDatastoreSystemMO(VmwareContext context, String morType, String morVa } public ManagedObjectReference findDatastore(String name) throws Exception { + ManagedObjectReference morDatastore = findSpecificDatastore(name); + if (morDatastore == null) { + morDatastore = findDatastoreCluster(name); + } + return morDatastore; + } + + public ManagedObjectReference findSpecificDatastore(String name) throws Exception { // added Apache CloudStack specific name convention, we will use custom field "cloud.uuid" as datastore name as well CustomFieldsManagerMO cfmMo = new CustomFieldsManagerMO(_context, _context.getServiceContent().getCustomFieldsManager()); int key = cfmMo.getCustomFieldKey("Datastore", CustomFieldConstants.CLOUD_UUID); @@ -79,6 +89,33 @@ public ManagedObjectReference findDatastore(String name) throws Exception { return null; } + public ManagedObjectReference findDatastoreCluster(String name) throws Exception { + // added Apache CloudStack specific name convention, we will use custom field "cloud.uuid" as datastore name as well + CustomFieldsManagerMO cfmMo = new CustomFieldsManagerMO(_context, _context.getServiceContent().getCustomFieldsManager()); + int key = cfmMo.getCustomFieldKey("StoragePod", CustomFieldConstants.CLOUD_UUID); + assert (key != 0); + + List ocs = getDatastoreClusterPropertiesOnHostDatastoreSystem(new String[] {"name", String.format("value[%d]", key)}); + if (ocs != null) { + for (ObjectContent oc : ocs) { + if (oc.getPropSet().get(0).getVal().equals(name)) + return oc.getObj(); + + if (oc.getPropSet().size() > 1) { + DynamicProperty prop = oc.getPropSet().get(1); + if (prop != null && prop.getVal() != null) { + if (prop.getVal() instanceof CustomFieldStringValue) { + String val = ((CustomFieldStringValue)prop.getVal()).getValue(); + if (val.equalsIgnoreCase(name)) + return oc.getObj(); + } + } + } + } + } + return null; + } + public List queryUnresolvedVmfsVolumes() throws Exception { return _context.getService().queryUnresolvedVmfsVolumes(_mor); } @@ -251,4 +288,90 @@ public List getDatastorePropertiesOnHostDatastoreSystem(String[] return _context.getService().retrieveProperties(_context.getPropertyCollector(), pfSpecArr); } + + public List getDatastoreClusterPropertiesOnHostDatastoreSystem(String[] propertyPaths) throws Exception { + ManagedObjectReference retVal = null; + // Create Property Spec + PropertySpec propertySpec = new PropertySpec(); + propertySpec.setAll(Boolean.FALSE); + propertySpec.setType("StoragePod"); + propertySpec.getPathSet().addAll(Arrays.asList(propertyPaths)); + + // Now create Object Spec + ObjectSpec objectSpec = new ObjectSpec(); + objectSpec.setObj(getContext().getRootFolder()); + objectSpec.setSkip(Boolean.TRUE); + objectSpec.getSelectSet().addAll( + Arrays.asList(getStorageTraversalSpec())); + + // Create PropertyFilterSpec using the PropertySpec and ObjectPec + // created above. + PropertyFilterSpec propertyFilterSpec = new PropertyFilterSpec(); + propertyFilterSpec.getPropSet().add(propertySpec); + propertyFilterSpec.getObjectSet().add(objectSpec); + + List listpfs = new ArrayList(); + listpfs.add(propertyFilterSpec); + return retrievePropertiesAllObjects(listpfs); + } + + private SelectionSpec[] getStorageTraversalSpec() { + // create a traversal spec that start from root folder + + SelectionSpec ssFolders = new SelectionSpec(); + ssFolders.setName("visitFolders"); + + TraversalSpec datacenterSpec = new TraversalSpec(); + datacenterSpec.setName("dcTodf"); + datacenterSpec.setType("Datacenter"); + datacenterSpec.setPath("datastoreFolder"); + datacenterSpec.setSkip(Boolean.FALSE); + datacenterSpec.getSelectSet().add(ssFolders); + + TraversalSpec visitFolder = new TraversalSpec(); + visitFolder.setType("Folder"); + visitFolder.setName("visitFolders"); + visitFolder.setPath("childEntity"); + visitFolder.setSkip(Boolean.FALSE); + + List ssSpecList = new ArrayList(); + ssSpecList.add(datacenterSpec); + ssSpecList.add(ssFolders); + + visitFolder.getSelectSet().addAll(ssSpecList); + return (new SelectionSpec[]{visitFolder}); + } + + private List retrievePropertiesAllObjects( + List listpfs) throws Exception { + + RetrieveOptions propObjectRetrieveOpts = new RetrieveOptions(); + + List listobjcontent = new ArrayList(); + + RetrieveResult rslts = + getContext().getService().retrievePropertiesEx(getContext().getServiceContent().getPropertyCollector(), listpfs, + propObjectRetrieveOpts); + if (rslts != null && rslts.getObjects() != null + && !rslts.getObjects().isEmpty()) { + listobjcontent.addAll(rslts.getObjects()); + } + String token = null; + if (rslts != null && rslts.getToken() != null) { + token = rslts.getToken(); + } + while (token != null && !token.isEmpty()) { + rslts = + getContext().getService().continueRetrievePropertiesEx(getContext().getServiceContent().getPropertyCollector(), token); + token = null; + if (rslts != null) { + token = rslts.getToken(); + if (rslts.getObjects() != null && !rslts.getObjects().isEmpty()) { + listobjcontent.addAll(rslts.getObjects()); + } + } + } + return listobjcontent; + } + } diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HostMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HostMO.java index 7877db980f46..68f87cad3f8a 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HostMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HostMO.java @@ -16,15 +16,9 @@ // under the License. package com.cloud.hypervisor.vmware.mo; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.regex.Pattern; - -import org.apache.log4j.Logger; - +import com.cloud.hypervisor.vmware.util.VmwareContext; +import com.cloud.hypervisor.vmware.util.VmwareHelper; +import com.cloud.utils.Pair; import com.google.gson.Gson; import com.vmware.vim25.AboutInfo; import com.vmware.vim25.AlreadyExistsFaultMsg; @@ -63,9 +57,14 @@ import com.vmware.vim25.TraversalSpec; import com.vmware.vim25.VirtualMachineConfigSpec; import com.vmware.vim25.VirtualNicManagerNetConfig; -import com.cloud.hypervisor.vmware.util.VmwareContext; -import com.cloud.hypervisor.vmware.util.VmwareHelper; -import com.cloud.utils.Pair; +import org.apache.log4j.Logger; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; public class HostMO extends BaseMO implements VmwareHypervisorHost { private static final Logger s_logger = Logger.getLogger(HostMO.class); @@ -757,7 +756,7 @@ public List> getLocalDatastoreOnHost() thro return dsList; } - public void importVmFromOVF(String ovfFilePath, String vmName, String datastoreName, String diskOption) throws Exception { + public void importVmFromOVF(String ovfFilePath, String vmName, String datastoreName, String diskOption, String configurationId) throws Exception { if (s_logger.isTraceEnabled()) s_logger.trace("vCenter API trace - importVmFromOVF(). target MOR: " + _mor.getValue() + ", ovfFilePath: " + ovfFilePath + ", vmName: " + vmName + ",datastoreName: " + datastoreName + ", diskOption: " + diskOption); @@ -766,19 +765,19 @@ public void importVmFromOVF(String ovfFilePath, String vmName, String datastoreN if (dsMo == null) throw new Exception("Invalid datastore name: " + datastoreName); - importVmFromOVF(ovfFilePath, vmName, dsMo, diskOption); + importVmFromOVF(ovfFilePath, vmName, dsMo, diskOption, configurationId); if (s_logger.isTraceEnabled()) s_logger.trace("vCenter API trace - importVmFromOVF() done"); } @Override - public void importVmFromOVF(String ovfFilePath, String vmName, DatastoreMO dsMo, String diskOption) throws Exception { + public void importVmFromOVF(String ovfFilePath, String vmName, DatastoreMO dsMo, String diskOption, String configurationId) throws Exception { ManagedObjectReference morRp = getHyperHostOwnerResourcePool(); assert (morRp != null); - HypervisorHostHelper.importVmFromOVF(this, ovfFilePath, vmName, dsMo, diskOption, morRp, _mor); + HypervisorHostHelper.importVmFromOVF(this, ovfFilePath, vmName, dsMo, diskOption, morRp, _mor, configurationId); } @Override @@ -833,12 +832,13 @@ public ManagedObjectReference getExistingDataStoreOnHost(boolean vmfsDatastore, } @Override - public ManagedObjectReference mountDatastore(boolean vmfsDatastore, String poolHostAddress, int poolHostPort, String poolPath, String poolUuid) throws Exception { + public ManagedObjectReference mountDatastore(boolean vmfsDatastore, String poolHostAddress, int poolHostPort, String poolPath, String poolUuid, boolean createBaseFolder) throws Exception { if (s_logger.isTraceEnabled()) s_logger.trace("vCenter API trace - mountDatastore(). target MOR: " + _mor.getValue() + ", vmfs: " + vmfsDatastore + ", poolHost: " + poolHostAddress + ", poolHostPort: " + poolHostPort + ", poolPath: " + poolPath + ", poolUuid: " + poolUuid); + DatastoreMO dsMo = null; HostDatastoreSystemMO hostDatastoreSystemMo = getHostDatastoreSystemMO(); ManagedObjectReference morDatastore = hostDatastoreSystemMo.findDatastore(poolUuid); if (morDatastore == null) { @@ -865,22 +865,30 @@ public ManagedObjectReference mountDatastore(boolean vmfsDatastore, String poolH s_logger.trace("vCenter API trace - mountDatastore() done(failed)"); throw new Exception(msg); } + dsMo = new DatastoreMO(_context, morDatastore); } else { morDatastore = _context.getDatastoreMorByPath(poolPath); if (morDatastore == null) { - String msg = "Unable to create VMFS datastore. host: " + poolHostAddress + ", port: " + poolHostPort + ", path: " + poolPath + ", uuid: " + poolUuid; - s_logger.error(msg); + morDatastore = findDatastore(_context.getDatastoreNameFromPath(poolPath)); + if (morDatastore == null) { + String msg = "Unable to create VMFS datastore. host: " + poolHostAddress + ", port: " + poolHostPort + ", path: " + poolPath + ", uuid: " + poolUuid; + s_logger.error(msg); - if (s_logger.isTraceEnabled()) - s_logger.trace("vCenter API trace - mountDatastore() done(failed)"); - throw new Exception(msg); + if (s_logger.isTraceEnabled()) + s_logger.trace("vCenter API trace - mountDatastore() done(failed)"); + throw new Exception(msg); + } } - DatastoreMO dsMo = new DatastoreMO(_context, morDatastore); + dsMo = new DatastoreMO(_context, morDatastore); dsMo.setCustomFieldValue(CustomFieldConstants.CLOUD_UUID, poolUuid); } } + if (dsMo != null && !"StoragePod".equals(morDatastore.getType()) && createBaseFolder) { + HypervisorHostHelper.createBaseFolderInDatastore(dsMo, this); + } + if (s_logger.isTraceEnabled()) s_logger.trace("vCenter API trace - mountDatastore() done(successfully)"); diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java index c3143ae99541..9345da92d525 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java @@ -37,6 +37,17 @@ import javax.xml.transform.dom.DOMSource; import javax.xml.transform.stream.StreamResult; +import com.vmware.vim25.ConcurrentAccessFaultMsg; +import com.vmware.vim25.DuplicateNameFaultMsg; +import com.vmware.vim25.FileFaultFaultMsg; +import com.vmware.vim25.InsufficientResourcesFaultFaultMsg; +import com.vmware.vim25.InvalidDatastoreFaultMsg; +import com.vmware.vim25.InvalidNameFaultMsg; +import com.vmware.vim25.InvalidStateFaultMsg; +import com.vmware.vim25.OutOfBoundsFaultMsg; +import com.vmware.vim25.RuntimeFaultFaultMsg; +import com.vmware.vim25.TaskInProgressFaultMsg; +import com.vmware.vim25.VmConfigFaultFaultMsg; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; @@ -54,6 +65,7 @@ import com.cloud.hypervisor.vmware.util.VmwareHelper; import com.cloud.network.Networks.BroadcastDomainType; import com.cloud.offering.NetworkOffering; +import com.cloud.storage.Storage.StoragePoolType; import com.cloud.utils.ActionDelegate; import com.cloud.utils.NumbersUtil; import com.cloud.utils.Pair; @@ -138,6 +150,8 @@ public class HypervisorHostHelper { private static final String UNTAGGED_VLAN_NAME = "untagged"; private static final String VMDK_PACK_DIR = "ova"; private static final String OVA_OPTION_KEY_BOOTDISK = "cloud.ova.bootdisk"; + public static final String VSPHERE_DATASTORE_BASE_FOLDER = "fcd"; + public static final String VSPHERE_DATASTORE_HIDDEN_FOLDER = ".hidden"; public static VirtualMachineMO findVmFromObjectContent(VmwareContext context, ObjectContent[] ocs, String name, String instanceNameCustomField) { @@ -1465,7 +1479,7 @@ public static boolean createBlankVm(VmwareHypervisorHost host, String vmName, St if (vmInternalCSName == null) vmInternalCSName = vmName; - VmwareHelper.setBasicVmConfig(vmConfig, cpuCount, cpuSpeedMHz, cpuReservedMHz, memoryMB, memoryReserveMB, guestOsIdentifier, limitCpuUse); + VmwareHelper.setBasicVmConfig(vmConfig, cpuCount, cpuSpeedMHz, cpuReservedMHz, memoryMB, memoryReserveMB, guestOsIdentifier, limitCpuUse, false); String recommendedController = host.getRecommendedDiskController(guestOsIdentifier); String newRootDiskController = controllerInfo.first(); @@ -1560,20 +1574,35 @@ public static boolean createBlankVm(VmwareHypervisorHost host, String vmName, St * - If the cluster hardware version is not set, check datacenter hardware version. If it is set, then it is set to vmConfig * - In case both cluster and datacenter hardware version are not set, hardware version is not set to vmConfig */ - protected static void setVMHardwareVersion(VirtualMachineConfigSpec vmConfig, ClusterMO clusterMO, DatacenterMO datacenterMO) throws Exception { + public static void setVMHardwareVersion(VirtualMachineConfigSpec vmConfig, ClusterMO clusterMO, DatacenterMO datacenterMO) throws Exception { + String version = getNewVMHardwareVersion(clusterMO, datacenterMO); + if (StringUtils.isNotBlank(version)) { + vmConfig.setVersion(version); + } + } + + /** + * Return the VM hardware version based on the information retrieved by the cluster and datacenter: + * - If the cluster hardware version is set, then return this hardware version + * - If the cluster hardware version is not set, check datacenter hardware version. If it is set, then return it + * - In case both cluster and datacenter hardware version are not set, return null + */ + public static String getNewVMHardwareVersion(ClusterMO clusterMO, DatacenterMO datacenterMO) throws Exception { + String version = null; ClusterConfigInfoEx clusterConfigInfo = clusterMO != null ? clusterMO.getClusterConfigInfo() : null; String clusterHardwareVersion = clusterConfigInfo != null ? clusterConfigInfo.getDefaultHardwareVersionKey() : null; if (StringUtils.isNotBlank(clusterHardwareVersion)) { s_logger.debug("Cluster hardware version found: " + clusterHardwareVersion + ". Creating VM with this hardware version"); - vmConfig.setVersion(clusterHardwareVersion); + version = clusterHardwareVersion; } else { DatacenterConfigInfo datacenterConfigInfo = datacenterMO != null ? datacenterMO.getDatacenterConfigInfo() : null; String datacenterHardwareVersion = datacenterConfigInfo != null ? datacenterConfigInfo.getDefaultHardwareVersionKey() : null; if (StringUtils.isNotBlank(datacenterHardwareVersion)) { s_logger.debug("Datacenter hardware version found: " + datacenterHardwareVersion + ". Creating VM with this hardware version"); - vmConfig.setVersion(datacenterHardwareVersion); + version = datacenterHardwareVersion; } } + return version; } private static VirtualDeviceConfigSpec getControllerSpec(String diskController, int busNum) { @@ -1612,6 +1641,9 @@ public static VirtualMachineMO createWorkerVM(VmwareHypervisorHost hyperHost, Da if (morCluster != null) hyperHost = new ClusterMO(hyperHost.getContext(), morCluster); + if (dsMo.getDatastoreType().equalsIgnoreCase("VVOL") && !vmName.startsWith(CustomFieldConstants.CLOUD_UUID)) { + vmName = CustomFieldConstants.CLOUD_UUID + "-" + vmName; + } VirtualMachineMO workingVM = null; VirtualMachineConfigSpec vmConfig = new VirtualMachineConfigSpec(); vmConfig.setName(vmName); @@ -1704,6 +1736,11 @@ public static String resolveHostNameInUrl(DatacenterMO dcMo, String url) { return url; } + /** + * removes the NetworkSection element from the {ovfString} if it is an ovf xml file + * @param ovfString input string + * @return like the input string but if xml elements by name {NetworkSection} removed + */ public static String removeOVFNetwork(final String ovfString) { if (ovfString == null || ovfString.isEmpty()) { return ovfString; @@ -1738,8 +1775,12 @@ public static String removeOVFNetwork(final String ovfString) { return ovfString; } + /** + * deploys a new VM from a ovf spec. It ignores network, defaults locale to 'US' + * @throws Exception shoud be a VmwareResourceException + */ public static void importVmFromOVF(VmwareHypervisorHost host, String ovfFilePath, String vmName, DatastoreMO dsMo, String diskOption, ManagedObjectReference morRp, - ManagedObjectReference morHost) throws Exception { + ManagedObjectReference morHost, String configurationId) throws CloudRuntimeException, IOException { assert (morRp != null); @@ -1747,24 +1788,34 @@ public static void importVmFromOVF(VmwareHypervisorHost host, String ovfFilePath importSpecParams.setHostSystem(morHost); importSpecParams.setLocale("US"); importSpecParams.setEntityName(vmName); - importSpecParams.setDeploymentOption(""); + String deploymentOption = StringUtils.isNotBlank(configurationId) ? configurationId : ""; + importSpecParams.setDeploymentOption(deploymentOption); importSpecParams.setDiskProvisioning(diskOption); // diskOption: thin, thick, etc String ovfDescriptor = removeOVFNetwork(HttpNfcLeaseMO.readOvfContent(ovfFilePath)); VmwareContext context = host.getContext(); - OvfCreateImportSpecResult ovfImportResult = - context.getService().createImportSpec(context.getServiceContent().getOvfManager(), ovfDescriptor, morRp, dsMo.getMor(), importSpecParams); - + OvfCreateImportSpecResult ovfImportResult = null; + try { + ovfImportResult = context.getService().createImportSpec(context.getServiceContent().getOvfManager(), ovfDescriptor, morRp, dsMo.getMor(), importSpecParams); + } catch (ConcurrentAccessFaultMsg + | FileFaultFaultMsg + | InvalidDatastoreFaultMsg + | InvalidStateFaultMsg + | RuntimeFaultFaultMsg + | TaskInProgressFaultMsg + | VmConfigFaultFaultMsg error) { + throw new CloudRuntimeException("ImportSpec creation failed", error); + } if (ovfImportResult == null) { String msg = "createImportSpec() failed. ovfFilePath: " + ovfFilePath + ", vmName: " + vmName + ", diskOption: " + diskOption; s_logger.error(msg); - throw new Exception(msg); + throw new CloudRuntimeException(msg); } if(!ovfImportResult.getError().isEmpty()) { for (LocalizedMethodFault fault : ovfImportResult.getError()) { s_logger.error("createImportSpec error: " + fault.getLocalizedMessage()); } - throw new CloudException("Failed to create an import spec from " + ovfFilePath + ". Check log for details."); + throw new CloudRuntimeException("Failed to create an import spec from " + ovfFilePath + ". Check log for details."); } if (!ovfImportResult.getWarning().isEmpty()) { @@ -1773,22 +1824,55 @@ public static void importVmFromOVF(VmwareHypervisorHost host, String ovfFilePath } } - DatacenterMO dcMo = new DatacenterMO(context, host.getHyperHostDatacenter()); - ManagedObjectReference morLease = context.getService().importVApp(morRp, ovfImportResult.getImportSpec(), dcMo.getVmFolder(), morHost); + DatacenterMO dcMo = null; + try { + dcMo = new DatacenterMO(context, host.getHyperHostDatacenter()); + } catch (Exception e) { + throw new CloudRuntimeException(String.format("no datacenter for host '%s' available in context", context.getServerAddress()), e); + } + ManagedObjectReference folderMO = null; + try { + folderMO = dcMo.getVmFolder(); + } catch (Exception e) { + throw new CloudRuntimeException("no management handle for VmFolder", e); + } + ManagedObjectReference morLease = null; + try { + morLease = context.getService().importVApp(morRp, ovfImportResult.getImportSpec(), folderMO, morHost); + } catch (DuplicateNameFaultMsg + | FileFaultFaultMsg + | InsufficientResourcesFaultFaultMsg + | InvalidDatastoreFaultMsg + | InvalidNameFaultMsg + | OutOfBoundsFaultMsg + | RuntimeFaultFaultMsg + | VmConfigFaultFaultMsg fault) { + throw new CloudRuntimeException("import vApp failed",fault); + } if (morLease == null) { String msg = "importVApp() failed. ovfFilePath: " + ovfFilePath + ", vmName: " + vmName + ", diskOption: " + diskOption; s_logger.error(msg); - throw new Exception(msg); + throw new CloudRuntimeException(msg); } boolean importSuccess = true; final HttpNfcLeaseMO leaseMo = new HttpNfcLeaseMO(context, morLease); - HttpNfcLeaseState state = leaseMo.waitState(new HttpNfcLeaseState[] {HttpNfcLeaseState.READY, HttpNfcLeaseState.ERROR}); + HttpNfcLeaseState state = null; + try { + state = leaseMo.waitState(new HttpNfcLeaseState[] {HttpNfcLeaseState.READY, HttpNfcLeaseState.ERROR}); + } catch (Exception e) { + throw new CloudRuntimeException("exception while waiting for leaseMO", e); + } try { if (state == HttpNfcLeaseState.READY) { final long totalBytes = HttpNfcLeaseMO.calcTotalBytes(ovfImportResult); File ovfFile = new File(ovfFilePath); - HttpNfcLeaseInfo httpNfcLeaseInfo = leaseMo.getLeaseInfo(); + HttpNfcLeaseInfo httpNfcLeaseInfo = null; + try { + httpNfcLeaseInfo = leaseMo.getLeaseInfo(); + } catch (Exception e) { + throw new CloudRuntimeException("error waiting for lease info", e); + } List deviceUrls = httpNfcLeaseInfo.getDeviceUrl(); long bytesAlreadyWritten = 0; @@ -1799,6 +1883,7 @@ public static void importVmFromOVF(VmwareHypervisorHost host, String ovfFilePath for (OvfFileItem ovfFileItem : ovfImportResult.getFileItem()) { if (deviceKey.equals(ovfFileItem.getDeviceId())) { String absoluteFile = ovfFile.getParent() + File.separator + ovfFileItem.getPath(); + s_logger.info("Uploading file: " + absoluteFile); File f = new File(absoluteFile); if (f.exists()){ String urlToPost = deviceUrl.getUrl(); @@ -1818,31 +1903,44 @@ public void action(Long param) { String erroMsg = "File upload task failed to complete due to: " + e.getMessage(); s_logger.error(erroMsg); importSuccess = false; // Set flag to cleanup the stale template left due to failed import operation, if any - throw new Exception(erroMsg, e); + throw new CloudRuntimeException(erroMsg, e); } catch (Throwable th) { String errorMsg = "throwable caught during file upload task: " + th.getMessage(); s_logger.error(errorMsg); importSuccess = false; // Set flag to cleanup the stale template left due to failed import operation, if any - throw new Exception(errorMsg, th); + throw new CloudRuntimeException(errorMsg, th); } finally { progressReporter.close(); } if (bytesAlreadyWritten == totalBytes) { - leaseMo.updateLeaseProgress(100); + try { + leaseMo.updateLeaseProgress(100); + } catch (Exception e) { + throw new CloudRuntimeException("error while waiting for lease update", e); + } } } else if (state == HttpNfcLeaseState.ERROR) { - LocalizedMethodFault error = leaseMo.getLeaseError(); + LocalizedMethodFault error = null; + try { + error = leaseMo.getLeaseError(); + } catch (Exception e) { + throw new CloudRuntimeException("error getting lease error", e); + } MethodFault fault = error.getFault(); String erroMsg = "Object creation on vCenter failed due to: Exception: " + fault.getClass().getName() + ", message: " + error.getLocalizedMessage(); s_logger.error(erroMsg); - throw new Exception(erroMsg); + throw new CloudRuntimeException(erroMsg); } } finally { - if (!importSuccess) { - s_logger.error("Aborting the lease on " + vmName + " after import operation failed."); - leaseMo.abortLease(); - } else { - leaseMo.completeLease(); + try { + if (!importSuccess) { + s_logger.error("Aborting the lease on " + vmName + " after import operation failed."); + leaseMo.abortLease(); + } else { + leaseMo.completeLease(); + } + } catch (Exception e) { + throw new CloudRuntimeException("error completing lease", e); } } } @@ -2077,4 +2175,29 @@ public static boolean isIdeController(String controller) { return DiskControllerType.getType(controller) == DiskControllerType.ide; } + public static void createBaseFolder(DatastoreMO dsMo, VmwareHypervisorHost hyperHost, StoragePoolType poolType) throws Exception { + if (poolType != null && poolType == StoragePoolType.DatastoreCluster) { + StoragepodMO storagepodMO = new StoragepodMO(hyperHost.getContext(), dsMo.getMor()); + List datastoresInCluster = storagepodMO.getDatastoresInDatastoreCluster(); + for (ManagedObjectReference datastore : datastoresInCluster) { + DatastoreMO childDsMo = new DatastoreMO(hyperHost.getContext(), datastore); + createBaseFolderInDatastore(childDsMo, hyperHost); + } + } else { + createBaseFolderInDatastore(dsMo, hyperHost); + } + } + + public static void createBaseFolderInDatastore(DatastoreMO dsMo, VmwareHypervisorHost hyperHost) throws Exception { + String dsPath = String.format("[%s]", dsMo.getName()); + String folderPath = String.format("[%s] %s", dsMo.getName(), VSPHERE_DATASTORE_BASE_FOLDER); + String hiddenFolderPath = String.format("%s/%s", folderPath, VSPHERE_DATASTORE_HIDDEN_FOLDER); + + if (!dsMo.folderExists(dsPath, VSPHERE_DATASTORE_BASE_FOLDER)) { + s_logger.info(String.format("vSphere datastore base folder: %s does not exist, now creating on datastore: %s", VSPHERE_DATASTORE_BASE_FOLDER, dsMo.getName())); + dsMo.makeDirectory(folderPath, hyperHost.getHyperHostDatacenter()); + // Adding another directory so vCentre doesn't remove the fcd directory when it's empty + dsMo.makeDirectory(hiddenFolderPath, hyperHost.getHyperHostDatacenter()); + } + } } diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/PbmPlacementSolverMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/PbmPlacementSolverMO.java new file mode 100644 index 000000000000..3eb909fc31bd --- /dev/null +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/PbmPlacementSolverMO.java @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.hypervisor.vmware.mo; + +import com.cloud.hypervisor.vmware.util.VmwareContext; +import com.vmware.pbm.PbmPlacementCompatibilityResult; +import com.vmware.pbm.PbmPlacementHub; +import com.vmware.pbm.PbmProfile; +import com.vmware.pbm.PbmProfileId; +import com.vmware.vim25.ManagedObjectReference; +import org.apache.commons.collections.CollectionUtils; +import org.apache.log4j.Logger; + +import java.util.ArrayList; +import java.util.List; + +public class PbmPlacementSolverMO extends BaseMO { + + private static final Logger LOGGER = Logger.getLogger(PbmPlacementSolverMO.class); + + public PbmPlacementSolverMO (VmwareContext context) { + super(context, context.getPbmServiceContent().getPlacementSolver()); + } + + public PbmPlacementSolverMO(VmwareContext context, ManagedObjectReference morPlacementSolver) { + super(context, morPlacementSolver); + } + + public PbmPlacementSolverMO(VmwareContext context, String morType, String morValue) { + super(context, morType, morValue); + } + + public boolean isDatastoreCompatibleWithStorageProfile(ManagedObjectReference dsMor, PbmProfile profile) throws Exception { + boolean isDatastoreCompatibleWithStorageProfile = false; + + PbmPlacementHub placementHub = new PbmPlacementHub(); + placementHub.setHubId(dsMor.getValue()); + placementHub.setHubType(dsMor.getType()); + + List placementHubList = new ArrayList(); + placementHubList.add(placementHub); + PbmProfileId profileId = profile.getProfileId(); + List placementCompatibilityResultList = _context.getPbmService().pbmCheckCompatibility(_mor, placementHubList, profileId); + if (CollectionUtils.isNotEmpty(placementCompatibilityResultList)) { + for (PbmPlacementCompatibilityResult placementResult : placementCompatibilityResultList) { + // Check for error and warning + if (CollectionUtils.isEmpty(placementResult.getError()) && CollectionUtils.isEmpty(placementResult.getWarning())) { + isDatastoreCompatibleWithStorageProfile = true; + } + } + } + return isDatastoreCompatibleWithStorageProfile; + } +} diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/PbmProfileManagerMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/PbmProfileManagerMO.java new file mode 100644 index 000000000000..a4142ecde8ac --- /dev/null +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/PbmProfileManagerMO.java @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.hypervisor.vmware.mo; + +import com.cloud.hypervisor.vmware.util.VmwareContext; + +import com.cloud.utils.exception.CloudRuntimeException; +import com.vmware.pbm.PbmCapabilityProfile; +import com.vmware.pbm.PbmProfile; +import com.vmware.pbm.PbmProfileCategoryEnum; +import com.vmware.pbm.PbmProfileId; +import com.vmware.pbm.PbmProfileResourceType; +import com.vmware.pbm.PbmProfileResourceTypeEnum; +import com.vmware.vim25.ManagedObjectReference; + +import org.apache.log4j.Logger; + +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +public class PbmProfileManagerMO extends BaseMO { + + private static final Logger LOGGER = Logger.getLogger(PbmProfileManagerMO.class); + + public PbmProfileManagerMO (VmwareContext context) { + super(context, context.getPbmServiceContent().getProfileManager()); + } + + public PbmProfileManagerMO (VmwareContext context, ManagedObjectReference morProfileMgr) { + super(context, morProfileMgr); + } + + public PbmProfileManagerMO (VmwareContext context, String morType, String morValue) { + super(context, morType, morValue); + } + + public List getStorageProfileIds() throws Exception { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Querying vCenter " + _context.getServerAddress() + " for profiles"); + } + List profileIds = _context.getPbmService().pbmQueryProfile(_mor, getStorageResourceType(), null); + return profileIds; + } + + public List getStorageProfiles() throws Exception { + List profileIds = getStorageProfileIds(); + List profiles = _context.getPbmService().pbmRetrieveContent(_mor, profileIds); + + List requirementCategoryProfiles = profiles.stream() + .filter(x -> ((PbmCapabilityProfile)x).getProfileCategory().equals(PbmProfileCategoryEnum.REQUIREMENT.toString())) + .collect(Collectors.toList()); + return requirementCategoryProfiles; + } + + public PbmProfile getStorageProfile(String storageProfileId) throws Exception { + List profileIds = getStorageProfileIds(); + + PbmProfileId profileId = profileIds.stream() + .filter(x -> x.getUniqueId().equals(storageProfileId)) + .findFirst().orElse(null); + + if (profileId == null) { + String errMsg = String.format("Storage profile with id %s not found", storageProfileId); + LOGGER.debug(errMsg); + throw new CloudRuntimeException(errMsg); + } + + List profile = _context.getPbmService().pbmRetrieveContent(_mor, Collections.singletonList(profileId)); + return profile.get(0); + } + + private PbmProfileResourceType getStorageResourceType() { + PbmProfileResourceType resourceType = new PbmProfileResourceType(); + resourceType.setResourceType(PbmProfileResourceTypeEnum.STORAGE.value()); + return resourceType; + } +} + + diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/StoragepodMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/StoragepodMO.java new file mode 100644 index 000000000000..afa3a0221267 --- /dev/null +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/StoragepodMO.java @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.hypervisor.vmware.mo; + +import com.cloud.hypervisor.vmware.util.VmwareContext; +import com.vmware.vim25.ManagedObjectReference; +import com.vmware.vim25.StoragePodSummary; +import org.apache.log4j.Logger; + +import java.util.List; + +public class StoragepodMO extends BaseMO { + + private static final Logger LOGGER = Logger.getLogger(StoragepodMO.class); + + public StoragepodMO(VmwareContext context, ManagedObjectReference mor) { + super(context, mor); + } + + public StoragepodMO(VmwareContext context, String morType, String morValue) { + super(context, morType, morValue); + } + + public StoragePodSummary getDatastoreClusterSummary() throws Exception { + return (StoragePodSummary)_context.getVimClient().getDynamicProperty(_mor, "summary"); + } + + public List getDatastoresInDatastoreCluster() throws Exception { + List datastoresInCluster = _context.getVimClient().getDynamicProperty(_mor, "childEntity"); + return datastoresInCluster; + } + +} diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/TaskMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/TaskMO.java index 65c6a6bd3626..9cf9d9554c42 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/TaskMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/TaskMO.java @@ -77,4 +77,8 @@ public static String getTaskFailureInfo(VmwareContext context, ManagedObjectRefe return sb.toString(); } + + public static TaskInfo getTaskInfo(VmwareContext context, ManagedObjectReference morTask) throws Exception { + return (TaskInfo)context.getVimClient().getDynamicProperty(morTask, "info"); + } } diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java index 0e70799dfcb2..fd3c893aba8e 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java @@ -35,6 +35,7 @@ import java.util.concurrent.Future; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; import com.google.gson.Gson; @@ -300,7 +301,7 @@ public boolean safePowerOff(int shutdownWaitMs) throws Exception { try { Thread.sleep(1000); } catch (InterruptedException e) { - s_logger.debug("[ignored] interupted while powering of vm."); + s_logger.debug("[ignored] interrupted while powering of vm."); } } @@ -333,7 +334,7 @@ private boolean powerOffNoCheck() throws Exception { try { Thread.sleep(1000); } catch (InterruptedException e) { - s_logger.debug("[ignored] interupted while powering of vm unconditionaly."); + s_logger.debug("[ignored] interrupted while powering of vm unconditionally."); } } return true; @@ -367,7 +368,7 @@ public VirtualMachinePowerState getResetSafePowerState() throws Exception { try { Thread.sleep(1000); } catch (InterruptedException e) { - s_logger.debug("[ignored] interupted while pausing after power off."); + s_logger.debug("[ignored] interrupted while pausing after power off."); } } else { break; @@ -722,6 +723,44 @@ public boolean hasSnapshot() throws Exception { return false; } + public boolean createFullCloneWithSpecificDisk(String cloneName, ManagedObjectReference morFolder, ManagedObjectReference morResourcePool, ManagedObjectReference morDs, Pair volumeDeviceInfo) + throws Exception { + + assert (morFolder != null); + assert (morResourcePool != null); + assert (morDs != null); + VirtualDisk requiredDisk = volumeDeviceInfo.first(); + + VirtualMachineRelocateSpec rSpec = new VirtualMachineRelocateSpec(); + List diskLocator = new ArrayList(1); + VirtualMachineRelocateSpecDiskLocator loc = new VirtualMachineRelocateSpecDiskLocator(); + loc.setDatastore(morDs); + loc.setDiskId(requiredDisk.getKey()); + loc.setDiskMoveType(VirtualMachineRelocateDiskMoveOptions.MOVE_ALL_DISK_BACKINGS_AND_DISALLOW_SHARING.value()); + diskLocator.add(loc); + + rSpec.setDiskMoveType(VirtualMachineRelocateDiskMoveOptions.MOVE_ALL_DISK_BACKINGS_AND_DISALLOW_SHARING.value()); + rSpec.getDisk().addAll(diskLocator); + rSpec.setPool(morResourcePool); + + VirtualMachineCloneSpec cloneSpec = new VirtualMachineCloneSpec(); + cloneSpec.setPowerOn(false); + cloneSpec.setTemplate(false); + cloneSpec.setLocation(rSpec); + + ManagedObjectReference morTask = _context.getService().cloneVMTask(_mor, morFolder, cloneName, cloneSpec); + + boolean result = _context.getVimClient().waitForTask(morTask); + if (result) { + _context.waitForTaskProgressDone(morTask); + return true; + } else { + s_logger.error("VMware cloneVM_Task failed due to " + TaskMO.getTaskFailureInfo(_context, morTask)); + } + + return false; + } + public boolean createFullClone(String cloneName, ManagedObjectReference morFolder, ManagedObjectReference morResourcePool, ManagedObjectReference morDs) throws Exception { @@ -1189,7 +1228,18 @@ public void createDisk(String vmdkDatastorePath, VirtualDiskType diskType, Virtu s_logger.trace("vCenter API trace - createDisk() done(successfully)"); } - public void updateVmdkAdapter(String vmdkFileName, String newAdapterType) throws Exception { + public void updateVmdkAdapter(String vmdkFileName, String diskController) throws Exception { + + DiskControllerType diskControllerType = DiskControllerType.getType(diskController); + VmdkAdapterType vmdkAdapterType = VmdkAdapterType.getAdapterType(diskControllerType); + if (vmdkAdapterType == VmdkAdapterType.none) { + String message = "Failed to attach disk due to invalid vmdk adapter type for vmdk file [" + + vmdkFileName + "] with controller : " + diskControllerType; + s_logger.debug(message); + throw new Exception(message); + } + + String newAdapterType = vmdkAdapterType.toString(); Pair vmdkInfo = getVmdkFileInfo(vmdkFileName); VmdkFileDescriptor vmdkFileDescriptor = vmdkInfo.first(); boolean isVmfsSparseFile = vmdkFileDescriptor.isVmfsSparseFile(); @@ -1234,6 +1284,10 @@ public void updateAdapterTypeIfRequired(String vmdkFileName) throws Exception { } } + public void attachDisk(String[] vmdkDatastorePathChain, ManagedObjectReference morDs) throws Exception { + attachDisk(vmdkDatastorePathChain, morDs, null); + } + public void attachDisk(String[] vmdkDatastorePathChain, ManagedObjectReference morDs, String diskController) throws Exception { if(s_logger.isTraceEnabled()) @@ -1256,24 +1310,20 @@ public void attachDisk(String[] vmdkDatastorePathChain, ManagedObjectReference m controllerKey = getIDEControllerKey(ideDeviceCount); unitNumber = getFreeUnitNumberOnIDEController(controllerKey); } else { - controllerKey = getScsiDiskControllerKey(diskController); + if (StringUtils.isNotBlank(diskController)) { + controllerKey = getScsiDiskControllerKey(diskController); + } else { + controllerKey = getScsiDeviceControllerKey(); + } unitNumber = -1; } + synchronized (_mor.getValue().intern()) { VirtualDevice newDisk = VmwareHelper.prepareDiskDevice(this, null, controllerKey, vmdkDatastorePathChain, morDs, unitNumber, 1); - controllerKey = newDisk.getControllerKey(); - unitNumber = newDisk.getUnitNumber(); - VirtualDiskFlatVer2BackingInfo backingInfo = (VirtualDiskFlatVer2BackingInfo)newDisk.getBacking(); - String vmdkFileName = backingInfo.getFileName(); - DiskControllerType diskControllerType = DiskControllerType.getType(diskController); - VmdkAdapterType vmdkAdapterType = VmdkAdapterType.getAdapterType(diskControllerType); - if (vmdkAdapterType == VmdkAdapterType.none) { - String message = "Failed to attach disk due to invalid vmdk adapter type for vmdk file [" + - vmdkFileName + "] with controller : " + diskControllerType; - s_logger.debug(message); - throw new Exception(message); + if (StringUtils.isNotBlank(diskController)) { + String vmdkFileName = vmdkDatastorePathChain[0]; + updateVmdkAdapter(vmdkFileName, diskController); } - updateVmdkAdapter(vmdkFileName, vmdkAdapterType.toString()); VirtualMachineConfigSpec reConfigSpec = new VirtualMachineConfigSpec(); VirtualDeviceConfigSpec deviceConfigSpec = new VirtualDeviceConfigSpec(); @@ -1313,69 +1363,6 @@ private int getControllerBusNumber(int controllerKey) throws Exception { } - public void attachDisk(String[] vmdkDatastorePathChain, ManagedObjectReference morDs) throws Exception { - - if (s_logger.isTraceEnabled()) - s_logger.trace("vCenter API trace - attachDisk(). target MOR: " + _mor.getValue() + ", vmdkDatastorePath: " + new Gson().toJson(vmdkDatastorePathChain) + - ", datastore: " + morDs.getValue()); - - synchronized (_mor.getValue().intern()) { - VirtualDevice newDisk = VmwareHelper.prepareDiskDevice(this, null, getScsiDeviceControllerKey(), vmdkDatastorePathChain, morDs, -1, 1); - VirtualMachineConfigSpec reConfigSpec = new VirtualMachineConfigSpec(); - VirtualDeviceConfigSpec deviceConfigSpec = new VirtualDeviceConfigSpec(); - - deviceConfigSpec.setDevice(newDisk); - deviceConfigSpec.setOperation(VirtualDeviceConfigSpecOperation.ADD); - - reConfigSpec.getDeviceChange().add(deviceConfigSpec); - - ManagedObjectReference morTask = _context.getService().reconfigVMTask(_mor, reConfigSpec); - boolean result = _context.getVimClient().waitForTask(morTask); - - if (!result) { - if (s_logger.isTraceEnabled()) - s_logger.trace("vCenter API trace - attachDisk() done(failed)"); - throw new Exception("Failed to attach disk due to " + TaskMO.getTaskFailureInfo(_context, morTask)); - } - - _context.waitForTaskProgressDone(morTask); - } - - if (s_logger.isTraceEnabled()) - s_logger.trace("vCenter API trace - attachDisk() done(successfully)"); - } - - public void attachDisk(Pair[] vmdkDatastorePathChain, int controllerKey) throws Exception { - - if (s_logger.isTraceEnabled()) - s_logger.trace("vCenter API trace - attachDisk(). target MOR: " + _mor.getValue() + ", vmdkDatastorePath: " + new Gson().toJson(vmdkDatastorePathChain)); - - synchronized (_mor.getValue().intern()) { - VirtualDevice newDisk = VmwareHelper.prepareDiskDevice(this, controllerKey, vmdkDatastorePathChain, -1, 1); - VirtualMachineConfigSpec reConfigSpec = new VirtualMachineConfigSpec(); - VirtualDeviceConfigSpec deviceConfigSpec = new VirtualDeviceConfigSpec(); - - deviceConfigSpec.setDevice(newDisk); - deviceConfigSpec.setOperation(VirtualDeviceConfigSpecOperation.ADD); - - reConfigSpec.getDeviceChange().add(deviceConfigSpec); - - ManagedObjectReference morTask = _context.getService().reconfigVMTask(_mor, reConfigSpec); - boolean result = _context.getVimClient().waitForTask(morTask); - - if (!result) { - if (s_logger.isTraceEnabled()) - s_logger.trace("vCenter API trace - attachDisk() done(failed)"); - throw new Exception("Failed to attach disk due to " + TaskMO.getTaskFailureInfo(_context, morTask)); - } - - _context.waitForTaskProgressDone(morTask); - } - - if (s_logger.isTraceEnabled()) - s_logger.trace("vCenter API trace - attachDisk() done(successfully)"); - } - // vmdkDatastorePath: [datastore name] vmdkFilePath public List> detachDisk(String vmdkDatastorePath, boolean deleteBackingFile) throws Exception { @@ -1787,7 +1774,7 @@ public void action(Long param) { command.add((new File(name).getName())); } - s_logger.info("Package OVA with commmand: " + command.toString()); + s_logger.info("Package OVA with command: " + command.toString()); command.execute(); // to be safe, physically test existence of the target OVA file @@ -1800,7 +1787,7 @@ public void action(Long param) { success = true; } } - s_logger.info("volss: copy vmdk and ovf file finishes " + System.currentTimeMillis()); + s_logger.info("volss: copy vmdk and ovf file finished " + System.currentTimeMillis()); } catch (Throwable e) { s_logger.error("Unexpected exception ", e); } finally { @@ -2481,7 +2468,6 @@ public Pair getDiskDevice(String vmdkDatastorePath) throws String deviceNumbering = getDeviceBusName(devices, device); s_logger.info("Disk backing : " + diskBackingInfo.getFileName() + " matches ==> " + deviceNumbering); - return new Pair<>((VirtualDisk)device, deviceNumbering); } @@ -2552,15 +2538,15 @@ public Pair getDiskDevice(String vmdkDatastorePath, boolean if (matchExactly) { if (backingBaseName.equalsIgnoreCase(srcBaseName)) { String deviceNumbering = getDeviceBusName(devices, device); - s_logger.info("Disk backing : " + diskBackingInfo.getFileName() + " matches ==> " + deviceNumbering); + return new Pair((VirtualDisk)device, deviceNumbering); } } else { if (backingBaseName.contains(trimmedSrcBaseName)) { String deviceNumbering = getDeviceBusName(devices, device); - s_logger.info("Disk backing : " + diskBackingInfo.getFileName() + " matches ==> " + deviceNumbering); + return new Pair((VirtualDisk)device, deviceNumbering); } } @@ -2624,7 +2610,6 @@ public VirtualMachineDiskInfoBuilder getDiskInfoBuilder() throws Exception { VirtualDeviceBackingInfo backingInfo = ((VirtualDisk)device).getBacking(); if (backingInfo instanceof VirtualDiskFlatVer2BackingInfo) { VirtualDiskFlatVer2BackingInfo diskBackingInfo = (VirtualDiskFlatVer2BackingInfo)backingInfo; - while (diskBackingInfo != null) { String deviceBusName = getDeviceBusName(devices, device); builder.addDisk(deviceBusName, diskBackingInfo.getFileName()); @@ -3481,4 +3466,29 @@ public boolean consolidateVmDisks() throws Exception { } return false; } + + /** + * Upgrades this virtual machine's virtual hardware to the latest revision that is supported by the virtual machine's current host. + * @param version If specified, upgrade to that specified version. If not specified, upgrade to the most current virtual hardware supported on the host. + * @return true if success, false if not + */ + public boolean upgradeVirtualHardwareVersion(String version) { + try { + String targetHwVersion = StringUtils.isNotBlank(version) ? version : "the highest available"; + s_logger.info("Upgrading the VM hardware version to " + targetHwVersion); + ManagedObjectReference morTask = _context.getService().upgradeVMTask(_mor, version); + boolean result = _context.getVimClient().waitForTask(morTask); + if (result) { + _context.waitForTaskProgressDone(morTask); + } else { + s_logger.error("VMware upgradeVMTask failed due to " + TaskMO.getTaskFailureInfo(_context, morTask)); + return false; + } + return true; + } catch (Exception e) { + String msg = "Attempted to upgrade VM hardware version failed: " + e.getMessage(); + s_logger.error(msg, e); + return false; + } + } } diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualStorageObjectManagerMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualStorageObjectManagerMO.java new file mode 100644 index 000000000000..d5f4eb3af060 --- /dev/null +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualStorageObjectManagerMO.java @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.hypervisor.vmware.mo; + +import com.vmware.vim25.ID; +import com.vmware.vim25.TaskInfo; +import com.vmware.vim25.VStorageObject; +import com.vmware.vim25.VirtualDiskType; +import com.vmware.vim25.VslmCreateSpec; +import com.vmware.vim25.VslmCreateSpecDiskFileBackingSpec; +import org.apache.log4j.Logger; + +import com.vmware.vim25.ManagedObjectReference; + +import com.cloud.hypervisor.vmware.util.VmwareContext; + +public class VirtualStorageObjectManagerMO extends BaseMO { + @SuppressWarnings("unused") + private static final Logger LOGGER = Logger.getLogger(VirtualStorageObjectManagerMO.class); + + public VirtualStorageObjectManagerMO(VmwareContext context) { + super(context, context.getServiceContent().getVStorageObjectManager()); + } + + public VirtualStorageObjectManagerMO(VmwareContext context, ManagedObjectReference morDiskMgr) { + super(context, morDiskMgr); + } + + public VirtualStorageObjectManagerMO(VmwareContext context, String morType, String morValue) { + super(context, morType, morValue); + } + + public VStorageObject registerVirtualDisk(DatastoreFile datastoreFile, String name, String dcName) throws Exception { + StringBuilder sb = new StringBuilder(); + //https://10.2.2.254/folder/i-2-4-VM/89e3756d9b7444dc92388eb36ddd026b.vmdk?dcPath=datacenter-21&dsName=c84e4af9b6ac33e887a25d9242650091 + sb.append("https://").append(_context.getServerAddress()).append("/folder/"); + sb.append(datastoreFile.getRelativePath()); + sb.append("?dcPath="); + sb.append(dcName); + sb.append("&dsName="); + sb.append(datastoreFile.getDatastoreName()); + return _context.getService().registerDisk(_mor, sb.toString(), name); + } + + public VStorageObject retrieveVirtualDisk (ID id, ManagedObjectReference morDS) throws Exception { + return _context.getService().retrieveVStorageObject(_mor, id, morDS); + } + + public VStorageObject createDisk(ManagedObjectReference morDS, VirtualDiskType diskType, long currentSizeInBytes, String datastoreFilepath, String filename) throws Exception { + long currentSizeInMB = currentSizeInBytes/(1024*1024); + + VslmCreateSpecDiskFileBackingSpec diskFileBackingSpec = new VslmCreateSpecDiskFileBackingSpec(); + diskFileBackingSpec.setDatastore(morDS); + diskFileBackingSpec.setProvisioningType(diskType.value()); + // path should be just the folder name. For example, instead of '[datastore1] folder1/filename.vmdk' you would just do 'folder1'. + // path is introduced from 6.7. In 6.5 disk will be created in the default folder "fcd" + diskFileBackingSpec.setPath(null); + + VslmCreateSpec vslmCreateSpec = new VslmCreateSpec(); + vslmCreateSpec.setBackingSpec(diskFileBackingSpec); + vslmCreateSpec.setCapacityInMB(currentSizeInMB); + vslmCreateSpec.setName(filename); + + ManagedObjectReference morTask = _context.getService().createDiskTask(_mor, vslmCreateSpec); + boolean result = _context.getVimClient().waitForTask(morTask); + + VStorageObject vStorageObject = null; + if (result) { + _context.waitForTaskProgressDone(morTask); + //_context.getService().reconcileDatastoreInventoryTask(_mor, morDS); + TaskInfo taskInfo = TaskMO.getTaskInfo(_context, morTask); + vStorageObject = (VStorageObject)taskInfo.getResult(); + + } else { + LOGGER.error("VMware CreateDisk_Task failed due to " + TaskMO.getTaskFailureInfo(_context, morTask)); + } + + return vStorageObject; + } +} diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VmwareHypervisorHost.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VmwareHypervisorHost.java index a9ceb5d806ec..87414b17c651 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VmwareHypervisorHost.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VmwareHypervisorHost.java @@ -65,13 +65,13 @@ boolean createBlankVm(String vmName, String vmInternalCSName, int cpuCount, int int memoryReserveMB, String guestOsIdentifier, ManagedObjectReference morDs, boolean snapshotDirToParent, Pair controllerInfo, Boolean systemVm) throws Exception; - void importVmFromOVF(String ovfFilePath, String vmName, DatastoreMO dsMo, String diskOption) throws Exception; + void importVmFromOVF(String ovfFilePath, String vmName, DatastoreMO dsMo, String diskOption, String configurationId) throws Exception; ObjectContent[] getVmPropertiesOnHyperHost(String[] propertyPaths) throws Exception; ObjectContent[] getDatastorePropertiesOnHyperHost(String[] propertyPaths) throws Exception; - ManagedObjectReference mountDatastore(boolean vmfsDatastore, String poolHostAddress, int poolHostPort, String poolPath, String poolUuid) throws Exception; + ManagedObjectReference mountDatastore(boolean vmfsDatastore, String poolHostAddress, int poolHostPort, String poolPath, String poolUuid, boolean createBaseFolder) throws Exception; void unmountDatastore(String poolUuid) throws Exception; diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VcenterSessionHandler.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VcenterSessionHandler.java new file mode 100644 index 000000000000..9efab7b8eceb --- /dev/null +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VcenterSessionHandler.java @@ -0,0 +1,88 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.hypervisor.vmware.util; + +import java.util.Set; + +import javax.xml.namespace.QName; +import javax.xml.soap.SOAPElement; +import javax.xml.soap.SOAPException; +import javax.xml.soap.SOAPHeader; +import javax.xml.ws.handler.MessageContext; +import javax.xml.ws.handler.soap.SOAPHandler; +import javax.xml.ws.handler.soap.SOAPMessageContext; + +import org.apache.log4j.Logger; +import org.w3c.dom.DOMException; + +import com.cloud.utils.exception.CloudRuntimeException; + +public class VcenterSessionHandler implements SOAPHandler { + public static final Logger s_logger = Logger.getLogger(VcenterSessionHandler.class); + private final String vcSessionCookie; + + public VcenterSessionHandler(String vcSessionCookie) { + this.vcSessionCookie = vcSessionCookie; + } + + @Override + public boolean handleMessage(SOAPMessageContext smc) { + if (isOutgoingMessage(smc)) { + try { + SOAPHeader header = getSOAPHeader(smc); + + SOAPElement vcsessionHeader = header.addChildElement(new javax.xml.namespace.QName("#", + "vcSessionCookie")); + vcsessionHeader.setValue(vcSessionCookie); + + } catch (DOMException e) { + s_logger.debug(e); + throw new CloudRuntimeException(e); + } catch (SOAPException e) { + s_logger.debug(e); + throw new CloudRuntimeException(e); + } + } + return true; + } + + @Override + public void close(MessageContext arg0) { + } + + @Override + public boolean handleFault(SOAPMessageContext arg0) { + return false; + } + + @Override + public Set getHeaders() { + return null; + } + + SOAPHeader getSOAPHeader(SOAPMessageContext smc) throws SOAPException { + return smc.getMessage().getSOAPPart().getEnvelope().getHeader() == null ? smc + .getMessage().getSOAPPart().getEnvelope().addHeader() + : smc.getMessage().getSOAPPart().getEnvelope().getHeader(); + } + + boolean isOutgoingMessage(SOAPMessageContext smc) { + Boolean outboundProperty = (Boolean)smc.get(MessageContext.MESSAGE_OUTBOUND_PROPERTY); + return outboundProperty; + } + +} diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareClient.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareClient.java index 3d80ffdfae74..2395ccf53262 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareClient.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareClient.java @@ -17,8 +17,11 @@ package com.cloud.hypervisor.vmware.util; import java.lang.reflect.Method; +import java.net.URI; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.StringTokenizer; @@ -29,9 +32,16 @@ import javax.xml.ws.BindingProvider; import javax.xml.ws.WebServiceException; import javax.xml.ws.handler.MessageContext; +import javax.xml.ws.handler.Handler; +import javax.xml.ws.handler.HandlerResolver; +import javax.xml.ws.handler.PortInfo; + import org.apache.cloudstack.utils.security.SSLUtils; import org.apache.cloudstack.utils.security.SecureSSLSocketFactory; +import com.vmware.pbm.PbmPortType; +import com.vmware.pbm.PbmService; +import com.vmware.pbm.PbmServiceInstanceContent; import org.apache.log4j.Logger; import org.w3c.dom.Element; @@ -101,6 +111,7 @@ public boolean verify(String urlHostName, SSLSession session) { HttpsURLConnection.setDefaultHostnameVerifier(hv); vimService = new VimService(); + pbmService = new PbmService(); } catch (Exception e) { s_logger.info("[ignored]" + "failed to trust all certificates blindly: ", e); @@ -120,8 +131,16 @@ private static void trustAllHttpsCertificates() throws Exception { } private final ManagedObjectReference svcInstRef = new ManagedObjectReference(); + private final ManagedObjectReference pbmSvcInstRef = new ManagedObjectReference(); + private static VimService vimService; + private static PbmService pbmService; + private PbmServiceInstanceContent pbmServiceContent; private VimPortType vimPort; + private PbmPortType pbmPort; + private static final String PBM_SERVICE_INSTANCE_TYPE = "PbmServiceInstance"; + private static final String PBM_SERVICE_INSTANCE_VALUE = "ServiceInstance"; + private String serviceCookie; private final static String SVC_INST_NAME = "ServiceInstance"; private int vCenterSessionTimeout = 1200000; // Timeout in milliseconds @@ -176,10 +195,38 @@ public void connect(String url, String userName, String password) throws Excepti cookieValue = tokenizer.nextToken(); String pathData = "$" + tokenizer.nextToken(); serviceCookie = "$Version=\"1\"; " + cookieValue + "; " + pathData; - + Map> map = new HashMap>(); + map.put("Cookie", Collections.singletonList(serviceCookie)); + ((BindingProvider)vimPort).getRequestContext().put(MessageContext.HTTP_REQUEST_HEADERS, map); + pbmConnect(url, cookieValue); isConnected = true; } + private void pbmConnect(String url, String cookieValue) throws Exception { + URI uri = new URI(url); + String pbmurl = "https://" + uri.getHost() + "/pbm"; + String[] tokens = cookieValue.split("="); + String extractedCookie = tokens[1]; + + HandlerResolver soapHandlerResolver = new HandlerResolver() { + @Override + public List getHandlerChain(PortInfo portInfo) { + VcenterSessionHandler VcSessionHandler = new VcenterSessionHandler(extractedCookie); + List handlerChain = new ArrayList(); + handlerChain.add((Handler)VcSessionHandler); + return handlerChain; + } + }; + pbmService.setHandlerResolver(soapHandlerResolver); + + pbmSvcInstRef.setType(PBM_SERVICE_INSTANCE_TYPE); + pbmSvcInstRef.setValue(PBM_SERVICE_INSTANCE_VALUE); + pbmPort = pbmService.getPbmPort(); + Map pbmCtxt = ((BindingProvider)pbmPort).getRequestContext(); + pbmCtxt.put(BindingProvider.SESSION_MAINTAIN_PROPERTY, true); + pbmCtxt.put(BindingProvider.ENDPOINT_ADDRESS_PROPERTY, pbmurl); + } + /** * Disconnects the user session. * @@ -211,6 +258,24 @@ public ServiceContent getServiceContent() { return null; } + /** + * @return PBM service instance + */ + public PbmPortType getPbmService() { + return pbmPort; + } + + /** + * @return Service instance content + */ + public PbmServiceInstanceContent getPbmServiceContent() { + try { + return pbmPort.pbmRetrieveServiceContent(pbmSvcInstRef); + } catch (com.vmware.pbm.RuntimeFaultFaultMsg e) { + } + return null; + } + /** * @return cookie used in service connection */ diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareContext.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareContext.java index 9b477aef42bc..af44962322e3 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareContext.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareContext.java @@ -20,6 +20,8 @@ import com.cloud.hypervisor.vmware.mo.DatastoreFile; import com.cloud.utils.ActionDelegate; import com.cloud.utils.StringUtils; +import com.vmware.pbm.PbmPortType; +import com.vmware.pbm.PbmServiceInstanceContent; import com.vmware.vim25.ManagedObjectReference; import com.vmware.vim25.ObjectContent; import com.vmware.vim25.ObjectSpec; @@ -148,6 +150,14 @@ public ServiceContent getServiceContent() { return _vimClient.getServiceContent(); } + public PbmPortType getPbmService() { + return _vimClient.getPbmService(); + } + + public PbmServiceInstanceContent getPbmServiceContent() { + return _vimClient.getPbmServiceContent(); + } + public ManagedObjectReference getPropertyCollector() { return _vimClient.getPropCol(); } @@ -311,6 +321,24 @@ public ManagedObjectReference getDatastoreMorByPath(String inventoryPath) throws return dcMo.findDatastore(tokens[1]); } + // path in format of / + public String getDatastoreNameFromPath(String inventoryPath) throws Exception { + assert (inventoryPath != null); + + String[] tokens; + if (inventoryPath.startsWith("/")) + tokens = inventoryPath.substring(1).split("/"); + else + tokens = inventoryPath.split("/"); + + if (tokens == null || tokens.length != 2) { + s_logger.error("Invalid datastore inventory path. path: " + inventoryPath); + return null; + } + + return tokens[1]; + } + public void waitForTaskProgressDone(ManagedObjectReference morTask) throws Exception { while (true) { TaskInfo tinfo = (TaskInfo)_vimClient.getDynamicProperty(morTask, "info"); @@ -339,32 +367,32 @@ public void getFile(String urlString, String localFileFullName) throws Exception out.close(); } - public void uploadFile(String urlString, String localFileFullName) throws Exception { - uploadFile(urlString, new File(localFileFullName)); - } + public void uploadFile(String httpMethod, String urlString, String localFileName) throws Exception { + HttpURLConnection conn = getRawHTTPConnection(urlString); - public void uploadFile(String urlString, File localFile) throws Exception { - HttpURLConnection conn = getHTTPConnection(urlString, "PUT"); + conn.setDoOutput(true); + conn.setUseCaches(false); + + conn.setChunkedStreamingMode(ChunkSize); + conn.setRequestMethod(httpMethod); + conn.setRequestProperty("Connection", "Keep-Alive"); + String contentType = "application/octet-stream"; + conn.setRequestProperty("Content-Type", contentType); + conn.setRequestProperty("Content-Length", Long.toString(new File(localFileName).length())); + connectWithRetry(conn); OutputStream out = null; InputStream in = null; BufferedReader br = null; try { out = conn.getOutputStream(); - in = new FileInputStream(localFile); + in = new FileInputStream(localFileName); byte[] buf = new byte[ChunkSize]; int len = 0; while ((len = in.read(buf)) > 0) { out.write(buf, 0, len); } out.flush(); - - br = new BufferedReader(new InputStreamReader(conn.getInputStream(), getCharSetFromConnection(conn))); - String line; - while ((line = br.readLine()) != null) { - if (s_logger.isTraceEnabled()) - s_logger.trace("Upload " + urlString + " response: " + line); - } } finally { if (in != null) in.close(); @@ -374,6 +402,7 @@ public void uploadFile(String urlString, File localFile) throws Exception { if (br != null) br.close(); + conn.disconnect(); } } @@ -398,8 +427,14 @@ public void uploadVmdkFile(String httpMethod, String urlString, String localFile conn.setChunkedStreamingMode(ChunkSize); conn.setRequestMethod(httpMethod); + if (urlString.endsWith(".iso")) { + conn.setRequestProperty("Overwrite", "t"); + } conn.setRequestProperty("Connection", "Keep-Alive"); - conn.setRequestProperty("Content-Type", "application/x-vnd.vmware-streamVmdk"); + String contentType = urlString.endsWith(".iso") ? + "application/octet-stream" : + "application/x-vnd.vmware-streamVmdk"; + conn.setRequestProperty("Content-Type", contentType); conn.setRequestProperty("Content-Length", Long.toString(new File(localFileName).length())); connectWithRetry(conn); diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java index 181b2ef183f6..424027ba7121 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java @@ -25,6 +25,7 @@ import java.io.StringWriter; import java.lang.reflect.Method; import java.util.ArrayList; +import java.util.Arrays; import java.util.Date; import java.util.GregorianCalendar; import java.util.List; @@ -75,7 +76,9 @@ import com.vmware.vim25.VirtualVmxnet3; import com.cloud.hypervisor.vmware.mo.DiskControllerType; +import com.cloud.hypervisor.vmware.mo.DatastoreMO; import com.cloud.hypervisor.vmware.mo.HostMO; +import com.cloud.hypervisor.vmware.mo.CustomFieldConstants; import com.cloud.hypervisor.vmware.mo.LicenseAssignmentManagerMO; import com.cloud.hypervisor.vmware.mo.VirtualEthernetCardType; import com.cloud.hypervisor.vmware.mo.VirtualMachineMO; @@ -212,35 +215,46 @@ public static VirtualDevice prepareDvNicDevice(VirtualMachineMO vmMo, ManagedObj } // vmdkDatastorePath: [datastore name] vmdkFilePath - public static VirtualDevice prepareDiskDevice(VirtualMachineMO vmMo, int controllerKey, String vmdkDatastorePath, int sizeInMb, ManagedObjectReference morDs, - int deviceNumber, int contextNumber) throws Exception { + public static VirtualDevice prepareDiskDevice(VirtualMachineMO vmMo, VirtualDisk device, int controllerKey, String vmdkDatastorePathChain[], + ManagedObjectReference morDs, int deviceNumber, int contextNumber) throws Exception { - VirtualDisk disk = new VirtualDisk(); + assert (vmdkDatastorePathChain != null); + assert (vmdkDatastorePathChain.length >= 1); - VirtualDiskFlatVer2BackingInfo backingInfo = new VirtualDiskFlatVer2BackingInfo(); - backingInfo.setDiskMode(VirtualDiskMode.PERSISTENT.value()); - backingInfo.setThinProvisioned(true); - backingInfo.setEagerlyScrub(false); - backingInfo.setDatastore(morDs); - backingInfo.setFileName(vmdkDatastorePath); - disk.setBacking(backingInfo); + VirtualDisk disk; + VirtualDiskFlatVer2BackingInfo backingInfo; + if (device != null) { + disk = device; + backingInfo = (VirtualDiskFlatVer2BackingInfo)disk.getBacking(); + } else { + disk = new VirtualDisk(); + backingInfo = new VirtualDiskFlatVer2BackingInfo(); + backingInfo.setDatastore(morDs); + backingInfo.setDiskMode(VirtualDiskMode.PERSISTENT.value()); + disk.setBacking(backingInfo); - int ideControllerKey = vmMo.getIDEDeviceControllerKey(); - if (controllerKey < 0) - controllerKey = ideControllerKey; - if (deviceNumber < 0) { - deviceNumber = vmMo.getNextDeviceNumber(controllerKey); - } - disk.setControllerKey(controllerKey); + int ideControllerKey = vmMo.getIDEDeviceControllerKey(); + if (controllerKey < 0) + controllerKey = ideControllerKey; + if (deviceNumber < 0) { + deviceNumber = vmMo.getNextDeviceNumber(controllerKey); + } - disk.setKey(-contextNumber); - disk.setUnitNumber(deviceNumber); - disk.setCapacityInKB(sizeInMb * 1024); + disk.setControllerKey(controllerKey); + disk.setKey(-contextNumber); + disk.setUnitNumber(deviceNumber); - VirtualDeviceConnectInfo connectInfo = new VirtualDeviceConnectInfo(); - connectInfo.setConnected(true); - connectInfo.setStartConnected(true); - disk.setConnectable(connectInfo); + VirtualDeviceConnectInfo connectInfo = new VirtualDeviceConnectInfo(); + connectInfo.setConnected(true); + connectInfo.setStartConnected(true); + disk.setConnectable(connectInfo); + } + + backingInfo.setFileName(vmdkDatastorePathChain[0]); + if (vmdkDatastorePathChain.length > 1) { + String[] parentDisks = Arrays.copyOfRange(vmdkDatastorePathChain, 1, vmdkDatastorePathChain.length); + setParentBackingInfo(backingInfo, morDs, parentDisks); + } return disk; } @@ -314,96 +328,6 @@ public static VirtualDevice prepareDiskDevice(VirtualMachineMO vmMo, int control return disk; } - // vmdkDatastorePath: [datastore name] vmdkFilePath - public static VirtualDevice prepareDiskDevice(VirtualMachineMO vmMo, VirtualDisk device, int controllerKey, String vmdkDatastorePathChain[], - ManagedObjectReference morDs, int deviceNumber, int contextNumber) throws Exception { - - assert (vmdkDatastorePathChain != null); - assert (vmdkDatastorePathChain.length >= 1); - - VirtualDisk disk; - VirtualDiskFlatVer2BackingInfo backingInfo; - if (device != null) { - disk = device; - backingInfo = (VirtualDiskFlatVer2BackingInfo)disk.getBacking(); - } else { - disk = new VirtualDisk(); - backingInfo = new VirtualDiskFlatVer2BackingInfo(); - backingInfo.setDatastore(morDs); - backingInfo.setDiskMode(VirtualDiskMode.PERSISTENT.value()); - disk.setBacking(backingInfo); - - int ideControllerKey = vmMo.getIDEDeviceControllerKey(); - if (controllerKey < 0) - controllerKey = ideControllerKey; - if (deviceNumber < 0) { - deviceNumber = vmMo.getNextDeviceNumber(controllerKey); - } - - disk.setControllerKey(controllerKey); - disk.setKey(-contextNumber); - disk.setUnitNumber(deviceNumber); - - VirtualDeviceConnectInfo connectInfo = new VirtualDeviceConnectInfo(); - connectInfo.setConnected(true); - connectInfo.setStartConnected(true); - disk.setConnectable(connectInfo); - } - - backingInfo.setFileName(vmdkDatastorePathChain[0]); - if (vmdkDatastorePathChain.length > 1) { - String[] parentDisks = new String[vmdkDatastorePathChain.length - 1]; - for (int i = 0; i < vmdkDatastorePathChain.length - 1; i++) - parentDisks[i] = vmdkDatastorePathChain[i + 1]; - - setParentBackingInfo(backingInfo, morDs, parentDisks); - } - - return disk; - } - - @SuppressWarnings("unchecked") - public static VirtualDevice prepareDiskDevice(VirtualMachineMO vmMo, int controllerKey, Pair[] vmdkDatastorePathChain, - int deviceNumber, int contextNumber) throws Exception { - - assert (vmdkDatastorePathChain != null); - assert (vmdkDatastorePathChain.length >= 1); - - VirtualDisk disk = new VirtualDisk(); - - VirtualDiskFlatVer2BackingInfo backingInfo = new VirtualDiskFlatVer2BackingInfo(); - backingInfo.setDatastore(vmdkDatastorePathChain[0].second()); - backingInfo.setFileName(vmdkDatastorePathChain[0].first()); - backingInfo.setDiskMode(VirtualDiskMode.PERSISTENT.value()); - if (vmdkDatastorePathChain.length > 1) { - Pair[] parentDisks = new Pair[vmdkDatastorePathChain.length - 1]; - for (int i = 0; i < vmdkDatastorePathChain.length - 1; i++) - parentDisks[i] = vmdkDatastorePathChain[i + 1]; - - setParentBackingInfo(backingInfo, parentDisks); - } - - disk.setBacking(backingInfo); - - int ideControllerKey = vmMo.getIDEDeviceControllerKey(); - if (controllerKey < 0) - controllerKey = ideControllerKey; - if (deviceNumber < 0) { - deviceNumber = vmMo.getNextDeviceNumber(controllerKey); - } - - disk.setControllerKey(controllerKey); - disk.setKey(-contextNumber); - disk.setUnitNumber(deviceNumber); - - VirtualDeviceConnectInfo connectInfo = new VirtualDeviceConnectInfo(); - connectInfo.setConnected(true); - connectInfo.setStartConnected(true); - disk.setConnectable(connectInfo); - - return disk; - } - private static void setParentBackingInfo(VirtualDiskFlatVer2BackingInfo backingInfo, ManagedObjectReference morDs, String[] parentDatastorePathList) { VirtualDiskFlatVer2BackingInfo parentBacking = new VirtualDiskFlatVer2BackingInfo(); @@ -610,7 +534,7 @@ public static void setVmScaleUpConfig(VirtualMachineConfigSpec vmConfig, int cpu } public static void setBasicVmConfig(VirtualMachineConfigSpec vmConfig, int cpuCount, int cpuSpeedMHz, int cpuReservedMhz, int memoryMB, int memoryReserveMB, - String guestOsIdentifier, boolean limitCpuUse) { + String guestOsIdentifier, boolean limitCpuUse, boolean deployAsIs) { // VM config basics vmConfig.setMemoryMB((long)memoryMB); @@ -636,7 +560,11 @@ public static void setBasicVmConfig(VirtualMachineConfigSpec vmConfig, int cpuCo memInfo.setReservation((long)memoryReserveMB); vmConfig.setMemoryAllocation(memInfo); - vmConfig.setGuestId(guestOsIdentifier); + if (!deployAsIs) { + // Deploy as-is uses the cloned VM guest OS + vmConfig.setGuestId(guestOsIdentifier); + } + } public static VirtualDevice prepareUSBControllerDevice() { @@ -770,9 +698,13 @@ public static boolean isFeatureLicensed(VmwareHypervisorHost hyperHost, String f return hotplugSupportedByLicense; } - public static String getVCenterSafeUuid() { + public static String getVCenterSafeUuid(DatastoreMO dsMo) throws Exception{ // Object name that is greater than 32 is not safe in vCenter - return UUID.randomUUID().toString().replaceAll("-", ""); + String uuid = UUID.randomUUID().toString().replaceAll("-", ""); + if (dsMo.getDatastoreType().equalsIgnoreCase("VVOL")) { + return CustomFieldConstants.CLOUD_UUID + "-" + uuid; + } + return uuid; } public static String getRecommendedDiskControllerFromDescriptor(GuestOsDescriptor guestOsDescriptor) throws Exception {