Skip to content

Commit 95364a4

Browse files
author
Marcus Sorensen
committed
CLOUDSTACK-5531
Initial support for vhd, raw, vmdk image formats on KVM. Tested all formats with local and CLVM.
1 parent 7e4407d commit 95364a4

12 files changed

Lines changed: 289 additions & 96 deletions

File tree

‎api/src/com/cloud/storage/Storage.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@ public static enum ImageFormat {
2828
OVA(true, true, true, "ova"),
2929
VHDX(true, true, true, "vhdx"),
3030
BAREMETAL(false, false, false, "BAREMETAL"),
31+
VMDK(true, true, false, "vmdk"),
32+
VDI(true, true, false, "vdi"),
3133
TAR(false, false, false, "tar");
3234

3335
private final boolean thinProvisioned;

‎core/src/com/cloud/storage/template/OVAProcessor.java‎

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
package com.cloud.storage.template;
18+
19+
import java.io.File;
20+
import java.util.Map;
21+
22+
import javax.ejb.Local;
23+
import javax.naming.ConfigurationException;
24+
import javax.xml.parsers.DocumentBuilderFactory;
25+
26+
import org.apache.log4j.Logger;
27+
import org.w3c.dom.Document;
28+
import org.w3c.dom.Element;
29+
30+
import com.cloud.exception.InternalErrorException;
31+
import com.cloud.storage.Storage.ImageFormat;
32+
import com.cloud.storage.StorageLayer;
33+
import com.cloud.utils.component.AdapterBase;
34+
import com.cloud.utils.script.Script;
35+
36+
@Local(value = Processor.class)
37+
public class OVAProcessor extends AdapterBase implements Processor {
38+
private static final Logger s_logger = Logger.getLogger(OVAProcessor.class);
39+
40+
StorageLayer _storage;
41+
42+
@Override
43+
public FormatInfo process(String templatePath, ImageFormat format, String templateName) throws InternalErrorException {
44+
if (format != null) {
45+
if (s_logger.isInfoEnabled()) {
46+
s_logger.info("We currently don't handle conversion from " + format + " to OVA.");
47+
}
48+
return null;
49+
}
50+
51+
s_logger.info("Template processing. templatePath: " + templatePath + ", templateName: " + templateName);
52+
String templateFilePath = templatePath + File.separator + templateName + "." + ImageFormat.OVA.getFileExtension();
53+
if (!_storage.exists(templateFilePath)) {
54+
if (s_logger.isInfoEnabled()) {
55+
s_logger.info("Unable to find the vmware template file: " + templateFilePath);
56+
}
57+
return null;
58+
}
59+
60+
s_logger.info("Template processing - untar OVA package. templatePath: " + templatePath + ", templateName: " + templateName);
61+
String templateFileFullPath = templatePath + File.separator + templateName + "." + ImageFormat.OVA.getFileExtension();
62+
File templateFile = new File(templateFileFullPath);
63+
64+
Script command = new Script("tar", 0, s_logger);
65+
command.add("--no-same-owner");
66+
command.add("-xf", templateFileFullPath);
67+
command.setWorkDir(templateFile.getParent());
68+
String result = command.execute();
69+
if (result != null) {
70+
s_logger.info("failed to untar OVA package due to " + result + ". templatePath: " + templatePath + ", templateName: " + templateName);
71+
return null;
72+
}
73+
74+
FormatInfo info = new FormatInfo();
75+
info.format = ImageFormat.OVA;
76+
info.filename = templateName + "." + ImageFormat.OVA.getFileExtension();
77+
info.size = _storage.getSize(templateFilePath);
78+
info.virtualSize = getTemplateVirtualSize(templatePath, info.filename);
79+
80+
// delete original OVA file
81+
// templateFile.delete();
82+
return info;
83+
}
84+
85+
@Override
86+
public Long getVirtualSize(File file) {
87+
try {
88+
long size = getTemplateVirtualSize(file.getParent(), file.getName());
89+
return size;
90+
} catch (Exception e) {
91+
92+
}
93+
return file.length();
94+
}
95+
96+
public long getTemplateVirtualSize(String templatePath, String templateName) throws InternalErrorException {
97+
// get the virtual size from the OVF file meta data
98+
long virtualSize = 0;
99+
String templateFileFullPath = templatePath.endsWith(File.separator) ? templatePath : templatePath + File.separator;
100+
templateFileFullPath += templateName.endsWith(ImageFormat.OVA.getFileExtension()) ? templateName : templateName + "." + ImageFormat.OVA.getFileExtension();
101+
String ovfFileName = getOVFFilePath(templateFileFullPath);
102+
if (ovfFileName == null) {
103+
String msg = "Unable to locate OVF file in template package directory: " + templatePath;
104+
s_logger.error(msg);
105+
throw new InternalErrorException(msg);
106+
}
107+
try {
108+
Document ovfDoc = null;
109+
ovfDoc = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(new File(ovfFileName));
110+
Element disk = (Element)ovfDoc.getElementsByTagName("Disk").item(0);
111+
virtualSize = Long.parseLong(disk.getAttribute("ovf:capacity"));
112+
String allocationUnits = disk.getAttribute("ovf:capacityAllocationUnits");
113+
if ((virtualSize != 0) && (allocationUnits != null)) {
114+
long units = 1;
115+
if (allocationUnits.equalsIgnoreCase("KB") || allocationUnits.equalsIgnoreCase("KiloBytes") || allocationUnits.equalsIgnoreCase("byte * 2^10")) {
116+
units = 1024;
117+
} else if (allocationUnits.equalsIgnoreCase("MB") || allocationUnits.equalsIgnoreCase("MegaBytes") || allocationUnits.equalsIgnoreCase("byte * 2^20")) {
118+
units = 1024 * 1024;
119+
} else if (allocationUnits.equalsIgnoreCase("GB") || allocationUnits.equalsIgnoreCase("GigaBytes") || allocationUnits.equalsIgnoreCase("byte * 2^30")) {
120+
units = 1024 * 1024 * 1024;
121+
}
122+
virtualSize = virtualSize * units;
123+
} else {
124+
throw new InternalErrorException("Failed to read capacity and capacityAllocationUnits from the OVF file: " + ovfFileName);
125+
}
126+
return virtualSize;
127+
} catch (Exception e) {
128+
String msg = "Unable to parse OVF XML document to get the virtual disk size due to" + e;
129+
s_logger.error(msg);
130+
throw new InternalErrorException(msg);
131+
}
132+
}
133+
134+
private String getOVFFilePath(String srcOVAFileName) {
135+
File file = new File(srcOVAFileName);
136+
assert (_storage != null);
137+
String[] files = _storage.listFiles(file.getParent());
138+
if (files != null) {
139+
for (String fileName : files) {
140+
if (fileName.toLowerCase().endsWith(".ovf")) {
141+
File ovfFile = new File(fileName);
142+
return file.getParent() + File.separator + ovfFile.getName();
143+
}
144+
}
145+
}
146+
return null;
147+
}
148+
149+
@Override
150+
public boolean configure(String name, Map<String, Object> params) throws ConfigurationException {
151+
_storage = (StorageLayer)params.get(StorageLayer.InstanceConfigKey);
152+
if (_storage == null) {
153+
throw new ConfigurationException("Unable to get storage implementation");
154+
}
155+
156+
return true;
157+
}
158+
}

‎core/src/com/cloud/storage/template/VmdkProcessor.java‎

Lines changed: 31 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -16,22 +16,24 @@
1616
// under the License.
1717
package com.cloud.storage.template;
1818

19+
import java.io.BufferedReader;
1920
import java.io.File;
21+
import java.io.FileReader;
22+
import java.io.FileNotFoundException;
23+
import java.io.IOException;
2024
import java.util.Map;
25+
import java.util.regex.Pattern;
26+
import java.util.regex.Matcher;
2127

2228
import javax.ejb.Local;
2329
import javax.naming.ConfigurationException;
24-
import javax.xml.parsers.DocumentBuilderFactory;
2530

2631
import org.apache.log4j.Logger;
27-
import org.w3c.dom.Document;
28-
import org.w3c.dom.Element;
2932

3033
import com.cloud.exception.InternalErrorException;
3134
import com.cloud.storage.Storage.ImageFormat;
3235
import com.cloud.storage.StorageLayer;
3336
import com.cloud.utils.component.AdapterBase;
34-
import com.cloud.utils.script.Script;
3537

3638
@Local(value = Processor.class)
3739
public class VmdkProcessor extends AdapterBase implements Processor {
@@ -49,36 +51,20 @@ public FormatInfo process(String templatePath, ImageFormat format, String templa
4951
}
5052

5153
s_logger.info("Template processing. templatePath: " + templatePath + ", templateName: " + templateName);
52-
String templateFilePath = templatePath + File.separator + templateName + "." + ImageFormat.OVA.getFileExtension();
54+
String templateFilePath = templatePath + File.separator + templateName + "." + ImageFormat.VMDK.getFileExtension();
5355
if (!_storage.exists(templateFilePath)) {
5456
if (s_logger.isInfoEnabled()) {
5557
s_logger.info("Unable to find the vmware template file: " + templateFilePath);
5658
}
5759
return null;
5860
}
5961

60-
s_logger.info("Template processing - untar OVA package. templatePath: " + templatePath + ", templateName: " + templateName);
61-
String templateFileFullPath = templatePath + File.separator + templateName + "." + ImageFormat.OVA.getFileExtension();
62-
File templateFile = new File(templateFileFullPath);
63-
64-
Script command = new Script("tar", 0, s_logger);
65-
command.add("--no-same-owner");
66-
command.add("-xf", templateFileFullPath);
67-
command.setWorkDir(templateFile.getParent());
68-
String result = command.execute();
69-
if (result != null) {
70-
s_logger.info("failed to untar OVA package due to " + result + ". templatePath: " + templatePath + ", templateName: " + templateName);
71-
return null;
72-
}
73-
7462
FormatInfo info = new FormatInfo();
75-
info.format = ImageFormat.OVA;
76-
info.filename = templateName + "." + ImageFormat.OVA.getFileExtension();
63+
info.format = ImageFormat.VMDK;
64+
info.filename = templateName + "." + ImageFormat.VMDK.getFileExtension();
7765
info.size = _storage.getSize(templateFilePath);
7866
info.virtualSize = getTemplateVirtualSize(templatePath, info.filename);
7967

80-
// delete original OVA file
81-
// templateFile.delete();
8268
return info;
8369
}
8470

@@ -94,56 +80,37 @@ public Long getVirtualSize(File file) {
9480
}
9581

9682
public long getTemplateVirtualSize(String templatePath, String templateName) throws InternalErrorException {
97-
// get the virtual size from the OVF file meta data
9883
long virtualSize = 0;
9984
String templateFileFullPath = templatePath.endsWith(File.separator) ? templatePath : templatePath + File.separator;
100-
templateFileFullPath += templateName.endsWith(ImageFormat.OVA.getFileExtension()) ? templateName : templateName + "." + ImageFormat.OVA.getFileExtension();
101-
String ovfFileName = getOVFFilePath(templateFileFullPath);
102-
if (ovfFileName == null) {
103-
String msg = "Unable to locate OVF file in template package directory: " + templatePath;
104-
s_logger.error(msg);
105-
throw new InternalErrorException(msg);
106-
}
85+
templateFileFullPath += templateName.endsWith(ImageFormat.VMDK.getFileExtension()) ? templateName : templateName + "." + ImageFormat.VMDK.getFileExtension();
86+
String vmdkHeader = "";
87+
10788
try {
108-
Document ovfDoc = null;
109-
ovfDoc = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(new File(ovfFileName));
110-
Element disk = (Element)ovfDoc.getElementsByTagName("Disk").item(0);
111-
virtualSize = Long.parseLong(disk.getAttribute("ovf:capacity"));
112-
String allocationUnits = disk.getAttribute("ovf:capacityAllocationUnits");
113-
if ((virtualSize != 0) && (allocationUnits != null)) {
114-
long units = 1;
115-
if (allocationUnits.equalsIgnoreCase("KB") || allocationUnits.equalsIgnoreCase("KiloBytes") || allocationUnits.equalsIgnoreCase("byte * 2^10")) {
116-
units = 1024;
117-
} else if (allocationUnits.equalsIgnoreCase("MB") || allocationUnits.equalsIgnoreCase("MegaBytes") || allocationUnits.equalsIgnoreCase("byte * 2^20")) {
118-
units = 1024 * 1024;
119-
} else if (allocationUnits.equalsIgnoreCase("GB") || allocationUnits.equalsIgnoreCase("GigaBytes") || allocationUnits.equalsIgnoreCase("byte * 2^30")) {
120-
units = 1024 * 1024 * 1024;
89+
FileReader fileReader = new FileReader(templateFileFullPath);
90+
BufferedReader bufferedReader = new BufferedReader(fileReader);
91+
Pattern regex = Pattern.compile("(RW|RDONLY|NOACCESS) (\\d+) (FLAT|SPARSE|ZERO|VMFS|VMFSSPARSE|VMFSDRM|VMFSRAW)");
92+
String line = null;
93+
while((line = bufferedReader.readLine()) != null) {
94+
Matcher m = regex.matcher(line);
95+
if (m.find( )) {
96+
long sectors = Long.parseLong(m.group(2));
97+
virtualSize = sectors * 512;
98+
break;
12199
}
122-
virtualSize = virtualSize * units;
123-
} else {
124-
throw new InternalErrorException("Failed to read capacity and capacityAllocationUnits from the OVF file: " + ovfFileName);
125100
}
126-
return virtualSize;
127-
} catch (Exception e) {
128-
String msg = "Unable to parse OVF XML document to get the virtual disk size due to" + e;
101+
bufferedReader.close();
102+
} catch(FileNotFoundException ex) {
103+
String msg = "Unable to open file '" + templateFileFullPath + "' " + ex.toString();
104+
s_logger.error(msg);
105+
throw new InternalErrorException(msg);
106+
} catch(IOException ex) {
107+
String msg = "Unable read open file '" + templateFileFullPath + "' " + ex.toString();
129108
s_logger.error(msg);
130109
throw new InternalErrorException(msg);
131110
}
132-
}
133111

134-
private String getOVFFilePath(String srcOVAFileName) {
135-
File file = new File(srcOVAFileName);
136-
assert (_storage != null);
137-
String[] files = _storage.listFiles(file.getParent());
138-
if (files != null) {
139-
for (String fileName : files) {
140-
if (fileName.toLowerCase().endsWith(".ovf")) {
141-
File ovfFile = new File(fileName);
142-
return file.getParent() + File.separator + ovfFile.getName();
143-
}
144-
}
145-
}
146-
return null;
112+
s_logger.debug("vmdk file had size="+virtualSize);
113+
return virtualSize;
147114
}
148115

149116
@Override

‎plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -189,6 +189,7 @@ public Answer copyTemplateToPrimaryStorage(CopyCommand cmd) {
189189
}
190190

191191
/* Copy volume to primary storage */
192+
s_logger.debug("Copying template to primary storage, template format is " + tmplVol.getFormat() );
192193
KVMStoragePool primaryPool = storagePoolMgr.getStoragePool(primaryStore.getPoolType(), primaryStore.getUuid());
193194

194195
KVMPhysicalDisk primaryVol = null;

‎plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -978,6 +978,7 @@ public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMSt
978978
String sourcePath = disk.getPath();
979979

980980
KVMPhysicalDisk newDisk;
981+
s_logger.debug("copyPhysicalDisk: disk size:" + disk.getSize() + ", virtualsize:" + disk.getVirtualSize()+" format:"+disk.getFormat());
981982
if (destPool.getType() != StoragePoolType.RBD) {
982983
if (disk.getFormat() == PhysicalDiskFormat.TAR) {
983984
newDisk = destPool.createPhysicalDisk(name, PhysicalDiskFormat.DIR, disk.getVirtualSize());
@@ -1015,7 +1016,8 @@ public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMSt
10151016
try {
10161017
Map<String, String> info = qemu.info(srcFile);
10171018
String backingFile = info.get(new String("backing_file"));
1018-
if (sourceFormat.equals(destFormat) && backingFile == null) {
1019+
// qcow2 templates can just be copied into place
1020+
if (sourceFormat.equals(destFormat) && backingFile == null && sourcePath.endsWith(".qcow2")) {
10191021
String result = Script.runSimpleBashScript("cp -f " + sourcePath + " " + destPath, timeout);
10201022
if (result != null) {
10211023
throw new CloudRuntimeException("Failed to create disk: " + result);

‎plugins/hypervisors/kvm/src/org/apache/cloudstack/utils/qemu/QemuImg.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -184,8 +184,9 @@ public void create(QemuImgFile file, Map<String, String> options) throws QemuImg
184184
public void convert(QemuImgFile srcFile, QemuImgFile destFile, Map<String, String> options) throws QemuImgException {
185185
Script s = new Script(_qemuImgPath, timeout);
186186
s.add("convert");
187-
s.add("-f");
188-
s.add(srcFile.getFormat().toString());
187+
// autodetect source format. Sometime int he future we may teach KVMPhysicalDisk about more formats, then we can explicitly pass them if necessary
188+
//s.add("-f");
189+
//s.add(srcFile.getFormat().toString());
189190
s.add("-O");
190191
s.add(destFile.getFormat().toString());
191192

@@ -350,4 +351,4 @@ public void resize(QemuImgFile file, long size, boolean delta) throws QemuImgExc
350351
public void resize(QemuImgFile file, long size) throws QemuImgException {
351352
this.resize(file, size, false);
352353
}
353-
}
354+
}

‎plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareStorageManagerImpl.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@
8080
import com.cloud.storage.Storage.ImageFormat;
8181
import com.cloud.storage.StorageLayer;
8282
import com.cloud.storage.Volume;
83-
import com.cloud.storage.template.VmdkProcessor;
83+
import com.cloud.storage.template.OVAProcessor;
8484
import com.cloud.utils.NumbersUtil;
8585
import com.cloud.utils.Pair;
8686
import com.cloud.utils.StringUtils;
@@ -649,10 +649,10 @@ private Ternary<String, Long, Long> createTemplateFromVolume(VirtualMachineMO vm
649649
clonedVm.exportVm(secondaryMountPoint + "/" + installPath, templateUniqueName, true, false);
650650

651651
long physicalSize = new File(installFullPath + "/" + templateUniqueName + ".ova").length();
652-
VmdkProcessor processor = new VmdkProcessor();
652+
OVAProcessor processor = new OVAProcessor();
653653
Map<String, Object> params = new HashMap<String, Object>();
654654
params.put(StorageLayer.InstanceConfigKey, _storage);
655-
processor.configure("VMDK Processor", params);
655+
processor.configure("OVA Processor", params);
656656
long virtualSize = processor.getTemplateVirtualSize(installFullPath, templateUniqueName);
657657

658658
postCreatePrivateTemplate(installFullPath, templateId, templateUniqueName, physicalSize, virtualSize);
@@ -771,11 +771,11 @@ private Ternary<String, Long, Long> createTemplateFromSnapshot(long accountId, l
771771
}
772772

773773
long physicalSize = new File(installFullPath + "/" + templateVMDKName).length();
774-
VmdkProcessor processor = new VmdkProcessor();
774+
OVAProcessor processor = new OVAProcessor();
775775
// long physicalSize = new File(installFullPath + "/" + templateUniqueName + ".ova").length();
776776
Map<String, Object> params = new HashMap<String, Object>();
777777
params.put(StorageLayer.InstanceConfigKey, _storage);
778-
processor.configure("VMDK Processor", params);
778+
processor.configure("OVA Processor", params);
779779
long virtualSize = processor.getTemplateVirtualSize(installFullPath, templateUniqueName);
780780

781781
postCreatePrivateTemplate(installFullPath, templateId, templateUniqueName, physicalSize, virtualSize);

0 commit comments

Comments
 (0)