Skip to content

Commit 9ad54a0

Browse files
author
Marcus Sorensen
committed
Summary: KVM - use virtio socket to communicate config to system vms
Detail: This gets rid of the patchdisk method of passing cmdline and authorized_keys to KVM system VMs. It instead passes them to a virtio socket, which the KVM guest reads from the character device /dev/vport0p1 during cloud-early-config. Tested to work on CentOS 6.3 and Ubuntu 12.04. Should work with even older versions of libvirt. Signed-off-by: Marcus Sorensen <marcus@betterservers.com> 1362691685 -0700
1 parent c9ee055 commit 9ad54a0

7 files changed

Lines changed: 87 additions & 247 deletions

File tree

‎debian/control‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ Description: CloudStack server library
2222

2323
Package: cloudstack-agent
2424
Architecture: all
25-
Depends: openjdk-6-jre | openjdk-7-jre, cloudstack-common (= ${source:Version}), lsb-base (>= 3.2), libcommons-daemon-java, libjna-java, openssh-client, libvirt0, sysvinit-utils, chkconfig, qemu-kvm, libvirt-bin, uuid-runtime, rsync, grep, iproute, ebtables, vlan, wget, jsvc
25+
Depends: openjdk-6-jre | openjdk-7-jre, cloudstack-common (= ${source:Version}), lsb-base (>= 3.2), libcommons-daemon-java, libjna-java, openssh-client, libvirt0, sysvinit-utils, chkconfig, qemu-kvm, libvirt-bin, uuid-runtime, rsync, grep, iproute, perl-base, perl-modules, ebtables, vlan, wget, jsvc
2626
Conflicts: cloud-agent, cloud-agent-libs, cloud-agent-deps, cloud-agent-scripts
2727
Description: CloudStack agent
2828
The CloudStack agent is in charge of managing shared computing resources in

‎packaging/centos63/cloud.spec‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,7 @@ Requires: ebtables
116116
Requires: jsvc
117117
Requires: jakarta-commons-daemon
118118
Requires: jakarta-commons-daemon-jsvc
119+
Requires: perl
119120
Provides: cloud-agent
120121
Obsoletes: cloud-agent < 4.1.0
121122
Obsoletes: cloud-test < 4.1.0

‎patches/systemvm/debian/config/etc/init.d/cloud-early-config‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -108,14 +108,17 @@ get_boot_params() {
108108
sed -i "s/%/ /g" /var/cache/cloud/cmdline
109109
;;
110110
kvm)
111-
# KVM needs to mount another disk, to get cmdline
112-
mkdir -p $EXTRA_MOUNT
113-
mount /dev/vdb $EXTRA_MOUNT
114-
cp -f $EXTRA_MOUNT/cmdline /var/cache/cloud/cmdline
115-
cp -f $EXTRA_MOUNT/authorized_keys /var/cache/cloud/authorized_keys
116-
privkey=/var/cache/cloud/authorized_keys
117-
umount $EXTRA_MOUNT
118-
cp -f $privkey /root/.ssh/ && chmod go-rwx /root/.ssh/authorized_keys
111+
while read line; do
112+
if [[ $line == cmdline:* ]]; then
113+
cmd=${line//cmdline:/}
114+
echo $cmd > /var/cache/cloud/cmdline
115+
elif [[ $line == pubkey:* ]]; then
116+
pubkey=${line//pubkey:/}
117+
echo $pubkey > /var/cache/cloud/authorized_keys
118+
echo $pubkey > /root/.ssh/authorized_keys
119+
fi
120+
done < /dev/vport0p1
121+
chmod go-rwx /root/.ssh/authorized_keys
119122
;;
120123
vmware)
121124
vmtoolsd --cmd 'machine.id.get' > /var/cache/cloud/cmdline

‎plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java‎

Lines changed: 14 additions & 89 deletions
Original file line numberDiff line numberDiff line change
@@ -255,7 +255,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements
255255

256256
private String _modifyVlanPath;
257257
private String _versionstringpath;
258-
private String _patchdomrPath;
258+
private String _patchViaSocketPath;
259259
private String _createvmPath;
260260
private String _manageSnapshotPath;
261261
private String _resizeVolumePath;
@@ -521,10 +521,10 @@ public boolean configure(String name, Map<String, Object> params)
521521
throw new ConfigurationException("Unable to find versions.sh");
522522
}
523523

524-
_patchdomrPath = Script.findScript(kvmScriptsDir + "/patch/",
525-
"rundomrpre.sh");
526-
if (_patchdomrPath == null) {
527-
throw new ConfigurationException("Unable to find rundomrpre.sh");
524+
_patchViaSocketPath = Script.findScript(kvmScriptsDir + "/patch/",
525+
"patchviasocket.pl");
526+
if (_patchViaSocketPath == null) {
527+
throw new ConfigurationException("Unable to find patchviasocket.pl");
528528
}
529529

530530
_heartBeatPath = Script.findScript(kvmScriptsDir, "kvmheartbeat.sh");
@@ -1014,13 +1014,11 @@ private String getVnetId(String vnetId) {
10141014
return vnetId;
10151015
}
10161016

1017-
private void patchSystemVm(String cmdLine, String dataDiskPath,
1018-
String vmName) throws InternalErrorException {
1017+
private void passCmdLine(String vmName, String cmdLine)
1018+
throws InternalErrorException {
1019+
final Script command = new Script(_patchViaSocketPath, _timeout, s_logger);
10191020
String result;
1020-
final Script command = new Script(_patchdomrPath, _timeout, s_logger);
1021-
command.add("-l", vmName);
1022-
command.add("-t", "all");
1023-
command.add("-d", dataDiskPath);
1021+
command.add("-n",vmName);
10241022
command.add("-p", cmdLine.replaceAll(" ", "%"));
10251023
result = command.execute();
10261024
if (result != null) {
@@ -1460,24 +1458,6 @@ public Answer execute(DestroyCommand cmd) {
14601458
pool.deletePhysicalDisk(vol.getPath());
14611459
String vmName = cmd.getVmName();
14621460
String poolPath = pool.getLocalPath();
1463-
1464-
/* if vol is a root disk for a system vm, try to remove accompanying patch disk as well
1465-
this is a bit tricky since the patchdisk is only a LibvirtComputingResource construct
1466-
and not tracked anywhere in cloudstack */
1467-
if (vol.getType() == Volume.Type.ROOT && vmName.matches("^[rsv]-\\d+-.+$")) {
1468-
File patchVbd = new File(poolPath + File.separator + vmName + "-patchdisk");
1469-
if(patchVbd.exists()){
1470-
try {
1471-
_storagePoolMgr.deleteVbdByPath(vol.getPoolType(),patchVbd.getAbsolutePath());
1472-
} catch(CloudRuntimeException e) {
1473-
s_logger.warn("unable to destroy patch disk '" + patchVbd.getAbsolutePath() +
1474-
"' while removing root disk for " + vmName + " : " + e);
1475-
}
1476-
} else {
1477-
s_logger.debug("file '" +patchVbd.getAbsolutePath()+ "' not found");
1478-
}
1479-
}
1480-
14811461
return new Answer(cmd, true, "Success");
14821462
} catch (CloudRuntimeException e) {
14831463
s_logger.debug("Failed to delete volume: " + e.toString());
@@ -3121,6 +3101,11 @@ protected synchronized StartAnswer execute(StartCommand cmd) {
31213101
}
31223102
}
31233103

3104+
// pass cmdline info to system vms
3105+
if (vmSpec.getType() != VirtualMachine.Type.User) {
3106+
passCmdLine(vmName, vmSpec.getBootArgs() );
3107+
}
3108+
31243109
state = State.Running;
31253110
return new StartAnswer(cmd);
31263111
} catch (LibvirtException e) {
@@ -3248,8 +3233,6 @@ public int compare(VolumeTO arg0, VolumeTO arg1) {
32483233
iso.defISODisk(_sysvmISOPath);
32493234
vm.getDevices().addDevice(iso);
32503235
}
3251-
3252-
createPatchVbd(conn, vmName, vm, vmSpec);
32533236
}
32543237
}
32553238

@@ -3263,64 +3246,6 @@ private VolumeTO getVolume(VirtualMachineTO vmSpec, Volume.Type type) {
32633246
return null;
32643247
}
32653248

3266-
private void createPatchVbd(Connect conn, String vmName, LibvirtVMDef vm,
3267-
VirtualMachineTO vmSpec) throws LibvirtException,
3268-
InternalErrorException {
3269-
3270-
List<DiskDef> disks = vm.getDevices().getDisks();
3271-
DiskDef rootDisk = disks.get(0);
3272-
VolumeTO rootVol = getVolume(vmSpec, Volume.Type.ROOT);
3273-
String patchName = vmName + "-patchdisk";
3274-
KVMStoragePool pool = _storagePoolMgr.getStoragePool(
3275-
rootVol.getPoolType(),
3276-
rootVol.getPoolUuid());
3277-
String patchDiskPath = pool.getLocalPath() + "/" + patchName;
3278-
3279-
List<KVMPhysicalDisk> phyDisks = pool.listPhysicalDisks();
3280-
boolean foundDisk = false;
3281-
3282-
for (KVMPhysicalDisk phyDisk : phyDisks) {
3283-
if (phyDisk.getPath().equals(patchDiskPath)) {
3284-
foundDisk = true;
3285-
break;
3286-
}
3287-
}
3288-
3289-
if (!foundDisk) {
3290-
s_logger.debug("generating new patch disk for " + vmName + " since none was found");
3291-
KVMPhysicalDisk disk = pool.createPhysicalDisk(patchName, KVMPhysicalDisk.PhysicalDiskFormat.RAW,
3292-
10L * 1024 * 1024);
3293-
} else {
3294-
s_logger.debug("found existing patch disk at " + patchDiskPath + " using it for " + vmName);
3295-
}
3296-
3297-
/* Format/create fs on this disk */
3298-
final Script command = new Script(_createvmPath, _timeout, s_logger);
3299-
command.add("-f", patchDiskPath);
3300-
String result = command.execute();
3301-
if (result != null) {
3302-
s_logger.debug("Failed to create data disk: " + result);
3303-
throw new InternalErrorException("Failed to create data disk: "
3304-
+ result);
3305-
}
3306-
3307-
/* add patch disk */
3308-
DiskDef patchDisk = new DiskDef();
3309-
3310-
if (pool.getType() == StoragePoolType.CLVM) {
3311-
patchDisk.defBlockBasedDisk(patchDiskPath, 1, rootDisk.getBusType());
3312-
} else {
3313-
patchDisk.defFileBasedDisk(patchDiskPath, 1, rootDisk.getBusType(),
3314-
DiskDef.diskFmtType.RAW);
3315-
}
3316-
3317-
disks.add(patchDisk);
3318-
3319-
String bootArgs = vmSpec.getBootArgs();
3320-
3321-
patchSystemVm(bootArgs, patchDiskPath, vmName);
3322-
}
3323-
33243249
private void createVif(LibvirtVMDef vm, NicTO nic)
33253250
throws InternalErrorException, LibvirtException {
33263251
vm.getDevices().addDevice(

‎plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -864,8 +864,8 @@ public String toString() {
864864
virtioSerialBuilder.append("<channel type='unix'>\n");
865865
virtioSerialBuilder.append("<source mode='bind' path='" + _path
866866
+ "/" + _name + ".agent'/>\n");
867-
virtioSerialBuilder.append("<target type='virtio' name='org.qemu.guest_agent.0'/>\n");
868-
virtioSerialBuilder.append("<address type='virtio-serial' controller='0' bus='0' port='1'/>\n");
867+
virtioSerialBuilder.append("<target type='virtio' name='" + _name + ".vport'/>\n");
868+
virtioSerialBuilder.append("<address type='virtio-serial'/>\n");
869869
virtioSerialBuilder.append("</channel>\n");
870870
return virtioSerialBuilder.toString();
871871
}

‎scripts/vm/hypervisor/kvm/patchviasocket.pl‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
#!/usr/bin/perl -w
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
19+
#############################################################
20+
# This script connects to the system vm socket and writes the
21+
# authorized_keys and cmdline data to it. The system VM then
22+
# reads it from /dev/vport0p1 in cloud_early_config
23+
#############################################################
24+
25+
use strict;
26+
use Getopt::Std;
27+
use IO::Socket;
28+
$|=1;
29+
30+
my $opts = {};
31+
getopt('pn',$opts);
32+
my $name = $opts->{n};
33+
my $cmdline = $opts->{p};
34+
my $sockfile = "/var/lib/libvirt/qemu/$name.agent";
35+
my $pubkeyfile = "/root/.ssh/id_rsa.pub.cloud";
36+
37+
if (! -S $sockfile) {
38+
print "ERROR: $sockfile socket not found\n";
39+
exit 1;
40+
}
41+
42+
if (! -f $pubkeyfile) {
43+
print "ERROR: ssh public key not found on host at $pubkeyfile\n";
44+
exit 1;
45+
}
46+
47+
open(FILE,$pubkeyfile) or die "ERROR: unable to open $pubkeyfile - $^E";
48+
my $key = <FILE>;
49+
close FILE;
50+
51+
$cmdline =~ s/%/ /g;
52+
my $msg = "pubkey:" . $key . "\ncmdline:" . $cmdline;
53+
54+
my $socket = IO::Socket::UNIX->new(Peer=>$sockfile,Type=>SOCK_STREAM)
55+
or die "ERROR: unable to connect to $sockfile - $^E\n";
56+
print $socket "$msg\r\n";
57+
close $socket;
58+

0 commit comments

Comments
 (0)