Skip to content

Commit 3872bf1

Browse files
davidjumanidavidjumani
authored
kvm: Enable PVLAN support on L2 networks (apache#4040)
This is an extention of apache#3732 for kvm. This is restricted to ovs > 2.9.2 Since Xen uses ovs 2.6, pvlan is unsupported. This also fixes issues of vms on the same pvlan unable to communicate if they're on the same host
1 parent 400641b commit 3872bf1

17 files changed

Lines changed: 636 additions & 135 deletions

File tree

‎agent/src/main/java/com/cloud/agent/Agent.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -811,7 +811,10 @@ public void processResponse(final Response response, final Link link) {
811811
public void processReadyCommand(final Command cmd) {
812812
final ReadyCommand ready = (ReadyCommand)cmd;
813813
// Set human readable sizes;
814-
NumbersUtil.enableHumanReadableSizes = ready.getEnableHumanReadableSizes();
814+
Boolean humanReadable = ready.getEnableHumanReadableSizes();
815+
if (humanReadable != null){
816+
NumbersUtil.enableHumanReadableSizes = humanReadable;
817+
}
815818

816819
s_logger.info("Processing agent ready command, agent id = " + ready.getHostId());
817820
if (ready.getHostId() != null) {

‎api/src/main/java/com/cloud/agent/api/PvlanSetupCommand.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ public enum Type {
3434
private String dhcpIp;
3535
private Type type;
3636
private String networkTag;
37+
private String pvlanType;
3738

3839
protected PvlanSetupCommand() {
3940
}
@@ -43,6 +44,7 @@ protected PvlanSetupCommand(Type type, String op, URI uri, String networkTag) {
4344
this.op = op;
4445
this.primary = NetUtils.getPrimaryPvlanFromUri(uri);
4546
this.isolated = NetUtils.getIsolatedPvlanFromUri(uri);
47+
this.pvlanType = NetUtils.getPvlanTypeFromUri(uri);
4648
this.networkTag = networkTag;
4749
}
4850

@@ -116,4 +118,8 @@ public void setDhcpName(String dhcpName) {
116118
public String getNetworkTag() {
117119
return networkTag;
118120
}
121+
122+
public String getPvlanType() {
123+
return pvlanType;
124+
}
119125
}

‎engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3756,6 +3756,7 @@ private NicProfile orchestrateAddVmToNetwork(final VirtualMachine vm, final Netw
37563756
try {
37573757
result = plugNic(network, nicTO, vmTO, context, dest);
37583758
if (result) {
3759+
_userVmMgr.setupVmForPvlan(true, vm.getHostId(), nic);
37593760
s_logger.debug("Nic is plugged successfully for vm " + vm + " in network " + network + ". Vm is a part of network now");
37603761
final long isDefault = nic.isDefaultNic() ? 1 : 0;
37613762
// insert nic's Id into DB as resource_name
@@ -3863,6 +3864,7 @@ private boolean orchestrateRemoveNicFromVm(final VirtualMachine vm, final Nic ni
38633864
s_logger.debug("Un-plugging nic " + nic + " for vm " + vm + " from network " + network);
38643865
final boolean result = unplugNic(network, nicTO, vmTO, context, dest);
38653866
if (result) {
3867+
_userVmMgr.setupVmForPvlan(false, vm.getHostId(), nicProfile);
38663868
s_logger.debug("Nic is unplugged successfully for vm " + vm + " in network " + network);
38673869
final long isDefault = nic.isDefaultNic() ? 1 : 0;
38683870
UsageEventUtils.publishUsageEvent(EventTypes.EVENT_NETWORK_OFFERING_REMOVE, vm.getAccountId(), vm.getDataCenterId(), vm.getId(),

‎engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java‎

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@
1616
// under the License.
1717
package org.apache.cloudstack.engine.orchestration;
1818

19-
2019
import java.net.URI;
2120
import java.util.ArrayList;
2221
import java.util.Arrays;
@@ -215,6 +214,7 @@
215214
import com.cloud.vm.NicVO;
216215
import com.cloud.vm.ReservationContext;
217216
import com.cloud.vm.ReservationContextImpl;
217+
import com.cloud.vm.UserVmManager;
218218
import com.cloud.vm.UserVmVO;
219219
import com.cloud.vm.VMInstanceVO;
220220
import com.cloud.vm.VirtualMachine;
@@ -299,6 +299,8 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
299299
RemoteAccessVpnDao _remoteAccessVpnDao;
300300
@Inject
301301
VpcVirtualNetworkApplianceService _routerService;
302+
@Inject
303+
UserVmManager _userVmMgr;
302304

303305
List<NetworkGuru> networkGurus;
304306

@@ -1792,6 +1794,11 @@ public void prepareNicForMigration(final VirtualMachineProfile vm, final DeployD
17921794
s_logger.error("NetworkGuru " + guru + " prepareForMigration failed."); // XXX: Transaction error
17931795
}
17941796
}
1797+
1798+
if (network.getGuestType() == Network.GuestType.L2 && vm.getType() == VirtualMachine.Type.User) {
1799+
_userVmMgr.setupVmForPvlan(false, vm.getVirtualMachine().getHostId(), profile);
1800+
}
1801+
17951802
final List<Provider> providersToImplement = getNetworkProviders(network.getId());
17961803
for (final NetworkElement element : networkElements) {
17971804
if (providersToImplement.contains(element.getProvider())) {
@@ -1912,6 +1919,11 @@ public void commitNicForMigration(final VirtualMachineProfile src, final Virtual
19121919
if (guru instanceof NetworkMigrationResponder) {
19131920
((NetworkMigrationResponder)guru).commitMigration(nicSrc, network, src, src_context, dst_context);
19141921
}
1922+
1923+
if (network.getGuestType() == Network.GuestType.L2 && src.getType() == VirtualMachine.Type.User) {
1924+
_userVmMgr.setupVmForPvlan(true, src.getVirtualMachine().getHostId(), nicSrc);
1925+
}
1926+
19151927
final List<Provider> providersToImplement = getNetworkProviders(network.getId());
19161928
for (final NetworkElement element : networkElements) {
19171929
if (providersToImplement.contains(element.getProvider())) {
@@ -1943,6 +1955,11 @@ public void rollbackNicForMigration(final VirtualMachineProfile src, final Virtu
19431955
if (guru instanceof NetworkMigrationResponder) {
19441956
((NetworkMigrationResponder)guru).rollbackMigration(nicDst, network, dst, src_context, dst_context);
19451957
}
1958+
1959+
if (network.getGuestType() == Network.GuestType.L2 && src.getType() == VirtualMachine.Type.User) {
1960+
_userVmMgr.setupVmForPvlan(true, dst.getVirtualMachine().getHostId(), nicDst);
1961+
}
1962+
19461963
final List<Provider> providersToImplement = getNetworkProviders(network.getId());
19471964
for (final NetworkElement element : networkElements) {
19481965
if (providersToImplement.contains(element.getProvider())) {
@@ -2498,6 +2515,12 @@ public Network doInTransaction(final TransactionStatus status) {
24982515
} else {
24992516
uri = BroadcastDomainType.fromString(vlanIdFinal);
25002517
}
2518+
2519+
if (_networksDao.listByPhysicalNetworkPvlan(physicalNetworkId, uri.toString()).size() > 0) {
2520+
throw new InvalidParameterValueException("Network with vlan " + vlanIdFinal +
2521+
" already exists or overlaps with other network pvlans in zone " + zoneId);
2522+
}
2523+
25012524
userNetwork.setBroadcastUri(uri);
25022525
if (!vlanIdFinal.equalsIgnoreCase(Vlan.UNTAGGED)) {
25032526
userNetwork.setBroadcastDomainType(BroadcastDomainType.Vlan);
@@ -2508,7 +2531,7 @@ public Network doInTransaction(final TransactionStatus status) {
25082531
if (vlanIdFinal.equalsIgnoreCase(Vlan.UNTAGGED)) {
25092532
throw new InvalidParameterValueException("Cannot support pvlan with untagged primary vlan!");
25102533
}
2511-
URI uri = NetUtils.generateUriForPvlan(vlanIdFinal, isolatedPvlan);
2534+
URI uri = NetUtils.generateUriForPvlan(vlanIdFinal, isolatedPvlan, isolatedPvlanType.toString());
25122535
if (_networksDao.listByPhysicalNetworkPvlan(physicalNetworkId, uri.toString(), isolatedPvlanType).size() > 0) {
25132536
throw new InvalidParameterValueException("Network with primary vlan " + vlanIdFinal +
25142537
" and secondary vlan " + isolatedPvlan + " type " + isolatedPvlanType +

‎engine/schema/src/main/java/com/cloud/network/dao/NetworkDao.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,4 +126,6 @@ public interface NetworkDao extends GenericDao<NetworkVO, Long>, StateDao<State,
126126
List<NetworkVO> listByAccountIdNetworkName(long accountId, String name);
127127

128128
List<NetworkVO> listByPhysicalNetworkPvlan(long physicalNetworkId, String broadcastUri, Network.PVlanType pVlanType);
129+
130+
List<NetworkVO> listByPhysicalNetworkPvlan(long physicalNetworkId, String broadcastUri);
129131
}

‎engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java‎

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -740,6 +740,7 @@ public List<NetworkVO> listByAccountIdNetworkName(final long accountId, final St
740740
* - The requested exact PVLAN pair exists
741741
* - The requested secondary VLAN ID is secondary VLAN ID of an existing PVLAN pair
742742
* - The requested secondary VLAN ID is primary VLAN ID of an existing PVLAN pair
743+
* - The requested primary VLAN ID is secondary VLAN ID of an existing PVLAN pair
743744
*/
744745
protected boolean isNetworkOverlappingRequestedPvlan(Integer existingPrimaryVlan, Integer existingSecondaryVlan, Network.PVlanType existingPvlanType,
745746
Integer requestedPrimaryVlan, Integer requestedSecondaryVlan, Network.PVlanType requestedPvlanType) {
@@ -749,14 +750,20 @@ protected boolean isNetworkOverlappingRequestedPvlan(Integer existingPrimaryVlan
749750
}
750751
boolean exactMatch = existingPrimaryVlan.equals(requestedPrimaryVlan) && existingSecondaryVlan.equals(requestedSecondaryVlan);
751752
boolean secondaryVlanUsed = requestedPvlanType != Network.PVlanType.Promiscuous && requestedSecondaryVlan.equals(existingPrimaryVlan) || requestedSecondaryVlan.equals(existingSecondaryVlan);
753+
boolean primaryVlanUsed = existingPvlanType != Network.PVlanType.Promiscuous && requestedPrimaryVlan.equals(existingSecondaryVlan);
752754
boolean isolatedMax = false;
753755
boolean promiscuousMax = false;
754756
if (requestedPvlanType == Network.PVlanType.Isolated && existingPrimaryVlan.equals(requestedPrimaryVlan) && existingPvlanType.equals(Network.PVlanType.Isolated)) {
755757
isolatedMax = true;
756758
} else if (requestedPvlanType == Network.PVlanType.Promiscuous && existingPrimaryVlan.equals(requestedPrimaryVlan) && existingPvlanType == Network.PVlanType.Promiscuous) {
757759
promiscuousMax = true;
758760
}
759-
return exactMatch || secondaryVlanUsed || isolatedMax || promiscuousMax;
761+
return exactMatch || secondaryVlanUsed || primaryVlanUsed || isolatedMax || promiscuousMax;
762+
}
763+
764+
// True when a VLAN ID overlaps with an existing PVLAN primary or secondary ID
765+
protected boolean isNetworkOverlappingRequestedPvlan(Integer existingPrimaryVlan, Integer existingSecondaryVlan, Integer requestedVlan) {
766+
return requestedVlan.equals(existingPrimaryVlan) || requestedVlan.equals(existingSecondaryVlan);
760767
}
761768

762769
protected Network.PVlanType getNetworkPvlanType(long networkId, List<Integer> existingPvlan) {
@@ -770,6 +777,38 @@ protected Network.PVlanType getNetworkPvlanType(long networkId, List<Integer> ex
770777
return existingPvlanType;
771778
}
772779

780+
@Override
781+
public List<NetworkVO> listByPhysicalNetworkPvlan(long physicalNetworkId, String broadcastUri) {
782+
final URI searchUri = BroadcastDomainType.fromString(broadcastUri);
783+
if (!searchUri.getScheme().equalsIgnoreCase("vlan")) {
784+
throw new CloudRuntimeException("VLAN requested but URI is not in the expected format: " + searchUri.toString());
785+
}
786+
final String searchRange = BroadcastDomainType.getValue(searchUri);
787+
final List<Integer> searchVlans = UriUtils.expandVlanUri(searchRange);
788+
final List<NetworkVO> overlappingNetworks = new ArrayList<>();
789+
790+
final SearchCriteria<NetworkVO> sc = PhysicalNetworkSearch.create();
791+
sc.setParameters("physicalNetworkId", physicalNetworkId);
792+
793+
for (final NetworkVO network : listBy(sc)) {
794+
if (network.getBroadcastUri() == null || !network.getBroadcastUri().getScheme().equalsIgnoreCase("pvlan")) {
795+
continue;
796+
}
797+
// Ensure existing and proposed VLAN don't overlap
798+
final String networkVlanRange = BroadcastDomainType.getValue(network.getBroadcastUri());
799+
if (networkVlanRange == null || networkVlanRange.isEmpty()) {
800+
continue;
801+
}
802+
List<Integer> existingPvlan = UriUtils.expandPvlanUri(networkVlanRange);
803+
if (isNetworkOverlappingRequestedPvlan(existingPvlan.get(0), existingPvlan.get(1), searchVlans.get(0))) {
804+
overlappingNetworks.add(network);
805+
break;
806+
}
807+
}
808+
809+
return overlappingNetworks;
810+
}
811+
773812
@Override
774813
public List<NetworkVO> listByPhysicalNetworkPvlan(long physicalNetworkId, String broadcastUri, Network.PVlanType pVlanType) {
775814
final URI searchUri = BroadcastDomainType.fromString(broadcastUri);

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -798,14 +798,14 @@ public boolean configure(final String name, final Map<String, Object> params) th
798798
throw new ConfigurationException("Unable to find the router_proxy.sh");
799799
}
800800

801-
_ovsPvlanDhcpHostPath = Script.findScript(networkScriptsDir, "ovs-pvlan-dhcp-host.sh");
801+
_ovsPvlanDhcpHostPath = Script.findScript(networkScriptsDir, "ovs-pvlan-kvm-dhcp-host.sh");
802802
if (_ovsPvlanDhcpHostPath == null) {
803-
throw new ConfigurationException("Unable to find the ovs-pvlan-dhcp-host.sh");
803+
throw new ConfigurationException("Unable to find the ovs-pvlan-kvm-dhcp-host.sh");
804804
}
805805

806-
_ovsPvlanVmPath = Script.findScript(networkScriptsDir, "ovs-pvlan-vm.sh");
806+
_ovsPvlanVmPath = Script.findScript(networkScriptsDir, "ovs-pvlan-kvm-vm.sh");
807807
if (_ovsPvlanVmPath == null) {
808-
throw new ConfigurationException("Unable to find the ovs-pvlan-vm.sh");
808+
throw new ConfigurationException("Unable to find the ovs-pvlan-kvm-vm.sh");
809809
}
810810

811811
String value = (String)params.get("developer");

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPvlanSetupCommandWrapper.java‎

Lines changed: 29 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -19,22 +19,17 @@
1919

2020
package com.cloud.hypervisor.kvm.resource.wrapper;
2121

22-
import java.util.List;
23-
2422
import org.apache.log4j.Logger;
2523
import org.joda.time.Duration;
26-
import org.libvirt.Connect;
27-
import org.libvirt.LibvirtException;
2824

2925
import com.cloud.agent.api.Answer;
3026
import com.cloud.agent.api.PvlanSetupCommand;
3127
import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource;
32-
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.InterfaceDef;
3328
import com.cloud.resource.CommandWrapper;
3429
import com.cloud.resource.ResourceWrapper;
3530
import com.cloud.utils.script.Script;
3631

37-
@ResourceWrapper(handles = PvlanSetupCommand.class)
32+
@ResourceWrapper(handles = PvlanSetupCommand.class)
3833
public final class LibvirtPvlanSetupCommandWrapper extends CommandWrapper<PvlanSetupCommand, Answer, LibvirtComputingResource> {
3934

4035
private static final Logger s_logger = Logger.getLogger(LibvirtPvlanSetupCommandWrapper.class);
@@ -43,66 +38,50 @@ public final class LibvirtPvlanSetupCommandWrapper extends CommandWrapper<PvlanS
4338
public Answer execute(final PvlanSetupCommand command, final LibvirtComputingResource libvirtComputingResource) {
4439
final String primaryPvlan = command.getPrimary();
4540
final String isolatedPvlan = command.getIsolated();
41+
final String pvlanType = "-" + command.getPvlanType();
4642
final String op = command.getOp();
47-
final String dhcpName = command.getDhcpName();
4843
final String dhcpMac = command.getDhcpMac();
49-
final String vmMac = command.getVmMac();
44+
final String vmMac = command.getVmMac() == null ? dhcpMac : command.getVmMac();
5045
final String dhcpIp = command.getDhcpIp();
51-
boolean add = true;
5246

5347
String opr = "-A";
5448
if (op.equals("delete")) {
5549
opr = "-D";
56-
add = false;
5750
}
5851

5952
String result = null;
60-
try {
61-
final String guestBridgeName = libvirtComputingResource.getGuestBridgeName();
62-
final Duration timeout = libvirtComputingResource.getTimeout();
63-
64-
if (command.getType() == PvlanSetupCommand.Type.DHCP) {
65-
final String ovsPvlanDhcpHostPath = libvirtComputingResource.getOvsPvlanDhcpHostPath();
66-
final Script script = new Script(ovsPvlanDhcpHostPath, timeout, s_logger);
67-
68-
if (add) {
69-
final LibvirtUtilitiesHelper libvirtUtilitiesHelper = libvirtComputingResource.getLibvirtUtilitiesHelper();
70-
final Connect conn = libvirtUtilitiesHelper.getConnectionByVmName(dhcpName);
71-
72-
final List<InterfaceDef> ifaces = libvirtComputingResource.getInterfaces(conn, dhcpName);
73-
final InterfaceDef guestNic = ifaces.get(0);
74-
script.add(opr, "-b", guestBridgeName, "-p", primaryPvlan, "-i", isolatedPvlan, "-n", dhcpName, "-d", dhcpIp, "-m", dhcpMac, "-I",
75-
guestNic.getDevName());
76-
} else {
77-
script.add(opr, "-b", guestBridgeName, "-p", primaryPvlan, "-i", isolatedPvlan, "-n", dhcpName, "-d", dhcpIp, "-m", dhcpMac);
78-
}
7953

80-
result = script.execute();
54+
final String guestBridgeName = libvirtComputingResource.getGuestBridgeName();
55+
final Duration timeout = libvirtComputingResource.getTimeout();
8156

82-
if (result != null) {
83-
s_logger.warn("Failed to program pvlan for dhcp server with mac " + dhcpMac);
84-
return new Answer(command, false, result);
85-
} else {
86-
s_logger.info("Programmed pvlan for dhcp server with mac " + dhcpMac);
87-
}
88-
} else if (command.getType() == PvlanSetupCommand.Type.VM) {
89-
final String ovsPvlanVmPath = libvirtComputingResource.getOvsPvlanVmPath();
57+
if (command.getType() == PvlanSetupCommand.Type.DHCP) {
58+
final String ovsPvlanDhcpHostPath = libvirtComputingResource.getOvsPvlanDhcpHostPath();
59+
final Script script = new Script(ovsPvlanDhcpHostPath, timeout, s_logger);
9060

91-
final Script script = new Script(ovsPvlanVmPath, timeout, s_logger);
92-
script.add(opr, "-b", guestBridgeName, "-p", primaryPvlan, "-i", isolatedPvlan, "-v", vmMac);
93-
result = script.execute();
61+
script.add(opr, pvlanType, "-b", guestBridgeName, "-p", primaryPvlan, "-s", isolatedPvlan, "-m", dhcpMac,
62+
"-d", dhcpIp);
63+
result = script.execute();
9464

95-
if (result != null) {
96-
s_logger.warn("Failed to program pvlan for vm with mac " + vmMac);
97-
return new Answer(command, false, result);
98-
} else {
99-
s_logger.info("Programmed pvlan for vm with mac " + vmMac);
100-
}
65+
if (result != null) {
66+
s_logger.warn("Failed to program pvlan for dhcp server with mac " + dhcpMac);
67+
} else {
68+
s_logger.info("Programmed pvlan for dhcp server with mac " + dhcpMac);
10169
}
102-
} catch (final LibvirtException e) {
103-
s_logger.error("Error whislt executing OVS Setup command! ==> " + e.getMessage());
104-
return new Answer(command, false, e.getMessage());
10570
}
71+
72+
// We run this even for DHCP servers since they're all vms after all
73+
final String ovsPvlanVmPath = libvirtComputingResource.getOvsPvlanVmPath();
74+
final Script script = new Script(ovsPvlanVmPath, timeout, s_logger);
75+
script.add(opr, pvlanType, "-b", guestBridgeName, "-p", primaryPvlan, "-s", isolatedPvlan, "-m", vmMac);
76+
result = script.execute();
77+
78+
if (result != null) {
79+
s_logger.warn("Failed to program pvlan for vm with mac " + vmMac);
80+
return new Answer(command, false, result);
81+
} else {
82+
s_logger.info("Programmed pvlan for vm with mac " + vmMac);
83+
}
84+
10685
return new Answer(command, true, result);
10786
}
10887
}

0 commit comments

Comments
 (0)