Skip to content

Commit 691be5c

Browse files
author
kishan
committed
bug CS-15577: Added per gateway network usage for VPC and VPN usage. VPN usage uses 525 mark for outgoing traffic and 524 mark for incoming traffic
status CS-15577: resolved fixed
1 parent a26de19 commit 691be5c

8 files changed

Lines changed: 461 additions & 232 deletions

File tree

‎api/src/com/cloud/agent/api/NetworkUsageCommand.java‎

Lines changed: 37 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,26 +17,27 @@
1717
package com.cloud.agent.api;
1818

1919
import com.cloud.agent.api.LogLevel.Log4jLevel;
20-
import com.cloud.agent.api.to.NicTO;
2120

2221
@LogLevel(Log4jLevel.Trace)
2322
public class NetworkUsageCommand extends Command {
2423
private String privateIP;
2524
private String domRName;
2625
private String option;
2726
boolean forVpc = false;
28-
NicTO guestNic;
27+
private String gatewayIP;
28+
private String vpcCIDR;
2929

3030
protected NetworkUsageCommand() {
3131

3232
}
3333

34-
public NetworkUsageCommand(String privateIP, String domRName, boolean forVpc, NicTO guestNic)
34+
public NetworkUsageCommand(String privateIP, String domRName, boolean forVpc, String gatewayIP)
3535
{
3636
this.privateIP = privateIP;
3737
this.domRName = domRName;
3838
this.forVpc = forVpc;
39-
this.guestNic = guestNic;
39+
this.gatewayIP = gatewayIP;
40+
this.option = "get";
4041
}
4142

4243
public NetworkUsageCommand(String privateIP, String domRName, String option, boolean forVpc)
@@ -47,6 +48,25 @@ public NetworkUsageCommand(String privateIP, String domRName, String option, boo
4748
this.forVpc = forVpc;
4849
}
4950

51+
public NetworkUsageCommand(String privateIP, String domRName, boolean forVpc, String gatewayIP, String vpcCIDR)
52+
{
53+
this.privateIP = privateIP;
54+
this.domRName = domRName;
55+
this.forVpc = forVpc;
56+
this.gatewayIP = gatewayIP;
57+
this.option = "create";
58+
this.vpcCIDR = vpcCIDR;
59+
}
60+
61+
public NetworkUsageCommand(String privateIP, String domRName, String option, boolean forVpc, String gatewayIP)
62+
{
63+
this.privateIP = privateIP;
64+
this.domRName = domRName;
65+
this.forVpc = forVpc;
66+
this.gatewayIP = gatewayIP;
67+
this.option = option;
68+
}
69+
5070
public String getPrivateIP() {
5171
return privateIP;
5272
}
@@ -59,8 +79,20 @@ public String getOption() {
5979
return option;
6080
}
6181

82+
public boolean isForVpc() {
83+
return forVpc;
84+
}
85+
86+
public String getVpcCIDR() {
87+
return vpcCIDR;
88+
}
89+
90+
public String getGatewayIP() {
91+
return gatewayIP;
92+
}
93+
6294
@Override
6395
public boolean executeInSequence() {
6496
return false;
6597
}
66-
}
98+
}

‎patches/systemvm/debian/config/opt/cloud/bin/ipsectunnel.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,9 +54,9 @@ start_ipsec() {
5454
enable_iptables_subnets() {
5555
for net in $rightnets
5656
do
57-
sudo iptables -A FORWARD -t mangle -s $leftnet -d $net -j MARK --set-mark $vpnoutmark
57+
sudo iptables -I FORWARD -t mangle -s $leftnet -d $net -j MARK --set-mark $vpnoutmark
5858
sudo iptables -A OUTPUT -t mangle -s $leftnet -d $net -j MARK --set-mark $vpnoutmark
59-
sudo iptables -A FORWARD -t mangle -s $net -d $leftnet -j MARK --set-mark $vpninmark
59+
sudo iptables -I FORWARD -t mangle -s $net -d $leftnet -j MARK --set-mark $vpninmark
6060
sudo iptables -A INPUT -t mangle -s $net -d $leftnet -j MARK --set-mark $vpninmark
6161
done
6262
return 0

‎patches/systemvm/debian/config/opt/cloud/bin/vpc_guestnw.sh‎

Lines changed: 0 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -97,21 +97,6 @@ desetup_dnsmasq() {
9797
sleep 1
9898
}
9999

100-
setup_usage() {
101-
sudo iptables -t mangle -N NETWORK_STATS_$dev
102-
sudo iptables -t mangle -A NETWORK_STATS_$dev -s $subnet/$mask ! -d $vpccidr
103-
sudo iptables -t mangle -A NETWORK_STATS_$dev -o $dev ! -s $vpccidr
104-
sudo iptables -t mangle -A POSTROUTING -s $subnet/$mask -j NETWORK_STATS_$dev
105-
sudo iptables -t mangle -A POSTROUTING -o $dev -j NETWORK_STATS_$dev
106-
}
107-
108-
desetup_usage() {
109-
sudo iptables -t mangle -F NETWORK_STATS_$dev
110-
sudo iptables -t mangle -D POSTROUTING -s $subnet/$mask -j NETWORK_STATS_$dev
111-
sudo iptables -t mangle -D POSTROUTING -o $dev -j NETWORK_STATS_$dev
112-
sudo iptables -t mangle -X NETWORK_STATS_$dev
113-
}
114-
115100
create_guest_network() {
116101
logger -t cloud " $(basename $0): Create network on interface $dev, gateway $gw, network $ip/$mask "
117102
# setup ip configuration
@@ -130,7 +115,6 @@ create_guest_network() {
130115
# set up hairpin
131116
sudo iptables -t nat -A POSTROUTING -s $subnet/$mask -o $dev -j SNAT --to-source $ip
132117
create_acl_chain
133-
setup_usage
134118
setup_dnsmasq
135119
setup_apache2
136120
}
@@ -144,7 +128,6 @@ destroy_guest_network() {
144128
sudo iptables -t mangle -D PREROUTING -i $dev -m state --state ESTABLISHED,RELATED -j CONNMARK --restore-mark
145129
sudo iptables -t nat -A POSTROUTING -s $subnet/$mask -o $dev -j SNAT --to-source $ip
146130
destroy_acl_chain
147-
desetup_usage
148131
desetup_dnsmasq
149132
desetup_apache2
150133
}

‎patches/systemvm/debian/config/opt/cloud/bin/vpc_netusage.sh‎

Lines changed: 80 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@
1515
source /root/func.sh
1616
source /opt/cloud/bin/vpc_func.sh
1717

18+
vpnoutmark="0x525"
19+
vpninmark="0x524"
1820
lock="biglock"
1921
locked=$(getLockFile $lock)
2022
if [ "$locked" != "1" ]
@@ -23,25 +25,64 @@ then
2325
fi
2426

2527
usage() {
26-
printf "Usage: %s -[c|g|r] [-[a|d] <public interface>]\n" $(basename $0) >&2
28+
printf "Usage: %s -[c|g|r|n|d] [-l <public gateway>] [-v <vpc cidr>] \n" $(basename $0) >&2
2729
}
2830

2931
create_usage_rules () {
30-
iptables-save|grep "NETWORK_STATS_$guestIp -i $ethDev" > /dev/null
32+
iptables-save|grep "NETWORK_STATS_$ethDev" > /dev/null
3133
if [ $? -gt 0 ]
3234
then
33-
iptables -A NETWORK_STATS_$guestIp -i $ethDev -s ! zcidr > /dev/null
34-
fi
35-
iptables-save|grep "NETWORK_STATS_$guestIp -o $ethDev" > /dev/null
35+
iptables -N NETWORK_STATS_$ethDev > /dev/null;
36+
iptables -I FORWARD -j NETWORK_STATS_$ethDev > /dev/null;
37+
iptables -A NETWORK_STATS_$ethDev -o $ethDev -s $vcidr > /dev/null;
38+
iptables -A NETWORK_STATS_$ethDev -i $ethDev -d $vcidr > /dev/null;
39+
fi
40+
return $?
41+
}
42+
43+
create_vpn_usage_rules () {
44+
iptables-save|grep "VPN_STATS_$ethDev" > /dev/null
3645
if [ $? -gt 0 ]
3746
then
38-
iptables -A NETWORK_STATS_$guestIp -o $ethDev -d ! zcidr > /dev/null
47+
iptables -t mangle -N VPN_STATS_$ethDev > /dev/null;
48+
iptables -t mangle -I FORWARD -j VPN_STATS_$ethDev > /dev/null;
49+
iptables -t mangle -A VPN_STATS_$ethDev -o $ethDev -m mark --mark $vpnoutmark > /dev/null;
50+
iptables -t mangle -A VPN_STATS_$ethDev -i $ethDev -m mark --mark $vpninmark > /dev/null;
3951
fi
4052
return $?
4153
}
4254

55+
remove_usage_rules () {
56+
echo $ethDev >> /root/removedVifs
57+
return $?
58+
}
59+
4360
get_usage () {
44-
iptables -t mangle -L NETWORK_STATS_$ethDev -n -v -x | awk '$1 ~ /^[0-9]+$/ { printf "%s:", $2}'; > /dev/null
61+
iptables -L NETWORK_STATS_$ethDev -n -v -x 2> /dev/null | awk '$1 ~ /^[0-9]+$/ { printf "%s:", $2}'; > /dev/null
62+
if [ -f /root/removedVifs ]
63+
then
64+
var=`cat /root/removedVifs`
65+
# loop through vifs to be cleared
66+
for i in $var; do
67+
# Make sure vif doesn't exist
68+
if [ ! -f /sys/class/net/$i ]
69+
then
70+
# flush rules and remove chain
71+
iptables -F NETWORK_STATS_$i > /dev/null;
72+
iptables -D FORWARD -j NETWORK_STATS_$i > /dev/null;
73+
iptables -X NETWORK_STATS_$i > /dev/null;
74+
iptables -t mangle -F VPN_STATS_$i > /dev/null;
75+
iptables -t mangle -D FORWARD -j VPN_STATS_$i > /dev/null;
76+
iptables -t mangle -X VPN_STATS_$i > /dev/null;
77+
fi
78+
done
79+
rm /root/removedVifs
80+
fi
81+
return 0
82+
}
83+
84+
get_vpn_usage () {
85+
iptables -t mangle -L VPN_STATS_$ethDev -n -v -x | awk '$1 ~ /^[0-9]+$/ { printf "%s:", $2}'; > /dev/null
4586
if [ $? -gt 0 ]
4687
then
4788
printf $?
@@ -50,7 +91,7 @@ get_usage () {
5091
}
5192

5293
reset_usage () {
53-
iptables -t mangle -Z NETWORK_STATS_$ethDev > /dev/null
94+
iptables -Z NETWORK_STATS_$ethDev > /dev/null
5495
if [ $? -gt 0 -a $? -ne 2 ]
5596
then
5697
return 1
@@ -63,9 +104,11 @@ cflag=
63104
gflag=
64105
rflag=
65106
lflag=
107+
vflag=
108+
nflag=
109+
dflag=
66110

67-
68-
while getopts 'cgrl:' OPTION
111+
while getopts 'cgndrl:v:' OPTION
69112
do
70113
case $OPTION in
71114
c) cflag=1
@@ -75,8 +118,15 @@ do
75118
r) rflag=1
76119
;;
77120
l) lflag=1
78-
guestIp="$OPTARG"
121+
publicIp="$OPTARG"
122+
;;
123+
v) vflag=1
124+
vcidr="$OPTARG"
79125
;;
126+
n) nflag=1
127+
;;
128+
d) dflag=1
129+
;;
80130
i) #Do nothing, since it's parameter for host script
81131
;;
82132
?) usage
@@ -85,18 +135,35 @@ do
85135
esac
86136
done
87137

138+
ethDev=$(getEthByIp $publicIp)
88139
if [ "$cflag" == "1" ]
89140
then
90-
unlock_exit 0 $lock $locked
141+
if [ "$ethDev" != "" ]
142+
then
143+
create_usage_rules
144+
create_vpn_usage_rules
145+
unlock_exit 0 $lock $locked
146+
fi
91147
fi
92148

93-
ethDev=$(getEthByIp $guestIp)
94149
if [ "$gflag" == "1" ]
95150
then
96151
get_usage
97152
unlock_exit $? $lock $locked
98153
fi
99154

155+
if [ "$nflag" == "1" ]
156+
then
157+
get_vpn_usage
158+
unlock_exit $? $lock $locked
159+
fi
160+
161+
if [ "$dflag" == "1" ]
162+
then
163+
remove_usage_rules
164+
unlock_exit 0 $lock $locked
165+
fi
166+
100167
if [ "$rflag" == "1" ]
101168
then
102169
reset_usage

‎plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -491,6 +491,9 @@ protected Answer execute(CheckNetworkCommand cmd) {
491491
}
492492

493493
protected Answer execute(NetworkUsageCommand cmd) {
494+
if ( cmd.isForVpc() ) {
495+
return VPCNetworkUsage(cmd);
496+
}
494497
if (s_logger.isInfoEnabled()) {
495498
s_logger.info("Executing resource NetworkUsageCommand " + _gson.toJson(cmd));
496499
}
@@ -505,6 +508,64 @@ protected Answer execute(NetworkUsageCommand cmd) {
505508
return answer;
506509
}
507510

511+
protected NetworkUsageAnswer VPCNetworkUsage(NetworkUsageCommand cmd) {
512+
String privateIp = cmd.getPrivateIP();
513+
String option = cmd.getOption();
514+
String publicIp = cmd.getGatewayIP();
515+
516+
517+
String args = "-l " + publicIp+ " ";
518+
if (option.equals("get")) {
519+
args += "-g";
520+
} else if (option.equals("create")) {
521+
args += "-c";
522+
String vpcCIDR = cmd.getVpcCIDR();
523+
args += " -v " + vpcCIDR;
524+
} else if (option.equals("reset")) {
525+
args += "-r";
526+
} else if (option.equals("vpn")) {
527+
args += "-n";
528+
} else if (option.equals("remove")) {
529+
args += "-d";
530+
} else {
531+
return new NetworkUsageAnswer(cmd, "success", 0L, 0L);
532+
}
533+
try {
534+
if (s_logger.isTraceEnabled()) {
535+
s_logger.trace("Executing /opt/cloud/bin/vpc_netusage.sh " + args + " on DomR " + privateIp);
536+
}
537+
VmwareManager mgr = getServiceContext().getStockObject(VmwareManager.CONTEXT_STOCK_NAME);
538+
539+
Pair<Boolean, String> resultPair = SshHelper.sshExecute(privateIp, DEFAULT_DOMR_SSHPORT, "root", mgr.getSystemVMKeyFile(), null, "/opt/cloud/bin/vpc_netusage.sh " + args);
540+
541+
if (!resultPair.first()) {
542+
throw new Exception(" vpc network usage plugin call failed ");
543+
}
544+
545+
if (option.equals("get") || option.equals("vpn")) {
546+
String result = resultPair.second();
547+
if (result == null || result.isEmpty()) {
548+
throw new Exception(" vpc network usage get returns empty ");
549+
}
550+
long[] stats = new long[2];
551+
if (result != null) {
552+
String[] splitResult = result.split(":");
553+
int i = 0;
554+
while (i < splitResult.length - 1) {
555+
stats[0] += (new Long(splitResult[i++])).longValue();
556+
stats[1] += (new Long(splitResult[i++])).longValue();
557+
}
558+
return new NetworkUsageAnswer(cmd, "success", stats[0], stats[1]);
559+
}
560+
}
561+
return new NetworkUsageAnswer(cmd, "success", 0L, 0L);
562+
} catch (Throwable e) {
563+
s_logger.error("Unable to execute NetworkUsage command on DomR (" + privateIp + "), domR may not be ready yet. failure due to "
564+
+ VmwareHelper.getExceptionMessage(e), e);
565+
}
566+
return new NetworkUsageAnswer(cmd, "success", 0L, 0L);
567+
}
568+
508569
protected Answer execute(SetPortForwardingRulesCommand cmd) {
509570
if (s_logger.isInfoEnabled()) {
510571
s_logger.info("Executing resource SetPortForwardingRulesCommand: " + _gson.toJson(cmd));

0 commit comments

Comments
 (0)