Skip to content

Commit 68c80e2

Browse files
Harikrishna Patnalamurali-reddy
Harikrishna Patnala
authored and
murali-reddy
committed
CLOUDSTACK-6253: Optimizing VR alerts getting algorithm In addition to this a new configuration parameter is added router.alerts.check.interval defaulted to 30minutes to check for alerts in Virtual Router
1 parent a9accd3 commit 68c80e2

9 files changed

Lines changed: 118 additions & 82 deletions

File tree

‎core/src/com/cloud/agent/api/GetRouterAlertsAnswer.java‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@
2222

2323
public class GetRouterAlertsAnswer extends Answer {
2424

25-
String routerName;
2625
String[] alerts;
2726
String timeStamp;
2827

@@ -36,8 +35,8 @@ public GetRouterAlertsAnswer(GetRouterAlertsCommand cmd, String alerts[], String
3635
}
3736

3837

39-
public GetRouterAlertsAnswer(GetRouterAlertsCommand cmd, Exception ex) {
40-
super(cmd, ex);
38+
public GetRouterAlertsAnswer(GetRouterAlertsCommand cmd, String details) {
39+
super(cmd, false, details);
4140
}
4241

4342
public void setAlerts(String[] alerts) {
@@ -56,7 +55,4 @@ public String getTimeStamp() {
5655
return timeStamp;
5756
}
5857

59-
public String getRouterName() {
60-
return routerName;
61-
}
6258
}

‎core/src/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java‎

Lines changed: 9 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -649,25 +649,23 @@ private CheckS2SVpnConnectionsAnswer execute(CheckS2SVpnConnectionsCommand cmd)
649649

650650
private GetRouterAlertsAnswer execute(GetRouterAlertsCommand cmd) {
651651

652-
String args = null;
653652
String routerIp = cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP);
654-
if (cmd.getPreviousAlertTimeStamp() != null) {
655-
args = cmd.getPreviousAlertTimeStamp();
656-
}
653+
String args = cmd.getPreviousAlertTimeStamp();
657654

658655
ExecutionResult result = _vrDeployer.executeInVR(routerIp, VRScripts.ROUTER_ALERTS, args);
659656
String alerts[] = null;
660657
String lastAlertTimestamp = null;
661-
// CallHostPlugin results "success" when there are no alerts on virtual router
658+
662659
if (result.isSuccess()) {
663-
if (!result.getDetails().isEmpty() && !result.getDetails().equals("No Alerts")) {
664-
alerts = result.getDetails().split("\\\\n");
665-
String[] lastAlert = alerts[alerts.length - 1].split(" ");
666-
lastAlertTimestamp = lastAlert[0] + " " + lastAlert[1];
660+
if (!result.getDetails().isEmpty() && !result.getDetails().trim().equals("No Alerts")) {
661+
alerts = result.getDetails().trim().split("\\\\n");
662+
String[] lastAlert = alerts[alerts.length - 1].split(",");
663+
lastAlertTimestamp = lastAlert[0];
667664
}
665+
return new GetRouterAlertsAnswer(cmd, alerts, lastAlertTimestamp);
666+
} else {
667+
return new GetRouterAlertsAnswer(cmd, result.getDetails());
668668
}
669-
670-
return new GetRouterAlertsAnswer(cmd, alerts, lastAlertTimestamp);
671669
}
672670

673671
protected Answer execute(CheckRouterCommand cmd) {
@@ -760,21 +758,6 @@ protected List<ConfigItem> generateConfig(SetMonitorServiceCommand cmd) {
760758
return cfg;
761759
}
762760

763-
protected List<ConfigItem> generateConfig(GetRouterAlertsCommand cmd) {
764-
LinkedList<ConfigItem> cfg = new LinkedList<>();
765-
766-
String args = null;
767-
String routerIp = cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP);
768-
if (cmd.getPreviousAlertTimeStamp() != null) {
769-
args = "getRouterAlerts.sh " + routerIp + " " + cmd.getPreviousAlertTimeStamp();
770-
} else {
771-
args = "getRouterAlerts.sh " + routerIp;
772-
}
773-
774-
cfg.add(new ConfigItem(VRScripts.ROUTER_ALERTS, args));
775-
return cfg;
776-
}
777-
778761
protected List<ConfigItem> generateConfig(SetupGuestNetworkCommand cmd) {
779762
LinkedList<ConfigItem> cfg = new LinkedList<>();
780763

‎engine/schema/src/com/cloud/vm/dao/DomainRouterDao.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,8 @@ public interface DomainRouterDao extends GenericDao<DomainRouterVO, Long> {
108108
*/
109109
List<DomainRouterVO> listByStateAndNetworkType(State state, Network.GuestType type, long mgmtSrvrId);
110110

111+
List<DomainRouterVO> listByStateAndManagementServer(State state, long mgmtSrvrId);
112+
111113
List<DomainRouterVO> findByNetworkOutsideThePod(long networkId, long podId, State state, Role role);
112114

113115
List<DomainRouterVO> listByNetworkAndPodAndRole(long networkId, long podId, Role role);

‎engine/schema/src/com/cloud/vm/dao/DomainRouterDaoImpl.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ public class DomainRouterDaoImpl extends GenericDaoBase<DomainRouterVO, Long> im
6060
protected SearchBuilder<DomainRouterVO> StateNetworkTypeSearch;
6161
protected SearchBuilder<DomainRouterVO> OutsidePodSearch;
6262
protected SearchBuilder<DomainRouterVO> clusterSearch;
63+
protected SearchBuilder<DomainRouterVO> SearchByStateAndManagementServerId;
6364
@Inject
6465
HostDao _hostsDao;
6566
@Inject
@@ -130,6 +131,14 @@ protected void init() {
130131
StateNetworkTypeSearch.join("host", joinHost, joinHost.entity().getId(), StateNetworkTypeSearch.entity().getHostId(), JoinType.INNER);
131132
StateNetworkTypeSearch.done();
132133

134+
SearchByStateAndManagementServerId = createSearchBuilder();
135+
SearchByStateAndManagementServerId.and("state", SearchByStateAndManagementServerId.entity().getState(), Op.EQ);
136+
137+
SearchBuilder<HostVO> joinHost2 = _hostsDao.createSearchBuilder();
138+
joinHost2.and("mgmtServerId", joinHost2.entity().getManagementServerId(), Op.EQ);
139+
SearchByStateAndManagementServerId.join("host", joinHost2, joinHost2.entity().getId(), SearchByStateAndManagementServerId.entity().getHostId(), JoinType.INNER);
140+
SearchByStateAndManagementServerId.done();
141+
133142
OutsidePodSearch = createSearchBuilder();
134143
SearchBuilder<RouterNetworkVO> joinRouterNetwork2 = _routerNetworkDao.createSearchBuilder();
135144
joinRouterNetwork2.and("networkId", joinRouterNetwork2.entity().getNetworkId(), Op.EQ);
@@ -293,6 +302,15 @@ public List<DomainRouterVO> listByStateAndNetworkType(State state, Network.Guest
293302
return routers;
294303
}
295304

305+
@Override
306+
public List<DomainRouterVO> listByStateAndManagementServer(State state, long mgmtSrvrId) {
307+
SearchCriteria<DomainRouterVO> sc = SearchByStateAndManagementServerId.create();
308+
sc.setParameters("state", state);
309+
sc.setJoinParameters("host", "mgmtServerId", mgmtSrvrId);
310+
311+
return listBy(sc);
312+
}
313+
296314
@Override
297315
public List<DomainRouterVO> findByNetworkOutsideThePod(long networkId, long podId, State state, Role role) {
298316
SearchCriteria<DomainRouterVO> sc = OutsidePodSearch.create();

‎plugins/hypervisors/simulator/src/com/cloud/agent/manager/SimulatorManagerImpl.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@
8686
import com.cloud.agent.api.proxy.WatchConsoleProxyLoadCommand;
8787
import com.cloud.agent.api.routing.AggregationControlCommand;
8888
import com.cloud.agent.api.routing.DhcpEntryCommand;
89+
import com.cloud.agent.api.routing.GetRouterAlertsCommand;
8990
import com.cloud.agent.api.routing.IpAssocCommand;
9091
import com.cloud.agent.api.routing.IpAssocVpcCommand;
9192
import com.cloud.agent.api.routing.LoadBalancerConfigCommand;
@@ -370,6 +371,8 @@ public Answer simulate(Command cmd, String hostGuid) {
370371
return _mockNetworkMgr.setupPVLAN((PvlanSetupCommand)cmd);
371372
} else if (cmd instanceof StorageSubSystemCommand) {
372373
return this.storageHandler.handleStorageCommands((StorageSubSystemCommand)cmd);
374+
} else if (cmd instanceof GetRouterAlertsCommand) {
375+
return new Answer(cmd);
373376
} else if (cmd instanceof VpnUsersCfgCommand || cmd instanceof RemoteAccessVpnCfgCommand || cmd instanceof SetMonitorServiceCommand || cmd instanceof AggregationControlCommand) {
374377
return new Answer(cmd);
375378
} else {

‎server/src/com/cloud/network/router/VirtualNetworkApplianceManager.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ public interface VirtualNetworkApplianceManager extends Manager, VirtualNetworkA
5151
static final String RouterTemplateHyperVCK = "router.template.hyperv";
5252
static final String RouterTemplateLxcCK = "router.template.lxc";
5353
static final String SetServiceMonitorCK = "network.router.EnableServiceMonitoring";
54+
static final String RouterAlertsCheckIntervalCK = "router.alerts.check.interval";
5455

5556
static final ConfigKey<String> RouterTemplateXen = new ConfigKey<String>(String.class, RouterTemplateXenCK, "Advanced", "SystemVM Template (XenServer)",
5657
"Name of the default router template on Xenserver.", true, ConfigKey.Scope.Zone, null);
@@ -66,6 +67,9 @@ public interface VirtualNetworkApplianceManager extends Manager, VirtualNetworkA
6667
static final ConfigKey<String> SetServiceMonitor = new ConfigKey<String>(String.class, SetServiceMonitorCK, "Advanced", "true",
6768
"service monitoring in router enable/disable option, default true", true, ConfigKey.Scope.Zone, null);
6869

70+
static final ConfigKey<Integer> RouterAlertsCheckInterval = new ConfigKey<Integer>(Integer.class, RouterAlertsCheckIntervalCK, "Advanced", "1800",
71+
"Interval (in seconds) to check for alerts in Virtual Router.", false, ConfigKey.Scope.Global, null);
72+
6973
public static final int DEFAULT_ROUTER_VM_RAMSIZE = 128; // 128M
7074
public static final int DEFAULT_ROUTER_CPU_MHZ = 500; // 500 MHz
7175
public static final boolean USE_POD_VLAN = false;

‎server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java‎

Lines changed: 63 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717

1818
package com.cloud.network.router;
1919

20+
import java.text.ParseException;
21+
import java.text.SimpleDateFormat;
2022
import java.util.ArrayList;
2123
import java.util.Arrays;
2224
import java.util.Calendar;
@@ -824,6 +826,13 @@ public boolean start() {
824826
s_logger.debug("router.check.interval - " + _routerCheckInterval + " so not scheduling the redundant router checking thread");
825827
}
826828

829+
int _routerAlertsCheckInterval = RouterAlertsCheckInterval.value();
830+
if (_routerAlertsCheckInterval > 0) {
831+
_checkExecutor.scheduleAtFixedRate(new CheckRouterAlertsTask(), _routerAlertsCheckInterval, _routerAlertsCheckInterval, TimeUnit.SECONDS);
832+
} else {
833+
s_logger.debug("router.alerts.check.interval - " + _routerAlertsCheckInterval + " so not scheduling the router alerts checking thread");
834+
}
835+
827836
return true;
828837
}
829838

@@ -1358,8 +1367,6 @@ protected void runInContext() {
13581367

13591368
updateSite2SiteVpnConnectionState(routers);
13601369

1361-
getRouterAlerts();
1362-
13631370
final List<NetworkVO> networks = _networkDao.listRedundantNetworks();
13641371
s_logger.debug("Found " + networks.size() + " networks to update RvR status. ");
13651372
for (final NetworkVO network : networks) {
@@ -1374,44 +1381,83 @@ protected void runInContext() {
13741381
}
13751382
}
13761383

1377-
private void getRouterAlerts() {
1384+
protected class CheckRouterAlertsTask extends ManagedContextRunnable {
1385+
public CheckRouterAlertsTask() {
1386+
}
1387+
1388+
@Override
1389+
protected void runInContext() {
1390+
try {
1391+
getRouterAlerts();
1392+
} catch (final Exception ex) {
1393+
s_logger.error("Fail to complete the CheckRouterAlertsTask! ", ex);
1394+
}
1395+
}
1396+
}
1397+
1398+
protected void getRouterAlerts() {
13781399
try{
1379-
List<DomainRouterVO> routersInIsolatedNetwork = _routerDao.listByStateAndNetworkType(State.Running, GuestType.Isolated, mgmtSrvrId);
1380-
List<DomainRouterVO> routersInSharedNetwork = _routerDao.listByStateAndNetworkType(State.Running, GuestType.Shared, mgmtSrvrId);
1400+
List<DomainRouterVO> routers = _routerDao.listByStateAndManagementServer(State.Running, mgmtSrvrId);
13811401

1382-
List<DomainRouterVO> routers = new ArrayList<DomainRouterVO>();
1383-
routers.addAll(routersInIsolatedNetwork);
1384-
routers.addAll(routersInSharedNetwork);
13851402
s_logger.debug("Found " + routers.size() + " running routers. ");
13861403

13871404
for (final DomainRouterVO router : routers) {
1388-
if (router.getVpcId() != null) {
1405+
String serviceMonitoringFlag = SetServiceMonitor.valueIn(router.getDataCenterId());
1406+
// Skip the routers in VPC network or skip the routers where Monitor service is not enabled in the corresponding Zone
1407+
if ( !Boolean.parseBoolean(serviceMonitoringFlag) || router.getVpcId() != null) {
13891408
continue;
13901409
}
1410+
13911411
String privateIP = router.getPrivateIpAddress();
13921412

13931413
if (privateIP != null) {
13941414
OpRouterMonitorServiceVO opRouterMonitorServiceVO = _opRouterMonitorServiceDao.findById(router.getId());
13951415

13961416
GetRouterAlertsCommand command = null;
13971417
if (opRouterMonitorServiceVO == null) {
1398-
command = new GetRouterAlertsCommand(null);
1418+
command = new GetRouterAlertsCommand(new String("1970-01-01 00:00:00")); // To avoid sending null value
13991419
} else {
14001420
command = new GetRouterAlertsCommand(opRouterMonitorServiceVO.getLastAlertTimestamp());
14011421
}
14021422

14031423
command.setAccessDetail(NetworkElementCommand.ROUTER_IP, router.getPrivateIpAddress());
1404-
command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName());
14051424

1406-
GetRouterAlertsAnswer answer = null;
14071425
try {
1408-
answer = (GetRouterAlertsAnswer) _agentMgr.easySend(router.getHostId(), command);
1426+
final Answer origAnswer = _agentMgr.easySend(router.getHostId(), command);
1427+
GetRouterAlertsAnswer answer = null;
1428+
1429+
if (origAnswer == null) {
1430+
s_logger.warn("Unable to get alerts from router " + router.getHostName());
1431+
continue;
1432+
}
1433+
if (origAnswer instanceof GetRouterAlertsAnswer) {
1434+
answer = (GetRouterAlertsAnswer)origAnswer;
1435+
} else {
1436+
s_logger.warn("Unable to get alerts from router " + router.getHostName());
1437+
continue;
1438+
}
1439+
if (!answer.getResult()) {
1440+
s_logger.warn("Unable to get alerts from router " + router.getHostName() + " " + answer.getDetails());
1441+
continue;
1442+
}
1443+
14091444
String alerts[] = answer.getAlerts();
1410-
if (alerts != null ) {
1445+
if (alerts != null) {
1446+
String lastAlertTimeStamp = answer.getTimeStamp();
1447+
SimpleDateFormat sdfrmt = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss");
1448+
sdfrmt.setLenient(false);
1449+
try
1450+
{
1451+
sdfrmt.parse(lastAlertTimeStamp);
1452+
}
1453+
catch (ParseException e)
1454+
{
1455+
s_logger.warn("Invalid last alert timestamp received while collecting alerts from router: " + router.getInstanceName());
1456+
continue;
1457+
}
14111458
for (String alert: alerts) {
14121459
_alertMgr.sendAlert(AlertType.ALERT_TYPE_DOMAIN_ROUTER, router.getDataCenterId(), router.getPodIdToDeployIn(), "Monitoring Service on VR " + router.getInstanceName(), alert);
14131460
}
1414-
String lastAlertTimeStamp = answer.getTimeStamp();
14151461
if (opRouterMonitorServiceVO == null) {
14161462
opRouterMonitorServiceVO = new OpRouterMonitorServiceVO(router.getId(), router.getHostName(), lastAlertTimeStamp);
14171463
_opRouterMonitorServiceDao.persist(opRouterMonitorServiceVO);
@@ -1421,7 +1467,7 @@ private void getRouterAlerts() {
14211467
}
14221468
}
14231469
} catch (Exception e) {
1424-
s_logger.warn("Error while collecting alerts from router: " + router.getInstanceName() + " from host: " + router.getHostId(), e);
1470+
s_logger.warn("Error while collecting alerts from router: " + router.getInstanceName(), e);
14251471
continue;
14261472
}
14271473
}
@@ -1431,7 +1477,6 @@ private void getRouterAlerts() {
14311477
}
14321478
}
14331479

1434-
14351480
private final static int DEFAULT_PRIORITY = 100;
14361481
private final static int DEFAULT_DELTA = 2;
14371482

@@ -4333,7 +4378,7 @@ public String getConfigComponentName() {
43334378

43344379
@Override
43354380
public ConfigKey<?>[] getConfigKeys() {
4336-
return new ConfigKey<?>[] {UseExternalDnsServers, routerVersionCheckEnabled, SetServiceMonitor};
4381+
return new ConfigKey<?>[] {UseExternalDnsServers, routerVersionCheckEnabled, SetServiceMonitor, RouterAlertsCheckInterval};
43374382
}
43384383

43394384
@Override

‎systemvm/patches/debian/config/opt/cloud/bin/getRouterAlerts.sh‎

Lines changed: 15 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -18,53 +18,38 @@
1818

1919
# getRouterAlerts.sh --- Send the alerts from routerServiceMonitor.log to Management Server
2020

21-
source /root/func.sh
22-
23-
lock="biglock"
24-
locked=$(getLockFile $lock)
25-
if [ "$locked" != "1" ]
26-
then
27-
exit 1
28-
fi
29-
3021
#set -x
3122

3223
filename=/var/log/routerServiceMonitor.log #Monitor service log file
3324
if [ -n "$1" -a -n "$2" ]
3425
then
35-
reqdateval=$(date -d $1 +"%Y%m%d");
36-
reqtimeval=$(date -d $2 +"%H%M%S");
26+
reqDateVal=$(date -d "$1 $2" "+%s");
3727
else
38-
reqdateval=0
39-
reqtimeval=0
28+
reqDateVal=0
4029
fi
4130
if [ -f $filename ]
4231
then
4332
while read line
4433
do
45-
if [ -n "$line" ]; then
46-
dateval=`echo $line |awk '{print $1}'`
47-
timeval=`echo $line |awk '{print $2}'`
48-
49-
todate=$(date -d "$dateval" +"%Y%m%d") > /dev/null
50-
totime=$(date -d "$timeval" +"%H%M%S") > /dev/null
51-
if [ "$todate" -gt "$reqdateval" ] > /dev/null
34+
if [ -n "$line" ]
5235
then
53-
if [ -n "$alerts" ]; then alerts="$alerts\n$line"; else alerts="$line"; fi #>> $outputfile
54-
elif [ "$todate" -eq "$reqdateval" ] > /dev/null
36+
dateval=`echo $line |awk '{print $1, $2}'`
37+
IFS=',' read -a array <<< "$dateval"
38+
dateval=${array[0]}
39+
40+
toDateVal=$(date -d "$dateval" "+%s")
41+
42+
if [ "$toDateVal" -gt "$reqDateVal" ]
5543
then
56-
if [ "$totime" -gt "$reqtimeval" ] > /dev/null
57-
then
58-
if [ -n "$alerts" ]; then alerts="$alerts\n$line"; else alerts="$line"; fi #>> $outputfile
59-
fi
44+
alerts="$line\n$alerts"
45+
else
46+
break
6047
fi
6148
fi
62-
done < $filename
49+
done < <(tac $filename)
6350
fi
6451
if [ -n "$alerts" ]; then
6552
echo $alerts
6653
else
6754
echo "No Alerts"
68-
fi
69-
70-
unlock_exit 0 $lock $locked
55+
fi

0 commit comments

Comments
 (0)